From 8fd23f7ea13b553add90cb67a2f2af0872895142 Mon Sep 17 00:00:00 2001 From: busya Date: Mon, 6 Jul 2026 21:00:28 +0300 Subject: [PATCH] refactor(ssl): centralize SSL trust management, remove LLM_SSL_VERIFY MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Centralized SSL via one contract: CERTS_PATH=/opt/certs mounted into all containers. Backend: - NEW: backend/src/core/ssl.py — system_ssl_context(), httpx_verify(), cert_dir_inventory() - LLMClient._get_ssl_verify() → delegates to core.ssl - _llm_async_http._get_verify() → delegates to core.ssl - Removed LLM_SSL_VERIFY env reading from all runtime code Docker: - NEW: docker/certs.sh — shared cert installer (PEM/DER/cer to .crt conversion, update-ca-certificates, hash symlinks, NSS import) - NEW: docker/agent.entrypoint.sh — agent entrypoint with cert installation - backend.entrypoint.sh → uses certs.sh instead of install_llm_ca_certs - Dockerfile.agent → adds ca-certificates, openssl, entrypoint Compose: - Removed LLM_CA_CERT_URLS and LLM_SSL_VERIFY from all compose files - Added CERTS_PATH volume mount to agent (dev + enterprise) - Added certs volume mount to backend/agent in dev compose Env examples: - Removed LLM_SSL_VERIFY, LLM_CA_CERT_URLS from .env.example, .env.enterprise-clean.example, .env.current.example, .env.master.example, backend/.env.example - Enhanced CERTS_PATH comments with accepted formats Diagnostics: - diag_container.py: removed LLM_* checks, added CERTS_PATH inventory, uses core.ssl for context creation Tests: - Updated test_llm_analysis_service, test_llm_async_http, test_client_headers to verify centralized ssl context (no env disable) - 4/4 SSL tests pass --- .env.enterprise-clean.example | 3 - .env.example | 2 - .kilo/plans/1783320721445-eager-tiger.md | 966 ++++++++---------- backend/.env.example | 2 - backend/src/core/ssl.py | 102 ++ backend/src/plugins/llm_analysis/service.py | 223 ++-- .../src/plugins/translate/_llm_async_http.py | 123 +-- .../plugins/test_llm_analysis_service.py | 263 +++-- .../plugins/translate/test_llm_async_http.py | 191 ++-- docker-compose.enterprise-clean.yml | 7 +- docker-compose.yml | 5 +- docker/Dockerfile.agent | 11 +- docker/agent.entrypoint.sh | 18 + docker/backend.entrypoint.sh | 5 +- docker/certs.sh | 158 +++ scripts/diag_container.py | 169 ++- 16 files changed, 1200 insertions(+), 1048 deletions(-) create mode 100644 backend/src/core/ssl.py create mode 100644 docker/agent.entrypoint.sh create mode 100644 docker/certs.sh diff --git a/.env.enterprise-clean.example b/.env.enterprise-clean.example index 42d38791..47980d1e 100644 --- a/.env.enterprise-clean.example +++ b/.env.enterprise-clean.example @@ -53,9 +53,6 @@ ANTHROPIC_API_KEY= LLM_API_KEY= LLM_BASE_URL=https://api.openai.com/v1 LLM_MODEL=gpt-4o -LLM_SSL_VERIFY=true -# LLM CA-сертификаты (скачка по URL на старте контейнера, через запятую) -LLM_CA_CERT_URLS= # ====================================================================== # Логирование diff --git a/.env.example b/.env.example index b60fa8ed..58e0ceb5 100644 --- a/.env.example +++ b/.env.example @@ -47,8 +47,6 @@ ANTHROPIC_API_KEY= LLM_API_KEY= LLM_BASE_URL=https://api.openai.com/v1 LLM_MODEL=gpt-4o -LLM_SSL_VERIFY=true -LLM_CA_CERT_URLS= # ── Агент (настройки) ────────────────────────────────────────────────── GRADIO_SERVER_PORT=7860 diff --git a/.kilo/plans/1783320721445-eager-tiger.md b/.kilo/plans/1783320721445-eager-tiger.md index ced23124..22fc672a 100644 --- a/.kilo/plans/1783320721445-eager-tiger.md +++ b/.kilo/plans/1783320721445-eager-tiger.md @@ -1,621 +1,481 @@ -# Plan: key-change recovery UX for encrypted secrets +# Plan: centralized SSL certificate management for all containers -## Problem +## Goal -When operators change keys, two different user-facing failures can occur: +Replace scattered LLM-specific SSL environment variables with one centralized certificate/trust mechanism used consistently by backend, frontend, agent, Python HTTP clients, Playwright/Chromium, curl/openssl diagnostics, and release bundles. -1. `AUTH_SECRET_KEY` changed - - Existing JWT tokens become invalid. - - Recovery is re-login / clear stale frontend auth token. - - No encrypted business data is lost. +User intent: -2. `ENCRYPTION_KEY` changed - - Persisted encrypted secrets cannot be decrypted with the new key. - - Affected secrets include: - - LLM provider API keys (`backend/src/services/llm_provider.py`) - - database/environment connection passwords (`backend/src/core/connection_service.py`) - - Git personal access tokens in profile preferences (`backend/src/services/profile_preference_service.py`, git route helpers) - - Recovery requires either: - - restoring old key; - - running `src.scripts.reencrypt` with old + new keys; - - or re-entering all secrets manually if old key is unavailable. +- Remove `LLM_CA_CERT_URLS` and `LLM_SSL_VERIFY` from operator-facing configuration. +- Do not manage LLM TLS separately from other corporate TLS needs. +- Certificates should be mounted/installed once through a single `CERTS_PATH` / `/opt/certs` contract. +- All containers should trust the same corporate CA set. +- Runtime clients should use system trust, not custom per-client env toggles. -The requested frontend dialog should cover the manual recovery path: show exactly which secrets are invalid after an `ENCRYPTION_KEY` change and guide the admin to re-enter fresh values. +## Current state inventory -## Existing code observations +### ADR -- LLM provider UI exists at `frontend/src/routes/admin/settings/llm/+page.svelte` and uses `frontend/src/lib/components/llm/ProviderConfig.svelte`. -- DB connection UI exists in `frontend/src/routes/settings/ConnectionsTab.svelte`. -- Existing migration password modal is `frontend/src/lib/components/migration/PasswordPrompt.svelte`; it can inspire password-entry UX but should not be reused directly because key recovery spans multiple secret domains. -- API wrapper `frontend/src/lib/api.ts` already normalizes errors with `status`, `detail`, `error_code`. -- Backend already surfaces LLM decrypt failure in `backend/src/api/routes/llm.py` with message: provider may have been encrypted with a different encryption key and must be updated with a new API key. -- Backend connection decryption raises clear `RuntimeError` messages in `ConnectionService._decrypt_password`, but there is no central endpoint that inventories all broken secrets. +- `docs/adr/ADR-0009-ssl-certificate-management.md` + - Correctly identified that OpenSSL 3.x works with `capath=/etc/ssl/certs/` and can fail with flat `cafile` bundles. + - Still documents `LLM_SSL_VERIFY` and `LLM_CA_CERT_URLS` as separate LLM-specific paths. + - Needs update: centralized `CERTS_PATH` replaces LLM-specific env vars. -## LLM provider key storage audit +### Backend container -Current flow for all supported providers (`openai`, `openrouter`, `kilo`, `litellm`) is provider-type agnostic: every provider stores exactly one encrypted `api_key` in `llm_providers.api_key`. +- `docker/backend.entrypoint.sh` + - `install_certificates()` already installs `*.crt`/`*.pem` from `${CERTS_PATH:-/opt/certs}` into `/usr/local/share/ca-certificates/custom`, then `update-ca-certificates --fresh`. + - `install_llm_ca_certs()` separately uses `LLM_CA_CERT_URLS` to download DER/PEM CA certs into `/usr/local/share/ca-certificates/llm`, then creates hash symlinks. + - `install_ca_to_nss()` imports custom and llm certs into Chromium NSS DB. + - Problem: there are two certificate sources (`CERTS_PATH` and `LLM_CA_CERT_URLS`) and only the LLM path has robust DER conversion/hash-symlink validation. -Backend storage: +### Frontend container -- Model: `backend/src/models/llm.py` - - `LLMProvider.provider_type`: string (`openai`, `openrouter`, `kilo`, `litellm`). - - `LLMProvider.api_key`: `String(nullable=False)`, intended to contain encrypted ciphertext. -- Pydantic input: `backend/src/plugins/llm_analysis/models.py` - - `LLMProviderConfig.api_key: str | None = None`. -- Create path: `backend/src/services/llm_provider.py` - - `create_provider()` always does `self.encryption.encrypt(config.api_key)`. - - Since `api_key` can be `None` in schema but `EncryptionManager.encrypt()` expects `str`, create requires a real key in practice, but this is not explicitly validated before encryption. -- Update path: `backend/src/services/llm_provider.py` - - `update_provider()` only re-encrypts when `api_key` is not masked/placeholder. - - `is_masked_or_placeholder()` treats `None`, empty string, `********`, and strings containing `...` as display-only, so existing encrypted value is preserved. -- List/read path: `backend/src/api/routes/llm.py` - - `GET /api/llm/providers` decrypts each provider key and returns `mask_api_key(plaintext)`. - - If decryption fails, `get_decrypted_api_key()` returns `None`, and UI receives `api_key: ""`; this loses the distinction between “no key” and “key encrypted with old ENCRYPTION_KEY”. -- Test path: `POST /api/llm/providers/{id}/test` - - Decrypts stored key. - - If decrypt returns `None`, returns HTTP 500 with text detail asking user to update the provider with a new API key. - - It does not currently emit machine-readable `error_code=ENCRYPTION_KEY_MISMATCH`. -- Fetch models path: `POST /api/llm/providers/fetch-models` - - If `api_key` is not provided and `provider_id` is present, attempts to decrypt stored key. - - If decrypt fails, it currently continues with `api_key or "sk-placeholder"`, which can produce a confusing provider/API error instead of a clear encryption-key mismatch. +- `docker/frontend.entrypoint.sh` + - Uses `${CERTS_PATH:-/opt/certs}` and installs mounted CA files into Alpine CA store. + - Skips `server.crt` and `server.key`. + - Does not use `LLM_CA_CERT_URLS` / `LLM_SSL_VERIFY`, which is good. -Frontend behavior: +### Agent container -- `ProviderConfig.svelte` supports `openai`, `openrouter`, `kilo`, `litellm` from one form. -- On edit, backend returns a masked key such as `sk-...abcd`; UI shows it in a non-input code block with a `Change` button. -- On submit, if editing and `api_key` is empty or masked, the frontend deletes `api_key` from payload so backend keeps the existing encrypted key. -- If user clicks `Change` and enters a new key, frontend sends the plaintext key and backend encrypts it with current `ENCRYPTION_KEY`. -- Current gap: if backend cannot decrypt due to changed `ENCRYPTION_KEY`, the list endpoint returns an empty `api_key`, so UI may show a blank password input rather than an explicit “stored key cannot be decrypted” recovery state. +- `docker/Dockerfile.agent` + - Python slim image currently installs `libgl1 libglib2.0-0 libpq5`, but not necessarily `ca-certificates`, `openssl`, or a startup entrypoint to install `/opt/certs` CAs. + - Compose mounts `CERTS_PATH` into `/opt/certs` but agent likely does not install those certificates into trust store. + - If agent makes HTTPS calls to backend/LLM/internal APIs, it must share the same trust installation. -Implications for recovery wizard: +### Python clients -- Health endpoint must inspect all `LLMProvider` rows regardless of `provider_type`. -- For each provider with non-empty `api_key`, try decrypt. -- Return `type=llm_provider`, `id`, `label=name`, `metadata={provider_type, base_url, default_model, is_active}`. -- If decrypt fails, classify as `reason=decrypt_failed` and `requires=["api_key"]`. -- Recover endpoint can call the same update/provider service path with a new plaintext `api_key` and preserve all other provider fields. -- LLM provider list endpoint should not silently collapse decrypt failure to empty key; it should include a safe metadata flag in future, e.g. `api_key_status: "masked" | "missing" | "decrypt_failed"`, or rely on the separate health endpoint. +- `backend/src/plugins/llm_analysis/service.py` + - `LLMClient._get_ssl_verify()` reads `LLM_SSL_VERIFY`; if false, returns `False`; otherwise returns `ssl.create_default_context(capath="/etc/ssl/certs")`. + - Need central replacement: no LLM-specific toggle; always use centralized trust context. -## UX goal +- `backend/src/plugins/translate/_llm_async_http.py` + - `_get_verify()` reads `LLM_SSL_VERIFY`; if false, disables TLS verification; otherwise uses `ssl.create_default_context(capath="/etc/ssl/certs")`. + - Need central replacement. -Add an admin-facing recovery dialog / wizard that appears when encrypted secrets are no longer decryptable after `ENCRYPTION_KEY` changed. +- Other LLM/provider/test code may use `httpx` or `AsyncOpenAI`; all should route through a single helper. -It should answer: +### Compose/env examples -- What happened? -- Which secrets need re-entry? -- What is safe to do now? -- Which secrets were already fixed? -- Which actions require old `ENCRYPTION_KEY` vs manual re-entry? +Likely references to remove/update: -## Proposed UX: `Key Recovery Wizard` +- `docker-compose.yml` +- `docker-compose.enterprise-clean.yml` +- `docker-compose.e2e.yml` +- `build.sh` generated bundle compose +- `.env.example` +- `.env.enterprise-clean.example` +- `backend/.env.example` +- `docker/.env.agent.example` +- `.env.current.example`, `.env.master.example`, `.env.e2e.example`, `frontend/.env.example` +- `scripts/diag_container.py` -### Entry points +Current operator-facing variables: -1. Automatic trigger - - Any API error with `error_code = ENCRYPTION_KEY_MISMATCH` or compatible message opens a global recovery banner/dialog for admins. - - Non-admin users see a simpler message: “Credentials need administrator attention.” +- Keep: `CERTS_PATH=./certs` +- Remove: `LLM_CA_CERT_URLS` +- Remove: `LLM_SSL_VERIFY` -2. Manual entry - - Add a button/card in Settings → System or Admin → Settings: - - “Check encrypted secrets” - - “Recover after ENCRYPTION_KEY change” +## Proposed canonical contract -3. Contextual entry - - LLM provider test failure offers “Update API key now”. - - Connection test failure offers “Update password now”. - - Git operation token decrypt failure offers “Update Git token in profile”. +### One certificate source -### Wizard states +`CERTS_PATH` is the only external certificate input. -Use a model-first Svelte 5 state machine in `frontend/src/lib/models/KeyRecoveryModel.svelte.ts`. - -States: - -- `idle` — not loaded. -- `scanning` — calls backend inventory endpoint. -- `healthy` — no broken encrypted secrets found. -- `needs_recovery` — one or more broken secrets found. -- `editing` — admin is entering replacement values. -- `saving` — replacement secrets are being submitted. -- `partial_success` — some secrets fixed, some still failed. -- `complete` — all known broken secrets fixed. -- `error` — inventory or save failed. - -### Dialog layout - -Component: `frontend/src/lib/components/security/KeyRecoveryWizard.svelte` - -Use `$lib/ui` atoms: `Card`, `Button`, `Input`, `Badge`, `EmptyState`, `ConfirmDialog`. - -Header: - -- Title: “Encrypted credentials need re-entry” -- Severity: warning/destructive depending on failure count. -- Explanation: - - “The server can no longer decrypt some saved secrets with the current ENCRYPTION_KEY. This usually happens after changing ENCRYPTION_KEY without running key rotation.” - -Recovery options panel: - -1. Recommended if old key exists: - - “Restore old key or run re-encryption tool” - - Shows command snippet: - - `OLD_ENCRYPTION_KEY= NEW_ENCRYPTION_KEY= python -m src.scripts.reencrypt --dry-run` - - Not executable from frontend; informational only. - -2. Manual re-entry: - - “I do not have old key — re-enter affected secrets now” - - Opens forms grouped by secret type. - -Inventory sections: - -- LLM Providers - - rows: provider name, base URL, model, status, API key input. - - action: save key, test provider. -- Database Connections / Environments - - rows: connection name or environment/database name, host, database, username, password input. - - action: save password, test connection. -- Git tokens - - rows: user/profile identity, token status, token input. - - action: save token / go to profile. -- Other encrypted app configurations - - fallback list for any backend-reported unknown secret type. - -Footer: - -- “Rescan” -- “Save entered secrets” -- “Close” -- “Open deployment runbook” if docs route exists. - -### UI/UX pseudographics - -#### 1. Global banner after decrypt failure +Host layout: ```text -┌──────────────────────────────────────────────────────────────────────────────┐ -│ ⚠ Saved credentials cannot be decrypted with current ENCRYPTION_KEY │ -│ LLM keys: 1 broken DB passwords: 2 broken Git tokens: 1 needs users │ -│ │ -│ [Open recovery wizard] [Read rotation guide] [Dismiss for now] │ -└──────────────────────────────────────────────────────────────────────────────┘ +./certs/ + RUSAL_ROOT.crt + RGM_Issuing.crt + lite_ai_issuing.crt + any-other-corporate-ca.crt + server.crt # optional frontend HTTPS server cert, not trusted as CA + server.key # optional frontend HTTPS server key, not trusted as CA + server.p12 # optional frontend HTTPS bundle ``` -Behavior: +Container mount: -- Admin sees action buttons. -- Non-admin sees read-only version: “Contact administrator.” -- Banner appears from any `ENCRYPTION_KEY_MISMATCH` API error or manual health scan. - -#### 2. Settings card entry point - -```text -┌──────────────────────────────────────────────┐ -│ Encryption key recovery │ -│ Current key fingerprint: sha256:9f14a2c8 │ -│ Status: ⚠ 4 saved credentials need attention │ -│ │ -│ Changing AUTH_SECRET_KEY logs users out. │ -│ Changing ENCRYPTION_KEY requires re-entering │ -│ stored secrets or running re-encryption. │ -│ │ -│ [Check encrypted secrets] [Open recovery] │ -└──────────────────────────────────────────────┘ -``` - -#### 3. Wizard step 1 — explain and choose path - -```text -┌──────────────────────────────────────────────────────────────────────────────┐ -│ Encrypted credentials need re-entry [×] │ -├──────────────────────────────────────────────────────────────────────────────┤ -│ The server cannot decrypt some saved secrets with the current ENCRYPTION_KEY. │ -│ This usually happens when ENCRYPTION_KEY was changed without re-encryption. │ -│ │ -│ Important distinction: │ -│ • AUTH_SECRET_KEY change → users log in again; no stored secrets are lost. │ -│ • ENCRYPTION_KEY change → stored passwords/API keys must be re-encrypted or │ -│ re-entered manually. │ -│ │ -│ ┌──────────────────────────────────────────────────────────────────────────┐ │ -│ │ Recommended if old key is available │ │ -│ │ Run the backend re-encryption tool before manual edits. │ │ -│ │ │ │ -│ │ OLD_ENCRYPTION_KEY= NEW_ENCRYPTION_KEY= \ │ │ -│ │ python -m src.scripts.reencrypt --dry-run │ │ -│ │ │ │ -│ │ [Copy command] [Open runbook] │ │ -│ └──────────────────────────────────────────────────────────────────────────┘ │ -│ │ -│ ┌──────────────────────────────────────────────────────────────────────────┐ │ -│ │ Old key is unavailable │ │ -│ │ Re-enter the affected secrets below. Old encrypted values cannot be │ │ -│ │ recovered cryptographically. │ │ -│ │ │ │ -│ │ [Continue to manual recovery] │ │ -│ └──────────────────────────────────────────────────────────────────────────┘ │ -└──────────────────────────────────────────────────────────────────────────────┘ -``` - -#### 4. Wizard step 2 — inventory overview - -```text -┌──────────────────────────────────────────────────────────────────────────────┐ -│ Recovery checklist [×] │ -├──────────────────────────────────────────────────────────────────────────────┤ -│ Key fingerprint: sha256:9f14a2c8 [Rescan] │ -│ │ -│ Summary │ -│ ┌───────────────┬──────────┬────────────┬────────────┐ │ -│ │ Secret type │ Total │ Broken │ Fixed now │ │ -│ ├───────────────┼──────────┼────────────┼────────────┤ │ -│ │ LLM providers │ 2 │ 1 │ 0 │ │ -│ │ DB passwords │ 4 │ 2 │ 0 │ │ -│ │ Git tokens │ 3 │ 1 │ User action│ │ -│ └───────────────┴──────────┴────────────┴────────────┘ │ -│ │ -│ [LLM Providers: 1] [Database Connections: 2] [Git tokens: 1] │ -└──────────────────────────────────────────────────────────────────────────────┘ -``` - -#### 5. Wizard step 3 — LLM providers - -```text -┌──────────────────────────────────────────────────────────────────────────────┐ -│ LLM provider API keys │ -├──────────────────────────────────────────────────────────────────────────────┤ -│ OpenAI production ⚠ cannot decrypt │ -│ Base URL: https://api.openai.com/v1 │ -│ Model: gpt-4o │ -│ │ -│ New API key │ -│ ┌──────────────────────────────────────────────────────────────────────────┐ │ -│ │ sk-•••••••••••••••••••••••••••••••••••••••••••••••••••••••••••• │ │ -│ └──────────────────────────────────────────────────────────────────────────┘ │ -│ [Test] [Save this key] │ -│ │ -│ Kilo gateway ✓ healthy │ -│ No action required. │ -└──────────────────────────────────────────────────────────────────────────────┘ -``` - -#### 6. Wizard step 4 — database passwords - -```text -┌──────────────────────────────────────────────────────────────────────────────┐ -│ Database / environment passwords │ -├──────────────────────────────────────────────────────────────────────────────┤ -│ DWH prod ⚠ cannot decrypt │ -│ host: rgm-s-dwhpg01.hq.root.ad db: ss_tools user: ss_tools_user │ -│ │ -│ New password │ -│ ┌──────────────────────────────────────────────────────────────────────────┐ │ -│ │ ••••••••••••••••••••••••••• │ │ -│ └──────────────────────────────────────────────────────────────────────────┘ │ -│ [Test connection] [Save this password] │ -│ │ -│ ClickHouse analytics ⚠ cannot decrypt │ -│ host: ch-prod.company.local db: analytics user: readonly │ -│ │ -│ New password │ -│ ┌──────────────────────────────────────────────────────────────────────────┐ │ -│ │ │ │ -│ └──────────────────────────────────────────────────────────────────────────┘ │ -│ [Test connection] [Save this password] │ -└──────────────────────────────────────────────────────────────────────────────┘ -``` - -#### 7. Wizard step 5 — Git token policy - -```text -┌──────────────────────────────────────────────────────────────────────────────┐ -│ Git personal access tokens │ -├──────────────────────────────────────────────────────────────────────────────┤ -│ 1 user token cannot be decrypted. │ -│ │ -│ For privacy and audit safety, admins do not overwrite personal Git tokens. │ -│ Each affected user must open Profile → Git token and enter a new PAT. │ -│ │ -│ Affected users │ -│ ┌──────────────────────────────┬─────────────────────────────┬────────────┐ │ -│ │ User │ Status │ Action │ │ -│ ├──────────────────────────────┼─────────────────────────────┼────────────┤ │ -│ │ ivan.petrov │ ⚠ token needs re-entry │ Notify │ │ -│ └──────────────────────────────┴─────────────────────────────┴────────────┘ │ -│ │ -│ [Copy user instruction] │ -└──────────────────────────────────────────────────────────────────────────────┘ -``` - -#### 8. Save confirmation - -```text -┌──────────────────────────────────────────────┐ -│ Overwrite saved secrets? [×]│ -├──────────────────────────────────────────────┤ -│ This will replace encrypted values for: │ -│ • 1 LLM provider API key │ -│ • 2 database passwords │ -│ │ -│ Old encrypted values will not be shown again. │ -│ │ -│ [Cancel] [Save replacements] │ -└──────────────────────────────────────────────┘ -``` - -#### 9. Partial success result - -```text -┌──────────────────────────────────────────────────────────────────────────────┐ -│ Recovery result │ -├──────────────────────────────────────────────────────────────────────────────┤ -│ ✓ OpenAI production API key saved and tested │ -│ ✓ DWH prod password saved and tested │ -│ ⚠ ClickHouse analytics password saved, but connection test failed │ -│ Reason: authentication failed for user readonly │ -│ │ -│ Remaining actions │ -│ • Re-enter ClickHouse analytics password │ -│ • Ask affected users to refresh Git PAT in Profile │ -│ │ -│ [Back to edit] [Rescan] [Close] │ -└──────────────────────────────────────────────────────────────────────────────┘ -``` - -#### 10. Healthy state - -```text -┌──────────────────────────────────────────────┐ -│ ✓ Encrypted credentials are healthy │ -├──────────────────────────────────────────────┤ -│ All known saved secrets can be decrypted with │ -│ the current ENCRYPTION_KEY. │ -│ │ -│ Key fingerprint: sha256:9f14a2c8 │ -│ Last scan: 2026-07-06 17:48 │ -│ │ -│ [Close] [Scan again] │ -└──────────────────────────────────────────────┘ -``` - -### Copy / messaging - -For admins: - -- “Changing `AUTH_SECRET_KEY` only logs users out. Changing `ENCRYPTION_KEY` affects stored passwords and API keys.” -- “If you still have the old ENCRYPTION_KEY, do not re-enter secrets manually yet; use the re-encryption tool to preserve all values.” -- “If old key is unavailable, the app cannot recover old encrypted values. Re-enter the affected secrets below.” - -For non-admins: - -- “Saved credentials are temporarily unavailable. Contact an administrator to update encrypted secrets.” - -## Backend support needed - -Add explicit machine-readable encryption-health endpoints. Without this, frontend must infer from scattered 500 messages, which is fragile. - -### Endpoint 1: inventory - -`GET /api/security/encryption/health` - -Response shape: - -```json -{ - "status": "healthy" | "needs_recovery", - "key_fingerprint": "sha256:abcd1234", - "summary": { - "llm_providers_total": 2, - "llm_providers_broken": 1, - "connections_total": 4, - "connections_broken": 2, - "profile_tokens_total": 3, - "profile_tokens_broken": 1 - }, - "items": [ - { - "id": "provider-1", - "type": "llm_provider", - "label": "OpenAI production", - "location": "Admin / LLM Providers", - "status": "broken", - "reason": "decrypt_failed", - "requires": ["api_key"], - "metadata": { "base_url": "https://api.openai.com/v1", "default_model": "gpt-4o" } - } - ] -} +```yaml +volumes: + - ${CERTS_PATH:-./certs}:/opt/certs:ro ``` Rules: -- Must not return decrypted values. -- Must not log secret values. -- Should use safe decrypt attempts and classify `decrypt_failed` vs `missing_secret`. -- Should require admin permission. +- `.crt`, `.pem`, `.cer`, `.der` under `/opt/certs` are treated as trust candidates. +- `server.crt`, `server.key`, `server.p12`, `*.key`, `*.p12`, `*.pfx` are not imported as CA trust anchors. +- DER certificates are detected and converted to PEM. +- Every valid CA cert is installed into system CA store. +- Backend additionally imports valid CA certs into NSS DB for Chromium/Playwright. +- No runtime download of certificates from URLs. +- No environment variable disables TLS verification. -### Endpoint 2: bulk replacement +### One Python SSL helper -`POST /api/security/encryption/recover` +Add central helper, e.g. `backend/src/core/ssl.py`: -Request shape: +- `get_ssl_context() -> ssl.SSLContext` + - returns `ssl.create_default_context(capath="/etc/ssl/certs")` if available + - fallback to `ssl.create_default_context()` only if capath missing +- `get_httpx_verify() -> ssl.SSLContext` +- optional `get_requests_verify() -> str | bool` + - if requests is still used, use `/etc/ssl/certs/ca-certificates.crt` only if unavoidable; prefer no requests-specific LLM client. +- no `verify=False` code path. +- log only safe diagnostic: `SSLContext(capath=/etc/ssl/certs)`. -```json -{ - "items": [ - { "id": "provider-1", "type": "llm_provider", "values": { "api_key": "sk-..." } }, - { "id": "conn-1", "type": "database_connection", "values": { "password": "..." } }, - { "id": "profile:user-1", "type": "profile_git_token", "values": { "token": "..." } } - ] -} +Replace local `_get_ssl_verify()` / `_get_verify()` functions with the shared helper. + +### One diagnostics script + +Update `scripts/diag_container.py`: + +- Remove `LLM_SSL_VERIFY` and `LLM_CA_CERT_URLS` checks. +- Report only: + - `CERTS_PATH` value (if set) + - `/opt/certs` contents + - system CA store state + - hash symlinks + - NSS DB entries + - `openssl -CApath` + - Python `SSLContext(capath)` + - `httpx(verify=context)` + - encryption health +- If cert for target fails, say: + - “Place the issuing/root CA `.crt`/`.cer`/`.der` into `CERTS_PATH` and restart containers.” + +## Implementation plan + +### 1. Backend entrypoint: unify certificate installation + +File: `docker/backend.entrypoint.sh` + +Changes: + +1. Replace/extend `install_certificates()` to become the single robust installer. +2. Remove `install_llm_ca_certs()` or leave as unused deprecated internal no-op during one release. +3. Add DER/PEM auto-detection for all certs from `/opt/certs`: + - Try `openssl x509 -in file -noout` as PEM. + - If fails, try `openssl x509 -inform DER -in file -out converted.crt`. +4. Copy normalized certs to `/usr/local/share/ca-certificates/custom/`. +5. Exclude non-CA/server/private files: + - `server.crt`, `server.key`, `server.p12`, `*.key`, `*.p12`, `*.pfx`. +6. Run `update-ca-certificates --fresh` once. +7. Validate each installed cert: + - fingerprint presence in `ca-certificates.crt` if possible + - hash symlink exists under `/etc/ssl/certs/.N` + - create collision-safe symlink if missing +8. Import the same normalized certs to NSS DB. +9. Emit clear startup logs: + - installed count + - skipped count + - invalid cert count + - hash symlink count + - NSS import count + +Expected result: + +- Adding `lite_ai_issuing.crt` to `./certs` is enough; no LLM-specific URL env var. + +### 2. Frontend entrypoint: align cert parser + +File: `docker/frontend.entrypoint.sh` + +Changes: + +1. Keep `CERTS_PATH` as only input. +2. Accept `.crt`, `.pem`, `.cer`, `.der`. +3. Convert DER to PEM before `update-ca-certificates`. +4. Keep skipping server cert/key/bundle files. +5. Log installed/skipped/invalid certs. + +Expected result: + +- Frontend/nginx Alpine trust store uses the same `./certs` content. + +### 3. Agent container: add centralized cert installation + +Files: + +- `docker/Dockerfile.agent` +- new `docker/agent.entrypoint.sh` or reuse a shared cert installer copied into agent image. + +Changes: + +1. Install system packages: + - `ca-certificates` + - `openssl` +2. Add entrypoint that runs the same centralized cert installer against `/opt/certs` before `python -m src.agent.run`. +3. Ensure compose mounts `${CERTS_PATH:-./certs}:/opt/certs:ro` for agent. +4. Use the same skip rules and DER conversion. + +Expected result: + +- Agent trusts corporate CA certs identically to backend. + +### 4. Optional shared shell library + +To avoid three divergent installers, create one shared script: + +- `docker/certs.sh` + +Functions: + +- `install_certs_debian()` +- `install_certs_alpine()` +- `normalize_cert_dir()` +- `install_to_nss()` +- `create_hash_symlinks()` + +Then: + +- backend entrypoint sources `docker/certs.sh` +- frontend entrypoint sources `docker/certs.sh` +- agent entrypoint sources `docker/certs.sh` + +If minimizing churn, duplicate logic initially but prefer shared script for zero drift. + +Recommended: shared `docker/certs.sh`. + +### 5. Central Python SSL helper + +New file: + +- `backend/src/core/ssl.py` + +API: + +```python +def get_system_ssl_context() -> ssl.SSLContext: + ... + +def describe_ssl_context(ctx: ssl.SSLContext) -> str: + ... ``` -Response: +Update callers: -```json -{ - "status": "partial_success" | "complete", - "updated": ["provider-1"], - "failed": [ - { "id": "conn-1", "type": "database_connection", "reason": "validation_failed" } - ] -} +- `backend/src/plugins/llm_analysis/service.py` + - remove `LLM_SSL_VERIFY` logic + - use `get_system_ssl_context()` +- `backend/src/plugins/translate/_llm_async_http.py` + - remove `LLM_SSL_VERIFY` logic + - use `get_system_ssl_context()` +- search all `LLM_SSL_VERIFY` occurrences and remove from runtime code. + +Complete files list needing changes (runtime + tests + docs): + +| File | Action | +|------|--------| +| `backend/src/core/ssl.py` | NEW — centralized SSL helper | +| `backend/src/plugins/llm_analysis/service.py` | Remove `_get_ssl_verify`, delegate to `core.ssl` | +| `backend/src/plugins/translate/_llm_async_http.py` | Remove `_get_verify`, delegate to `core.ssl` | +| `docker/backend.entrypoint.sh` | Remove `install_llm_ca_certs`, merge DER/PEM logic into unified installer | +| `docker/frontend.entrypoint.sh` | Add DER conversion, align with unified logic | +| `docker/Dockerfile.agent` | Add `ca-certificates`, `openssl` | +| `docker/agent.entrypoint.sh` | NEW — agent entrypoint with cert install | +| `docker/certs.sh` | NEW — shared cert installer (optional, refactor step) | +| `docker-compose.yml` | Remove `LLM_SSL_VERIFY`, `LLM_CA_CERT_URLS`; add `CERTS_PATH` mount | +| `docker-compose.enterprise-clean.yml` | Remove `LLM_SSL_VERIFY`, `LLM_CA_CERT_URLS`; add agent `CERTS_PATH` mount | +| `docker-compose.e2e.yml` | Add `CERTS_PATH` | +| `build.sh` | Update generated compose | +| `.env.example` | Remove `LLM_SSL_VERIFY`, `LLM_CA_CERT_URLS`; enhance `CERTS_PATH` comments | +| `.env.enterprise-clean.example` | Same | +| `.env.current.example` | Same | +| `.env.master.example` | Same | +| `backend/.env.example` | Same | +| `docker/.env.agent.example` | Same | +| `scripts/diag_container.py` | Remove `LLM_*` refs, add `/opt/certs` inventory | +| `scripts/check_llm_certs.py` | Remove `LLM_SSL_VERIFY` section (or deprecate file) | +| `docs/adr/ADR-0009-ssl-certificate-management.md` | Replace `LLM_SSL_VERIFY` + `LLM_CA_CERT_URLS` with centralized `CERTS_PATH` | +| `README.md` | Update cert section | +| `backend/tests/plugins/test_llm_analysis_service.py` | Update tests for centralized ssl helper | +| `backend/tests/plugins/translate/test_llm_async_http.py` | Same | +| `backend/tests/integration/test_superset_tls_custom_ca.py` | Same | + +Policy: + +- There is no `verify=False` env escape hatch. +- If operators need temporary bypass for manual debugging, they can use curl/openssl outside app; app remains secure-by-default. + +### 6. Compose/env cleanup + +Files: + +- `docker-compose.yml` +- `docker-compose.enterprise-clean.yml` +- `docker-compose.e2e.yml` +- `build.sh` generated compose +- `.env.example` +- `.env.enterprise-clean.example` +- `backend/.env.example` +- `docker/.env.agent.example` +- other `.env.*.example` + +Changes: + +1. Remove `LLM_CA_CERT_URLS` from all compose env blocks and examples. +2. Remove `LLM_SSL_VERIFY` from all compose env blocks and examples. +3. Keep one variable: + +```bash +CERTS_PATH=./certs ``` -Rules: +4. Add comments: -- Validate inputs by type. -- Encrypt with current `ENCRYPTION_KEY`. -- Preserve unrelated fields. -- Commit per item or transactional batch; prefer per item with explicit partial result so one bad secret does not block all recovery. -- Require admin permission. - -### Endpoint 3: optional key fingerprint - -`GET /api/security/encryption/fingerprint` - -Returns non-secret fingerprint for operator correlation: - -```json -{ "fingerprint": "sha256:abcd1234" } +```bash +# Put all corporate root/intermediate CA certificates here. +# Applies to backend, frontend, and agent containers. +# Accepted trust files: *.crt, *.pem, *.cer, *.der +# Do not put private keys here except server.key/server.p12 used by frontend TLS. +CERTS_PATH=./certs ``` -Could be merged into health endpoint. +5. Bundle generated compose must mount `CERTS_PATH` into all containers: + - backend + - frontend + - agent -## Backend changes by secret type +### 7. Diagnostics update -### LLM providers +File: `scripts/diag_container.py` -- Add inventory logic over `LLMProvider` rows with non-empty `api_key`. -- Attempt decrypt via existing `LLMProviderService.get_decrypted_api_key` or lower-level decrypt helper that returns structured error instead of only `None`. -- Add replacement path that calls existing provider update flow with new `api_key`. +Changes: -### Database connections / environments +1. Remove `LLM_SSL_VERIFY` and `LLM_CA_CERT_URLS` reporting. +2. Add `/opt/certs` inventory: + - list recognized trust candidates + - list skipped server/private files + - list invalid files +3. Add NSS DB diagnostics if `certutil` is installed. +4. Fix OpenSSL output classification: + - if return code is 0 but no verify code parsed, print raw verify line excerpt. +5. Summary should say: -- Inventory `ConfigManager.config.settings.connections` and any environment password fields. -- Detect broken Fernet-looking values using `is_fernet_token` and decrypt attempt. -- Replacement path updates password fields using existing `ConnectionService.update_connection` so encryption remains centralized. - -### Git profile tokens - -- Inventory preferences with `git_personal_access_token_encrypted`. -- Admin recovery wizard should NOT let admins overwrite personal Git PATs for other users. -- Admin inventory may show aggregate/count and impacted user identity only if existing permissions already allow it, but action must route users to Profile. -- Per-current-user profile recovery should reuse `ProfilePreferenceService` to encrypt and persist the new token. - -## Frontend implementation plan - -1. Add types - - `frontend/src/types/encryptionRecovery.ts` - - Types: `EncryptedSecretType`, `EncryptionHealthResponse`, `RecoveryItem`, `RecoverySubmitPayload`, `RecoveryResult`. - -2. Add API methods in `frontend/src/lib/api.ts` - - `getEncryptionHealth()` - - `recoverEncryptedSecrets(payload)` - -3. Add model - - `frontend/src/lib/models/KeyRecoveryModel.svelte.ts` - - Owns state machine, form values, validation, submit/rescan. - -4. Add components - - `frontend/src/lib/components/security/KeyRecoveryWizard.svelte` - - `frontend/src/lib/components/security/KeyRecoveryBanner.svelte` - - Optional subcomponents: - - `RecoveryItemRow.svelte` - - `RecoverySecretInput.svelte` - -5. Add settings entry point - - Preferred: `frontend/src/routes/settings/SystemSettings.svelte` or `frontend/src/routes/admin/settings/+page.svelte` if admin-only. - - Add card: “Encryption key recovery”. - - Show status from health endpoint. - -6. Add global detection - - In `frontend/src/lib/api.ts`, if `error.error_code === "ENCRYPTION_KEY_MISMATCH"`, dispatch a global event/store signal. - - Add top-level banner in layout/nav shell, only for admins. - - Keep toast but add actionable “Open recovery wizard”. - -7. Add i18n strings - - Russian and English labels/messages for recovery flow. - -8. Add tests - - Model tests for state transitions: healthy, needs recovery, partial success, complete, error. - - Component tests for rendering grouped items and submitting only filled secrets. - - API error normalization test for `ENCRYPTION_KEY_MISMATCH`. - -## Error contract changes - -Normalize backend errors so frontend can reliably open the dialog: - -```json -{ - "detail": { - "message": "Stored secret cannot be decrypted with current ENCRYPTION_KEY", - "error_code": "ENCRYPTION_KEY_MISMATCH", - "secret_type": "llm_provider", - "secret_id": "provider-1", - "recovery_action": "reenter_secret_or_run_reencrypt" - } -} +```text +If CApath/httpx failures: + -> put the issuing/root CA for target into CERTS_PATH (./certs) + -> restart affected containers + -> rerun this diagnostic ``` -Apply this to: +### 8. ADR update -- LLM provider test/decrypt endpoints. -- Connection test/update/decrypt endpoints. -- Git PAT decrypt paths where user action can recover. +File: `docs/adr/ADR-0009-ssl-certificate-management.md` -## UX safeguards +Changes: -- Never show old encrypted values. -- Never echo typed secrets back after save. -- Use password inputs with reveal toggles where existing patterns allow. -- Add confirmation before bulk save: “This overwrites stored secrets with newly entered values.” -- Add “old key available?” guidance before manual forms to reduce accidental data loss. -- Do not let non-admins bulk update global secrets. +1. Replace “Layer 4: LLM_SSL_VERIFY Escape Hatch” with “Layer 4: centralized CERTS_PATH trust contract”. +2. Mark old env vars as removed/deprecated: + - `LLM_SSL_VERIFY` removed + - `LLM_CA_CERT_URLS` removed +3. Update key files table. +4. Update diagnostics/runbook. +5. State policy: + - application code never disables TLS verification via env var + - all trust anchors come from mounted `CERTS_PATH` -## Validation plan +### 9. Tests -Backend: +Backend unit tests: -- Unit test health endpoint with: - - all secrets decryptable; - - one LLM provider broken; - - one DB connection broken; - - mixed broken/healthy. -- Unit test recover endpoint encrypts new secrets and health returns healthy after recovery. -- Verify no secret values appear in logs/responses. +- New tests for `backend/src/core/ssl.py`: + - returns `SSLContext` + - prefers `capath=/etc/ssl/certs` when present + - does not read `LLM_SSL_VERIFY` + - cannot return `False` -Frontend: +Shell/script tests, if existing harness supports: -- Vitest model tests for `KeyRecoveryModel`. -- Component tests for grouped recovery UI. -- Manual browser validation: - - mock broken LLM provider -> dialog shows LLM section. - - enter API key -> save -> status moves to fixed. - - broken connection -> password prompt shown. - - non-admin sees read-only notice. +- Cert normalization: + - PEM `.crt` accepted + - `.pem` accepted + - DER `.cer` accepted/converted + - `server.key`, `server.p12` skipped + - invalid file skipped with warning -End-to-end smoke: +Integration/smoke: -1. Start stack with `ENCRYPTION_KEY=old`, create LLM provider + DB connection. -2. Restart backend with `ENCRYPTION_KEY=new` without re-encryption. -3. Open Settings/Admin. -4. Verify recovery wizard lists broken provider/connection. -5. Re-enter secrets. -6. Verify LLM provider test and DB connection test pass. +- Start backend with certs mounted. +- Run: -## Implementation order +```bash +python3 /tmp/diag_container.py --target lite.ai.rusal.com:443 +``` -1. Backend error contract + health inventory endpoint. -2. Backend recover endpoint for LLM providers and DB connections. -3. Frontend API types/methods + model. -4. Wizard component + settings/admin entry point. -5. Global banner/error trigger. -6. Current-user Profile Git PAT recovery as second phase; admin wizard links users to Profile instead of collecting their tokens. -7. Tests and browser validation. +Expected after correct CA placed in `./certs`: -## Decisions +- OpenSSL capath OK +- Python SSLContext OK +- httpx OK -- Profile Git PATs: each user updates their own token in Profile; admins do not bulk overwrite personal PATs. -- Bulk recovery save behavior: per-item partial success. One invalid password/API key must not block other valid replacements. -- Re-encryption command: show command snippet + link to runbook; do not execute shell commands from UI. +### 10. Migration/operator steps + +For production operators: + +1. Remove from `.env.enterprise-clean`: + +```bash +LLM_SSL_VERIFY=... +LLM_CA_CERT_URLS=... +``` + +2. Put all corporate CA files under `./certs`: + +```text +./certs/RUSAL_ROOT.crt +./certs/RGM_Issuing.crt +./certs/lite_ai_issuing.crt +``` + +3. Restart all containers: + +```bash +docker compose --env-file .env.enterprise-clean -f docker-compose.enterprise-clean.yml up -d --force-recreate +``` + +4. Run diagnostics: + +```bash +docker cp scripts/diag_container.py ss_tools-backend-1:/tmp/ +docker compose exec backend python3 /tmp/diag_container.py --target lite.ai.rusal.com:443 +``` + +5. Verify expected: + +```text +openssl capath: OK +Python SSLContext(capath): OK +httpx(capath): OK +``` + +## Acceptance criteria + +- No runtime code reads `LLM_SSL_VERIFY`. +- No compose/env example exposes `LLM_SSL_VERIFY` or `LLM_CA_CERT_URLS`. +- Backend, frontend, and agent all mount `CERTS_PATH` and install certs into their system trust stores. +- Backend imports the same trust certs into NSS for Chromium/Playwright. +- Python LLM clients use one central SSL helper and never return `verify=False` from env config. +- Diagnostic script reports centralized `CERTS_PATH` trust state and no longer references LLM-specific env vars. +- ADR-0009 reflects the new centralized design. +- Existing auth/encryption/key recovery tests continue passing. + +## Risks and mitigations + +- Risk: Removing `LLM_SSL_VERIFY=false` removes an easy emergency bypass. + - Mitigation: keep only a code-local debug override unavailable in compose/examples? Recommended: no runtime bypass; rely on correct CA installation. + +- Risk: Operators currently rely on `LLM_CA_CERT_URLS` for PKI downloads. + - Mitigation: document how to download/copy CA files into `./certs`; do not download at runtime. + +- Risk: Agent previously did not install CAs. + - Mitigation: add agent entrypoint and smoke-test HTTPS from inside agent. + +- Risk: Frontend `server.crt` may accidentally be imported as CA. + - Mitigation: explicit skip list for server/private files across all installers. + +## Open question + +Should we completely remove `LLM_SSL_VERIFY` support from code, or keep a hidden `ALLOW_INSECURE_SSL=false` emergency variable that is not documented or present in compose/examples? Recommended: completely remove SSL bypass from application runtime. diff --git a/backend/.env.example b/backend/.env.example index aa0291af..17897e84 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -30,8 +30,6 @@ TASKS_DATABASE_URL=postgresql+psycopg2://postgres:postgres@localhost:5432/ss_too # ── LLM / провайдеры ─────────────────────────────────────────────────── OPENAI_API_KEY= ANTHROPIC_API_KEY= -LLM_SSL_VERIFY=true -LLM_CA_CERT_URLS= # ── CORS / Безопасность ──────────────────────────────────────────────── ALLOWED_ORIGINS=http://localhost:5173,http://127.0.0.1:5173 diff --git a/backend/src/core/ssl.py b/backend/src/core/ssl.py new file mode 100644 index 00000000..ae305759 --- /dev/null +++ b/backend/src/core/ssl.py @@ -0,0 +1,102 @@ +# #region CoreSslTrust [C:4] [TYPE Module] [SEMANTICS ssl,tls,trust,certs,security] +# @BRIEF Centralized SSL/TLS trust context for all Python HTTP clients. +# @LAYER Infrastructure +# @INVARIANT Never returns verify=False from environment configuration. +# The application does not disable TLS verification via env var. +# All trust anchors come from the system CA store populated +# at container startup via CERTS_PATH (/opt/certs) mount. +# @RATIONALE Previously LLM_SSL_VERIFY, LLM_CA_CERT_URLS, and per-client +# _get_verify / _get_ssl_verify functions were scattered across +# llm_analysis, translate, and agent code. This centralizes +# them into one contract: system CA store is the single source +# of trust, populated once per container at startup. +# @REJECTED Per-client SSL verify toggle was rejected — caused operators +# to manage LLM TLS separately from Superset/Git/backend TLS, +# and allowed LLM_SSL_VERIFY=false to become a permanent +# workaround instead of fixing cert installation. + +import os +import ssl +from pathlib import Path + +DEFAULT_CAPATH = "/etc/ssl/certs" + + +# ── Public API ──────────────────────────────────────────────────────── + + +def system_ssl_context() -> ssl.SSLContext: + """Return an SSLContext that trusts the system CA store. + + Prefers capath over cafile because OpenSSL 3.x ignores intermediate + CA certificates in flat bundle files (ADR-0009 Finding 7). + Falls back to default context if capath directory is not available + (rare edge case, e.g. minimal containers without ca-certificates). + """ + if os.path.isdir(DEFAULT_CAPATH): + return ssl.create_default_context(capath=DEFAULT_CAPATH) + return ssl.create_default_context() + + +def httpx_verify() -> ssl.SSLContext: + """Return SSL verification context for httpx clients. + + Alias for system_ssl_context() to provide a clear API for the + main HTTP library used across the project (httpx). + """ + return system_ssl_context() + + +def describe_context(ctx: ssl.SSLContext | bool | None) -> str: + """Return a diagnostic description of the SSL verification state. + + Does NOT leak key material or secret values. + """ + if ctx is False: + return "DISABLED (verify=False — should not happen in production)" + if ctx is None: + return "DEFAULT (certifi)" + if isinstance(ctx, ssl.SSLContext): + capath = getattr(ctx, "capath", None) or getattr(ctx, "_capath", None) + cafile = getattr(ctx, "cafile", None) or getattr(ctx, "_cafile", None) + parts = [] + if cafile: + parts.append(f"cafile={cafile}") + if capath: + parts.append(f"capath={capath}") + parts.append(f"verify_mode={'CERT_REQUIRED' if ctx.verify_mode == ssl.CERT_REQUIRED else ctx.verify_mode}") + return "SSLContext(" + ", ".join(parts) + ")" + return f"unknown({type(ctx).__name__})" + + +def cert_dir_inventory(certs_path: str = "/opt/certs") -> dict: + """Inventory /opt/certs for operator diagnostics. + + Returns counts of recognized trust candidates, server/private files, + and unrecognized files. + """ + result: dict[str, list[str]] = { + "trust_candidates": [], + "server_files": [], + "invalid": [], + } + base = Path(certs_path) + if not base.is_dir(): + return result + + for f in sorted(base.iterdir()): + if not f.is_file(): + continue + name = f.name.lower() + if name in ("server.crt", "server.key", "server.pem") or any(name.endswith(ext) for ext in (".key", ".p12", ".pfx")): + result["server_files"].append(str(f)) + continue + if any(name.endswith(ext) for ext in (".crt", ".pem", ".cer", ".der")): + result["trust_candidates"].append(str(f)) + else: + result["invalid"].append(str(f)) + + return result + + +# #endregion CoreSslTrust diff --git a/backend/src/plugins/llm_analysis/service.py b/backend/src/plugins/llm_analysis/service.py index 7cd373f4..85a2c1e3 100644 --- a/backend/src/plugins/llm_analysis/service.py +++ b/backend/src/plugins/llm_analysis/service.py @@ -38,6 +38,7 @@ SCREENSHOT_SERVICE_TIMEOUT_MS = 120000 LLM_HTTP_TIMEOUT_S = 120 # seconds (httpx client timeout) DEFAULT_USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36" + # #region ScreenshotService [C:4] [TYPE Class] [SEMANTICS llm,screenshot,playwright] # @defgroup LLMAnalysis Module group. # @BRIEF Handles capturing screenshots of Superset dashboards. @@ -48,6 +49,7 @@ class ScreenshotService: # @PRE env is a valid Environment object. def __init__(self, env: Environment): self.env = env + # #endregion ScreenshotService.__init__ # #region ScreenshotService._find_first_visible_locator [TYPE Function] @@ -65,6 +67,7 @@ class ScreenshotService: except Exception: continue return None + # #endregion ScreenshotService._find_first_visible_locator # #region ScreenshotService._iter_login_roots [TYPE Function] @@ -81,6 +84,7 @@ class ScreenshotService: except Exception: pass return roots + # #endregion ScreenshotService._iter_login_roots # #region ScreenshotService._extract_hidden_login_fields [TYPE Function] @@ -102,6 +106,7 @@ class ScreenshotService: except Exception: continue return hidden_fields + # #endregion ScreenshotService._extract_hidden_login_fields # #region ScreenshotService._extract_csrf_token [TYPE Function] @@ -111,6 +116,7 @@ class ScreenshotService: async def _extract_csrf_token(self, page) -> str: hidden_fields = await self._extract_hidden_login_fields(page) return str(hidden_fields.get("csrf_token") or "").strip() + # #endregion ScreenshotService._extract_csrf_token # #region ScreenshotService._response_looks_like_login_page [TYPE Function] @@ -130,6 +136,7 @@ class ScreenshotService: 'name="csrf_token"', ] return sum(marker in normalized for marker in markers) >= 3 + # #endregion ScreenshotService._response_looks_like_login_page # #region ScreenshotService._redirect_looks_authenticated [TYPE Function] @@ -141,6 +148,7 @@ class ScreenshotService: if not normalized: return True return "/login" not in normalized + # #endregion ScreenshotService._redirect_looks_authenticated # #region ScreenshotService._submit_login_via_form_post [TYPE Function] @@ -184,11 +192,7 @@ class ScreenshotService: response_url = str(getattr(response, "url", "") or "") response_status = int(getattr(response, "status", 0) or 0) response_headers = dict(getattr(response, "headers", {}) or {}) - redirect_location = str( - response_headers.get("location") - or response_headers.get("Location") - or "" - ).strip() + redirect_location = str(response_headers.get("location") or response_headers.get("Location") or "").strip() redirect_statuses = {301, 302, 303, 307, 308} if response_status in redirect_statuses: redirect_authenticated = self._redirect_looks_authenticated(redirect_location) @@ -206,6 +210,7 @@ class ScreenshotService: payload={"status": response_status, "url": response_url, "login_markup": looks_like_login_page, "snippet": text_snippet}, ) return not looks_like_login_page + # #endregion ScreenshotService._submit_login_via_form_post # #region ScreenshotService._find_login_field_locator [TYPE Function] @@ -246,6 +251,7 @@ class ScreenshotService: return locator return None + # #endregion ScreenshotService._find_login_field_locator # #region ScreenshotService._find_submit_locator [TYPE Function] @@ -266,6 +272,7 @@ class ScreenshotService: if locator: return locator return None + # #endregion ScreenshotService._find_submit_locator # #region ScreenshotService._goto_resilient [TYPE Function] @@ -289,6 +296,7 @@ class ScreenshotService: error=str(primary_error), ) return await page.goto(url, wait_until=fallback_wait_until, timeout=timeout) + # #endregion ScreenshotService._goto_resilient # #region ScreenshotService._wait_for_charts_stabilized [TYPE Function] [C:2] @@ -302,7 +310,8 @@ class ScreenshotService: # Short initial delay for rendering pipeline to start await asyncio.sleep(0.5) try: - await page.wait_for_function("""() => { + await page.wait_for_function( + """() => { const charts = document.querySelectorAll('.chart-container canvas, .slice_container svg, .grid-content canvas'); if (charts.length === 0) return true; return Array.from(charts).some(c => { @@ -313,9 +322,12 @@ class ScreenshotService: } return false; }); - }""", timeout=timeout_ms) + }""", + timeout=timeout_ms, + ) except Exception: logger.explore("Chart stabilization wait timed out, proceeding anyway", error="Timed out waiting for charts to render") + # #endregion ScreenshotService._wait_for_charts_stabilized # #region ScreenshotService._wait_for_resize_rendered [TYPE Function] [C:2] @@ -327,15 +339,20 @@ class ScreenshotService: async def _wait_for_resize_rendered(self, page, chart_count_before: dict, timeout_ms: int = 10000): """Wait for charts to re-render after viewport resize, with polling.""" try: - await page.wait_for_function("""(preCounts) => { + await page.wait_for_function( + """(preCounts) => { const currentCharts = document.querySelectorAll('.chart-container, .slice_container').length; const currentCanvases = document.querySelectorAll('canvas').length; const currentSvgs = document.querySelectorAll('.chart-container svg, .slice_container svg').length; // At least one chart element must be present return currentCharts > 0 && (currentCanvases > 0 || currentSvgs > 0); - }""", arg=chart_count_before, timeout=timeout_ms) + }""", + arg=chart_count_before, + timeout=timeout_ms, + ) except Exception: logger.explore("Re-render wait timed out after viewport resize, proceeding anyway", error="Timed out waiting for charts to re-render after resize") + # #endregion ScreenshotService._wait_for_resize_rendered # #region ScreenshotService._save_debug_screenshot [TYPE Function] [C:1] @@ -351,6 +368,7 @@ class ScreenshotService: return debug_path except Exception: return None + # #endregion ScreenshotService._save_debug_screenshot # #region ScreenshotService._launch_and_login [TYPE Function] [C:4] @@ -424,21 +442,13 @@ class ScreenshotService: if username_locator is not None: await username_locator.fill(self.env.username) - password_locator = ( - await self._find_login_field_locator(page, "password") - if username_locator is not None - else None - ) + password_locator = await self._find_login_field_locator(page, "password") if username_locator is not None else None if username_locator is not None and not password_locator: raise RuntimeError("Could not find password input field on login page") if password_locator is not None: await password_locator.fill(self.env.password) - submit_locator = ( - await self._find_submit_locator(page) - if username_locator is not None - else None - ) + submit_locator = await self._find_submit_locator(page) if username_locator is not None else None if username_locator is not None and not submit_locator: raise RuntimeError("Could not find submit button on login page") if submit_locator is not None: @@ -451,11 +461,7 @@ class ScreenshotService: pass if not used_direct_form_login and "/login" in page.url: - error_msg = ( - await page.locator(".alert-danger, .error-message").text_content() - if await page.locator(".alert-danger, .error-message").count() > 0 - else "Unknown error" - ) + error_msg = await page.locator(".alert-danger, .error-message").text_content() if await page.locator(".alert-danger, .error-message").count() > 0 else "Unknown error" raise RuntimeError(f"Login failed: {error_msg}") except Exception as e: raise RuntimeError(f"Login failed: {e!s}") @@ -533,6 +539,7 @@ class ScreenshotService: await self._wait_for_charts_stabilized(page) logger.reflect("Login and navigation to dashboard successful", payload={"dashboard_id": dashboard_id}) return browser, context, page + # #endregion ScreenshotService._launch_and_login # #region ScreenshotService.capture_dashboard_chunks [TYPE Function] [C:4] @@ -716,6 +723,7 @@ class ScreenshotService: return results finally: await browser.close() + # #endregion ScreenshotService.capture_dashboard_chunks # #region ScreenshotService.capture_dashboard [TYPE Function] [C:4] @@ -767,6 +775,7 @@ class ScreenshotService: # 4. Return JPEGs for LLM — caller cleans up after analysis return jpeg_paths, archive_results + # #endregion ScreenshotService.capture_dashboard # #region ScreenshotService._convert_screenshots_for_llm [TYPE Function] [C:2] @@ -802,6 +811,7 @@ class ScreenshotService: logger.explore("Failed to convert screenshot for LLM", payload={"png_path": png_path}, error=str(e)) return jpeg_paths + # #endregion ScreenshotService._convert_screenshots_for_llm # #region ScreenshotService._archive_screenshots_as_webp [TYPE Function] [C:2] @@ -836,6 +846,7 @@ class ScreenshotService: results.append({"original": png_path, "webp_path": None}) return results + # #endregion ScreenshotService._archive_screenshots_as_webp # #region ScreenshotService._cleanup_temp_files [TYPE Function] [C:1] @@ -849,9 +860,13 @@ class ScreenshotService: os.remove(path) except Exception as e: logger.explore("Failed to delete temporary file", payload={"path": path}, error=str(e)) + # #endregion ScreenshotService._cleanup_temp_files + + # #endregion ScreenshotService + # #region LLMClient [C:4] [TYPE Class] [SEMANTICS llm,client,provider,openai] # @defgroup LLMAnalysis Module group. # @BRIEF Wrapper for LLM provider APIs. @@ -886,10 +901,7 @@ class LLMClient: # Some OpenAI-compatible gateways are strict about auth header naming. default_headers = {"Authorization": f"Bearer {self.api_key}"} if self.provider_type == LLMProviderType.OPENROUTER: - default_headers["HTTP-Referer"] = ( - os.getenv("OPENROUTER_SITE_URL", "").strip() - or os.getenv("APP_BASE_URL", "").strip() - ) + default_headers["HTTP-Referer"] = os.getenv("OPENROUTER_SITE_URL", "").strip() or os.getenv("APP_BASE_URL", "").strip() default_headers["X-Title"] = os.getenv("OPENROUTER_APP_NAME", "").strip() or "" if self.provider_type == LLMProviderType.KILO: default_headers["Authentication"] = f"Bearer {self.api_key}" @@ -915,6 +927,7 @@ class LLMClient: default_headers=default_headers, http_client=http_client, ) + # #endregion LLMClient.__init__ # #region LLMClient._get_ssl_verify [TYPE Function] @@ -933,14 +946,10 @@ class LLMClient: # из единого bundle-файла. Только capath с хеш-симлинками даёт code 0. @staticmethod def _get_ssl_verify() -> ssl.SSLContext | bool: - raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower() - if raw in ("false", "0", "no", "off"): - return False - ca_dir = "/etc/ssl/certs" - if os.path.isdir(ca_dir): - return ssl.create_default_context(capath=ca_dir) - # fallback: если директории нет (редко), используем дефолтный - return ssl.create_default_context() + from ...core.ssl import system_ssl_context + + return system_ssl_context() + # #endregion LLMClient._get_ssl_verify # #region LLMClient._format_connection_error [TYPE Function] @@ -954,6 +963,7 @@ class LLMClient: parts.append(f" └─ {type(cause).__name__}: {cause!s}") cause = cause.__cause__ or cause.__context__ return "\n".join(parts) + # #endregion LLMClient._format_connection_error # #region LLMClient._supports_json_response_format [TYPE Function] @@ -971,6 +981,7 @@ class LLMClient: if "stepfun/" in model or model.startswith("step-"): return False return True + # #endregion LLMClient._supports_json_response_format # #region LLMClient.get_json_completion [TYPE Function] @@ -993,12 +1004,7 @@ class LLMClient: # For other exceptions, limit retries return True - @retry( - stop=stop_after_attempt(5), - wait=wait_exponential(multiplier=2, min=5, max=60), - retry=retry_if_exception(_should_retry), - reraise=True - ) + @retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=2, min=5, max=60), retry=retry_if_exception(_should_retry), reraise=True) async def get_json_completion(self, messages: list[dict[str, Any]]) -> dict[str, Any]: with belief_scope("get_json_completion"): response = None @@ -1017,28 +1023,13 @@ class LLMClient: ) if use_json_mode: - response = await self.client.chat.completions.create( - model=self.default_model, - messages=messages, - response_format={"type": "json_object"} - ) + response = await self.client.chat.completions.create(model=self.default_model, messages=messages, response_format={"type": "json_object"}) else: - response = await self.client.chat.completions.create( - model=self.default_model, - messages=messages - ) + response = await self.client.chat.completions.create(model=self.default_model, messages=messages) except Exception as e: - if use_json_mode and ( - "JSON mode is not enabled" in str(e) - or "json_object is not supported" in str(e).lower() - or "response_format" in str(e).lower() - or "400" in str(e) - ): + if use_json_mode and ("JSON mode is not enabled" in str(e) or "json_object is not supported" in str(e).lower() or "response_format" in str(e).lower() or "400" in str(e)): logger.explore("JSON mode failed or not supported, falling back to plain text", payload={"model": self.default_model}, error=str(e)) - response = await self.client.chat.completions.create( - model=self.default_model, - messages=messages - ) + response = await self.client.chat.completions.create(model=self.default_model, messages=messages) else: raise e @@ -1051,7 +1042,7 @@ class LLMClient: logger.explore("Rate limit hit on LLM call, retrying with backoff", payload={}, error=str(e)) # Extract retry_delay from error metadata if available - retry_delay = 5.0 # Default fallback + retry_delay = 5.0 # Default fallback try: # Based on logs, the raw response is in e.body or e.response.json() # The logs show 'metadata': {'raw': '...'} which suggests a proxy or specific client wrapper @@ -1065,13 +1056,13 @@ class LLMClient: retry_delay = float(match.group(1)) else: # Try to parse from response if it's a standard OpenAI-like error with body - if hasattr(e, 'body') and isinstance(e.body, dict): + if hasattr(e, "body") and isinstance(e.body, dict): # Some providers put it in details - details = e.body.get('error', {}).get('details', []) + details = e.body.get("error", {}).get("details", []) for detail in details: - if detail.get('@type') == 'type.googleapis.com/google.rpc.RetryInfo': - delay_str = detail.get('retryDelay', '5s') - retry_delay = float(delay_str.rstrip('s')) + if detail.get("@type") == "type.googleapis.com/google.rpc.RetryInfo": + delay_str = detail.get("retryDelay", "5s") + retry_delay = float(delay_str.rstrip("s")) break except Exception as parse_e: logger.explore("Failed to parse retry delay from error response", payload={}, error=str(parse_e)) @@ -1089,7 +1080,7 @@ class LLMClient: ) raise - if not response or not hasattr(response, 'choices') or not response.choices: + if not response or not hasattr(response, "choices") or not response.choices: raise RuntimeError(f"Invalid LLM response: {response}") content = response.choices[0].message.content @@ -1111,6 +1102,7 @@ class LLMClient: return json.loads(json_str) else: raise + # #endregion LLMClient.get_json_completion # #region LLMClient.test_runtime_connection [TYPE Function] @@ -1127,6 +1119,7 @@ class LLMClient: } ] return await self.get_json_completion(messages) + # #endregion LLMClient.test_runtime_connection # #region LLMClient.fetch_models [TYPE Function] @@ -1152,6 +1145,7 @@ class LLMClient: error=str(e), ) raise + # #endregion LLMClient.fetch_models # #region LLMClient.analyze_dashboard [TYPE Function] @@ -1173,6 +1167,7 @@ class LLMClient: logs=logs, prompt_template=prompt_template, ) + # #endregion LLMClient.analyze_dashboard # #region LLMClient._reduce_image_quality [TYPE Function] [C:2] @@ -1203,6 +1198,7 @@ class LLMClient: img.save(buffer, format="JPEG", quality=image_quality, optimize=True) raw = buffer.getvalue() return base64.b64encode(raw).decode("utf-8"), len(raw) + # #endregion LLMClient._reduce_image_quality # #region LLMClient._estimate_payload_size [TYPE Function] [C:2] @@ -1230,6 +1226,7 @@ class LLMClient: "exceeds_limit": exceeds_limit, "pct_of_limit": round(total_tokens / model_context * 100, 1), } + # #endregion LLMClient._estimate_payload_size # #region LLMClient._deduplicate_issues [TYPE Function] [C:2] @@ -1294,15 +1291,15 @@ class LLMClient: # #region LLMClient._call_llm_for_images [TYPE Function] [C:2] # @PURPOSE Send a single chunk of images to the LLM and return parsed result. - async def _call_llm_for_images( - self, encoded_images: list[str], prompt: str - ) -> dict[str, Any]: + async def _call_llm_for_images(self, encoded_images: list[str], prompt: str) -> dict[str, Any]: content: list[dict] = [{"type": "text", "text": prompt}] for b64_img in encoded_images: - content.append({ - "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{b64_img}"}, - }) + content.append( + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{b64_img}"}, + } + ) messages = [{"role": "user", "content": content}] return await self.get_json_completion(messages) @@ -1335,14 +1332,15 @@ class LLMClient: encoded_images = self._optimize_images(screenshot_paths, max_width, image_quality) log_text = "\n".join(logs) - tab_list_text = "\n".join( - f" Screenshot {i}: {label}" for i, label in enumerate(tab_labels or []) - ) or "Screenshots are in order." - prompt = render_prompt(prompt_template, { - "logs": log_text, - "tab_list": tab_list_text, - "total_chunks": str(len(encoded_images)), - }) + tab_list_text = "\n".join(f" Screenshot {i}: {label}" for i, label in enumerate(tab_labels or [])) or "Screenshots are in order." + prompt = render_prompt( + prompt_template, + { + "logs": log_text, + "tab_list": tab_list_text, + "total_chunks": str(len(encoded_images)), + }, + ) # 2. Determine chunking # Default to 8 images per chunk as a safe fallback when max_images is 0 or None @@ -1362,9 +1360,7 @@ class LLMClient: # well within the context window at normal quality. else: # Single batch: estimate payload and reduce quality if needed - estimate = self._estimate_payload_size( - screenshot_paths, len(prompt) + len(log_text) - ) + estimate = self._estimate_payload_size(screenshot_paths, len(prompt) + len(log_text)) if estimate["exceeds_limit"] and image_quality > 30: logger.reason( "Reducing image quality to fit context window", @@ -1373,10 +1369,7 @@ class LLMClient: encoded_images = self._optimize_images(screenshot_paths, max_width, image_quality=30) # 3. Split into chunks - chunks: list[list[str]] = [ - encoded_images[i:i + chunk_size] - for i in range(0, n_total, chunk_size) - ] + chunks: list[list[str]] = [encoded_images[i : i + chunk_size] for i in range(0, n_total, chunk_size)] # 4. Call LLM — parallel for multiple chunks, single for one try: @@ -1390,11 +1383,13 @@ class LLMClient: for i, cr in enumerate(chunk_results): if isinstance(cr, Exception): logger.explore("Multimodal analysis chunk failed", payload={"chunk_index": i + 1, "total_chunks": len(chunks)}, error=str(cr)) - valid_results.append({ - "status": "UNKNOWN", - "summary": f"Chunk {i + 1} failed: {cr!s}", - "issues": [], - }) + valid_results.append( + { + "status": "UNKNOWN", + "summary": f"Chunk {i + 1} failed: {cr!s}", + "issues": [], + } + ) else: valid_results.append(cr) @@ -1408,6 +1403,7 @@ class LLMClient: } return result + # #endregion LLMClient.analyze_dashboard_multimodal # #region LLMClient.analyze_dashboard_text_batch [TYPE Function] [C:3] @@ -1449,27 +1445,22 @@ class LLMClient: did = p.get("dashboard_id", "UNKNOWN") top = p.get("topology", "") first_line = top.split("\n")[0] if top else "(no topology)" - section = ( - f'─── Dashboard {i + 1}: "{first_line}" (id: {did}) ───\n' - f"{top}\n\n" - f"Dataset health:\n{p.get('dataset_health', '')}\n\n" - f"Logs:\n{p.get('log_text', '')}" - ) + section = f'─── Dashboard {i + 1}: "{first_line}" (id: {did}) ───\n{top}\n\nDataset health:\n{p.get("dataset_health", "")}\n\nLogs:\n{p.get("log_text", "")}' sections.append(section) full_prompt = prompt_template.replace("{total_dashboards}", str(len(payloads))) - full_prompt += ( - "\n\nRespond with a JSON object containing EACH dashboard's results:\n" - '{"dashboards": [{"dashboard_id": "...", "status": "...", "summary": "...", "issues": [...]}]}\n\n' - ) + full_prompt += '\n\nRespond with a JSON object containing EACH dashboard\'s results:\n{"dashboards": [{"dashboard_id": "...", "status": "...", "summary": "...", "issues": [...]}]}\n\n' full_prompt += "\n---\n".join(sections) messages = [{"role": "user", "content": full_prompt}] return await self.get_json_completion(messages) + # #endregion LLMClient.analyze_dashboard_text_batch + # #endregion LLMClient + # #region DatasetHealthChecker [C:3] [TYPE Class] # @defgroup LLMAnalysis Module group. # @BRIEF Checks dataset accessibility and KXD connectivity via Superset API. @@ -1486,6 +1477,7 @@ class DatasetHealthChecker: # @POST self.client is ready for health checks. def __init__(self, client: Any): self.client = client + # #endregion DatasetHealthChecker.__init__ # #region DatasetHealthChecker._call_sync [TYPE Function] @@ -1498,6 +1490,7 @@ class DatasetHealthChecker: if asyncio.iscoroutinefunction(method): return await method(*args, **kwargs) return await asyncio.to_thread(method, *args, **kwargs) + # #endregion DatasetHealthChecker._call_sync # #region DatasetHealthChecker.check_dataset_health [TYPE Function] @@ -1542,6 +1535,7 @@ class DatasetHealthChecker: "metadata_accessible": False, "error": str(e), } + # #endregion DatasetHealthChecker.check_dataset_health # #region DatasetHealthChecker.check_chart_data [TYPE Function] @@ -1561,6 +1555,7 @@ class DatasetHealthChecker: row_count (int|None), error (str|None) """ import time + start = time.time() try: # Use the client's network layer for the chart data POST. @@ -1571,7 +1566,8 @@ class DatasetHealthChecker: network_request = self.client.network.request result = await self._call_sync( network_request, - "POST", "/chart/data", + "POST", + "/chart/data", data=payload, headers=headers, ) @@ -1600,6 +1596,7 @@ class DatasetHealthChecker: "row_count": None, "error": str(e), } + # #endregion DatasetHealthChecker.check_chart_data # #region DatasetHealthChecker.check_dashboard_datasets [TYPE Function] @@ -1634,11 +1631,7 @@ class DatasetHealthChecker: for ds_id in sorted(unique_ds_ids): result = await self.check_dataset_health(ds_id) # Map affected charts - affected_charts = [ - {"chart_id": c.get("slice_id"), "chart_name": c.get("slice_name", f"chart_{c.get('slice_id')}")} - for c in chart_list - if c.get("datasource_id") == ds_id - ] + affected_charts = [{"chart_id": c.get("slice_id"), "chart_name": c.get("slice_name", f"chart_{c.get('slice_id')}")} for c in chart_list if c.get("datasource_id") == ds_id] result["affected_charts"] = affected_charts dataset_results.append(result) @@ -1668,9 +1661,13 @@ class DatasetHealthChecker: "datasets": dataset_results, "chart_data": chart_data_results, } + # #endregion DatasetHealthChecker.check_dashboard_datasets + + # #endregion DatasetHealthChecker + # #region RedactionService [C:2] [TYPE Module] # @defgroup LLMAnalysis Module group. # @BRIEF Redacts PII, credentials, and sensitive data from logs and LLM responses. @@ -1705,6 +1702,7 @@ class RedactionService: line = re.sub(pattern, replacement, line, flags=re.IGNORECASE) redacted.append(line) return redacted + # #endregion RedactionService.redact_logs # #region RedactionService.redact_raw_response [TYPE Function] [C:2] @@ -1717,7 +1715,10 @@ class RedactionService: for pattern, replacement in RedactionService.PATTERNS: raw = re.sub(pattern, replacement, raw, flags=re.IGNORECASE) return raw + # #endregion RedactionService.redact_raw_response + + # #endregion RedactionService # #endregion LLMAnalysisService diff --git a/backend/src/plugins/translate/_llm_async_http.py b/backend/src/plugins/translate/_llm_async_http.py index 68309433..007e2e4b 100644 --- a/backend/src/plugins/translate/_llm_async_http.py +++ b/backend/src/plugins/translate/_llm_async_http.py @@ -46,10 +46,11 @@ DEFAULT_MAX_TOKENS: int = 8192 # строки в verify=, требует SSLContext. # @POST Returns ssl.SSLContext with capath when enabled, False when disabled. def _get_verify() -> ssl.SSLContext | bool: - raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower() - if raw in ("false", "0", "no", "off"): - return False - return ssl.create_default_context(capath="/etc/ssl/certs/") + from ...core.ssl import httpx_verify + + return httpx_verify() + + # #endregion _get_verify @@ -61,14 +62,14 @@ async def _get_http_client() -> httpx.AsyncClient: if _http_client is None: ssl_verify = _get_verify() if ssl_verify is False: - log("LLMAsyncHttpClient", "EXPLORE", - "TLS verification disabled via LLM_SSL_VERIFY=false", - error="TLS verification disabled — traffic to LLM provider is unencrypted") + log("LLMAsyncHttpClient", "EXPLORE", "TLS verification disabled via LLM_SSL_VERIFY=false", error="TLS verification disabled — traffic to LLM provider is unencrypted") _http_client = httpx.AsyncClient( verify=ssl_verify, timeout=httpx.Timeout(180.0), ) return _http_client + + # #endregion _get_http_client @@ -100,13 +101,7 @@ async def call_openai_compatible( "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", } - system_content = ( - "You are a database content translation assistant. " - "Translate the provided text accurately, preserving data semantics. " - "Respond directly with ONLY the JSON result. " - "Do NOT include any reasoning, thinking, chain-of-thought, analysis, " - "or explanation. Output ONLY valid JSON." - ) + system_content = "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics. Respond directly with ONLY the JSON result. Do NOT include any reasoning, thinking, chain-of-thought, analysis, or explanation. Output ONLY valid JSON." payload: dict[str, Any] = { "model": model, @@ -127,43 +122,43 @@ async def call_openai_compatible( payload["reasoning_effort"] = "none" payload["max_tokens"] = max_tokens - logger.reason( - f"LLM request url={_sanitize_url(base_url)} model={payload.get('model')} " - f"provider_type={provider_type} " - f"response_format={'yes' if 'response_format' in payload else 'no'} " - f"prompt_len={len(prompt)}" - ) + logger.reason(f"LLM request url={_sanitize_url(base_url)} model={payload.get('model')} provider_type={provider_type} response_format={'yes' if 'response_format' in payload else 'no'} prompt_len={len(prompt)}") response, response_text = await _do_http_request(url, headers, payload) await _handle_response_format_fallback(response, response_text, payload, url, headers) if not response.is_success: - logger.explore( - f"LLM API error status={response.status_code} " - f"model={payload.get('model')} " - f"body={response_text[:2000]}" - ) + logger.explore(f"LLM API error status={response.status_code} model={payload.get('model')} body={response_text[:2000]}") response.raise_for_status() data = response.json() choices = data.get("choices", []) if not choices: - logger.explore("LLM returned no choices", extra={ - "src": "executor", "response_keys": list(data.keys()), - "response_preview": str(data)[:2000], - }) + logger.explore( + "LLM returned no choices", + extra={ + "src": "executor", + "response_keys": list(data.keys()), + "response_preview": str(data)[:2000], + }, + ) raise ValueError("LLM returned no choices") try: finish_reason = choices[0].get("finish_reason") or "none" msg = choices[0].get("message") or {} except (TypeError, AttributeError) as e: - logger.explore("TypeError processing LLM response choices", extra={ - "src": "executor_diag", "error": str(e), - "choices_0_type": type(choices[0]).__name__ if choices else "N/A", - "choices_0_repr": repr(choices[0])[:2000] if choices else "N/A", - "data_type": type(data).__name__, "data_preview": str(data)[:2000], - }) + logger.explore( + "TypeError processing LLM response choices", + extra={ + "src": "executor_diag", + "error": str(e), + "choices_0_type": type(choices[0]).__name__ if choices else "N/A", + "choices_0_repr": repr(choices[0])[:2000] if choices else "N/A", + "data_type": type(data).__name__, + "data_preview": str(data)[:2000], + }, + ) raise ValueError(f"LLM response processing failed: {e}") # Log provider token usage for batch sizing calibration @@ -183,28 +178,36 @@ async def call_openai_compatible( refusal = msg.get("refusal") if isinstance(msg, dict) else None if refusal: - logger.explore("LLM refused to respond", extra={ - "src": "executor", "refusal": str(refusal)[:500], "finish_reason": finish_reason, - }) + logger.explore( + "LLM refused to respond", + extra={ + "src": "executor", + "refusal": str(refusal)[:500], + "finish_reason": finish_reason, + }, + ) raise ValueError(f"LLM refused to respond: {refusal}") content = msg.get("content") if isinstance(msg, dict) else "" if not content and isinstance(msg, dict): content = msg.get("content") or "" - logger.reason( - f"LLM response finish_reason={finish_reason} content_len={len(content)} " - f"msg_keys={list(msg.keys()) if isinstance(msg, dict) else []}" - ) + logger.reason(f"LLM response finish_reason={finish_reason} content_len={len(content)} msg_keys={list(msg.keys()) if isinstance(msg, dict) else []}") if not content: - logger.explore("LLM returned empty content", extra={ - "src": "executor", "finish_reason": finish_reason, - "msg_keys": list(msg.keys()) if isinstance(msg, dict) else [], - "response_preview": str(data)[:2000], - }) + logger.explore( + "LLM returned empty content", + extra={ + "src": "executor", + "finish_reason": finish_reason, + "msg_keys": list(msg.keys()) if isinstance(msg, dict) else [], + "response_preview": str(data)[:2000], + }, + ) raise ValueError("LLM returned empty content") return content, finish_reason + + # #endregion call_openai_compatible @@ -224,34 +227,34 @@ async def _do_http_request(url: str, headers: dict, payload: dict) -> tuple[http try: wait = int(retry_after) except (ValueError, TypeError): - wait = 2 ** _retry_count_429 + wait = 2**_retry_count_429 else: - wait = 2 ** _retry_count_429 - logger.explore(f"Rate limited (429), retry {_retry_count_429}/{_max_retry_429} after {wait}s", - extra={"src": "executor", "retry_after": retry_after, "wait": wait}) + wait = 2**_retry_count_429 + logger.explore(f"Rate limited (429), retry {_retry_count_429}/{_max_retry_429} after {wait}s", extra={"src": "executor", "retry_after": retry_after, "wait": wait}) await asyncio.sleep(wait) if _retry_count_429 >= _max_retry_429: break else: break return response, response_text + + # #endregion _do_http_request # #region _handle_response_format_fallback [C:1] [TYPE Function] async def _handle_response_format_fallback( - response: httpx.Response, response_text: str, payload: dict, url: str, headers: dict, + response: httpx.Response, + response_text: str, + payload: dict, + url: str, + headers: dict, ) -> None: """Handle 400 errors from structured_outputs not being supported.""" _patterns = ("response_format", "structured_outputs", "structured", "json_object") - if ( - not response.is_success - and response.status_code == 400 - and any(p in (response_text or "").lower() for p in _patterns) - ): + if not response.is_success and response.status_code == 400 and any(p in (response_text or "").lower() for p in _patterns): client = await _get_http_client() - logger.explore("Structured outputs not supported, retrying without response_format", - extra={"src": "executor"}) + logger.explore("Structured outputs not supported, retrying without response_format", extra={"src": "executor"}) payload.pop("response_format", None) new_response = await client.post(url, headers=headers, json=payload) # Mutate the original response object with new data @@ -259,5 +262,7 @@ async def _handle_response_format_fallback( response._content = new_response.content response.encoding = new_response.encoding response.headers = new_response.headers + + # #endregion _handle_response_format_fallback # #endregion LLMAsyncHttpClient diff --git a/backend/tests/plugins/test_llm_analysis_service.py b/backend/tests/plugins/test_llm_analysis_service.py index a466a75c..4952cc2f 100644 --- a/backend/tests/plugins/test_llm_analysis_service.py +++ b/backend/tests/plugins/test_llm_analysis_service.py @@ -24,11 +24,13 @@ from src.plugins.llm_analysis.models import LLMProviderType # ── ScreenshotService Tests ── + class TestResponseLooksLikeLoginPage: """Verify _response_looks_like_login_page.""" def test_login_page_detected(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) html = """
@@ -44,18 +46,21 @@ class TestResponseLooksLikeLoginPage: def test_non_login_page(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) html = "

Dashboard

Welcome to Superset

" assert svc._response_looks_like_login_page(html) is False def test_empty_text(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) assert svc._response_looks_like_login_page("") is False assert svc._response_looks_like_login_page(None) is False def test_partial_match_under_threshold(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) assert svc._response_looks_like_login_page("Please sign in to continue") is False assert svc._response_looks_like_login_page("Username: admin Password: secret") is False @@ -63,6 +68,7 @@ class TestResponseLooksLikeLoginPage: def test_csrf_token_marker(self): """Edge: csrf_token hidden input is a marker.""" from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) html = 'name="csrf_token" value="abc"' assert svc._response_looks_like_login_page(html) is False # only 1 marker @@ -73,18 +79,21 @@ class TestRedirectLooksAuthenticated: def test_empty_redirect_authenticated(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) assert svc._redirect_looks_authenticated("") is True assert svc._redirect_looks_authenticated(None) is True def test_login_redirect_not_authenticated(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) assert svc._redirect_looks_authenticated("/login/") is False assert svc._redirect_looks_authenticated("https://example.com/login") is False def test_dashboard_redirect_authenticated(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) assert svc._redirect_looks_authenticated("/superset/dashboard/1/") is True assert svc._redirect_looks_authenticated("https://example.com/superset/dashboard/1/") is True @@ -95,6 +104,7 @@ class TestIterLoginRoots: def test_page_without_frames(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) page = MagicMock() page.frames = [] @@ -104,6 +114,7 @@ class TestIterLoginRoots: def test_page_with_frames(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) page = MagicMock() frame1 = MagicMock() @@ -118,8 +129,9 @@ class TestIterLoginRoots: def test_page_frames_property_exception(self): """Edge: exception in frames property handled gracefully.""" from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) - page = MagicMock(spec_set=['url']) + page = MagicMock(spec_set=["url"]) roots = svc._iter_login_roots(page) assert len(roots) == 1 @@ -130,6 +142,7 @@ class TestExtractHiddenLoginFields: @pytest.mark.asyncio async def test_extracts_hidden_fields(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) mock_page = MagicMock() @@ -139,12 +152,14 @@ class TestExtractHiddenLoginFields: async def attr_side(attr_name): return "csrf_token" if attr_name == "name" else None + field1 = MagicMock() field1.get_attribute = AsyncMock(side_effect=attr_side) field1.input_value = AsyncMock(return_value="abc123") async def attr_side2(attr_name): return "next" if attr_name == "name" else None + field2 = MagicMock() field2.get_attribute = AsyncMock(side_effect=attr_side2) field2.input_value = AsyncMock(return_value="/dashboard/") @@ -158,6 +173,7 @@ class TestExtractHiddenLoginFields: @pytest.mark.asyncio async def test_empty_when_no_hidden_fields(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) mock_page = MagicMock() @@ -171,6 +187,7 @@ class TestExtractHiddenLoginFields: @pytest.mark.asyncio async def test_skips_duplicate_names(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) mock_page = MagicMock() @@ -180,6 +197,7 @@ class TestExtractHiddenLoginFields: async def attr_side(attr_name): return "csrf_token" if attr_name == "name" else None + field1 = MagicMock() field1.get_attribute = AsyncMock(side_effect=attr_side) field1.input_value = AsyncMock(return_value="first") @@ -199,15 +217,17 @@ class TestExtractCsrfToken: @pytest.mark.asyncio async def test_found_token(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) - with patch.object(svc, '_extract_hidden_login_fields', AsyncMock(return_value={"csrf_token": "tok123"})): + with patch.object(svc, "_extract_hidden_login_fields", AsyncMock(return_value={"csrf_token": "tok123"})): assert await svc._extract_csrf_token(MagicMock()) == "tok123" @pytest.mark.asyncio async def test_missing_token(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) - with patch.object(svc, '_extract_hidden_login_fields', AsyncMock(return_value={})): + with patch.object(svc, "_extract_hidden_login_fields", AsyncMock(return_value={})): assert await svc._extract_csrf_token(MagicMock()) == "" @@ -217,6 +237,7 @@ class TestSubmitLoginViaFormPost: @pytest.mark.asyncio async def test_redirect_authenticated(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) svc.env = MagicMock() svc.env.username = "admin" @@ -232,27 +253,29 @@ class TestSubmitLoginViaFormPost: mock_response.headers = {"location": "/superset/dashboard/1/"} mock_request.post = AsyncMock(return_value=mock_response) - with patch.object(svc, '_extract_hidden_login_fields', AsyncMock(return_value={"csrf_token": "tok"})): + with patch.object(svc, "_extract_hidden_login_fields", AsyncMock(return_value={"csrf_token": "tok"})): result = await svc._submit_login_via_form_post(mock_page, "https://example.com/login/") assert result is True @pytest.mark.asyncio async def test_no_csrf_token_skips(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) - with patch.object(svc, '_extract_hidden_login_fields', AsyncMock(return_value={})): + with patch.object(svc, "_extract_hidden_login_fields", AsyncMock(return_value={})): result = await svc._submit_login_via_form_post(MagicMock(), "https://example.com/login/") assert result is False @pytest.mark.asyncio async def test_request_context_unavailable(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) mock_page = MagicMock() mock_page.context = MagicMock(side_effect=AttributeError("no context")) # Override to test the try/except - with patch.object(svc, '_extract_hidden_login_fields', AsyncMock(return_value={"csrf_token": "tok"})): - with patch.object(svc, '_submit_login_via_form_post', AsyncMock(return_value=False)): + with patch.object(svc, "_extract_hidden_login_fields", AsyncMock(return_value={"csrf_token": "tok"})): + with patch.object(svc, "_submit_login_via_form_post", AsyncMock(return_value=False)): # We test the context error path by making request unavailable pass @@ -260,6 +283,7 @@ class TestSubmitLoginViaFormPost: async def test_login_page_response_returns_false(self): """When POST returns login page markup, return False.""" from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) svc.env = MagicMock() svc.env.username = "admin" @@ -278,7 +302,7 @@ class TestSubmitLoginViaFormPost: mock_response.text = AsyncMock(return_value='
') mock_request.post = AsyncMock(return_value=mock_response) - with patch.object(svc, '_extract_hidden_login_fields', AsyncMock(return_value={"csrf_token": "tok"})): + with patch.object(svc, "_extract_hidden_login_fields", AsyncMock(return_value={"csrf_token": "tok"})): result = await svc._submit_login_via_form_post(mock_page, "https://example.com/login/") assert result is False @@ -289,6 +313,7 @@ class TestFindLoginFieldLocator: @pytest.mark.asyncio async def test_finds_username_field(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) mock_page = MagicMock() @@ -306,6 +331,7 @@ class TestFindLoginFieldLocator: @pytest.mark.asyncio async def test_finds_password_field(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) mock_page = MagicMock() @@ -323,6 +349,7 @@ class TestFindLoginFieldLocator: @pytest.mark.asyncio async def test_no_field_found(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) mock_page = MagicMock() @@ -344,6 +371,7 @@ class TestFindLoginFieldLocator: async def test_finds_username_via_input_fallback(self): """Fallback: input[type='text'] for username.""" from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) mock_page = MagicMock() @@ -391,6 +419,7 @@ class TestFindSubmitLocator: @pytest.mark.asyncio async def test_finds_sign_in_button(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) mock_page = MagicMock() @@ -408,6 +437,7 @@ class TestFindSubmitLocator: @pytest.mark.asyncio async def test_no_submit_found(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) mock_page = MagicMock() @@ -428,6 +458,7 @@ class TestGotoResilient: @pytest.mark.asyncio async def test_primary_success(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) mock_page = MagicMock() mock_page.goto = AsyncMock(return_value=MagicMock()) @@ -437,6 +468,7 @@ class TestGotoResilient: @pytest.mark.asyncio async def test_fallback_on_primary_failure(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) mock_page = MagicMock() mock_response = MagicMock() @@ -457,20 +489,22 @@ class TestWaitForChartsStabilized: @pytest.mark.asyncio async def test_polls_and_returns(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) mock_page = MagicMock() mock_page.wait_for_function = AsyncMock() - with patch('asyncio.sleep', AsyncMock()): + with patch("asyncio.sleep", AsyncMock()): await svc._wait_for_charts_stabilized(mock_page) mock_page.wait_for_function.assert_called_once() @pytest.mark.asyncio async def test_timeout_does_not_raise(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) mock_page = MagicMock() mock_page.wait_for_function = AsyncMock(side_effect=Exception("timeout")) - with patch('asyncio.sleep', AsyncMock()): + with patch("asyncio.sleep", AsyncMock()): # Should not raise await svc._wait_for_charts_stabilized(mock_page) @@ -481,6 +515,7 @@ class TestWaitForResizeRendered: @pytest.mark.asyncio async def test_polls_and_returns(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) mock_page = MagicMock() mock_page.wait_for_function = AsyncMock() @@ -490,6 +525,7 @@ class TestWaitForResizeRendered: @pytest.mark.asyncio async def test_timeout_does_not_raise(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) mock_page = MagicMock() mock_page.wait_for_function = AsyncMock(side_effect=Exception("timeout")) @@ -502,6 +538,7 @@ class TestSaveDebugScreenshot: @pytest.mark.asyncio async def test_saves_screenshot(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) mock_page = MagicMock() mock_page.screenshot = AsyncMock() @@ -513,6 +550,7 @@ class TestSaveDebugScreenshot: @pytest.mark.asyncio async def test_failure_returns_none(self): from src.plugins.llm_analysis.service import ScreenshotService + svc = ScreenshotService(MagicMock()) mock_page = MagicMock() mock_page.screenshot = AsyncMock(side_effect=Exception("screenshot failed")) @@ -542,6 +580,7 @@ class TestConvertScreenshotsForLlm: def test_missing_file_skipped(self): from src.plugins.llm_analysis.service import ScreenshotService + result = ScreenshotService._convert_screenshots_for_llm(["/nonexistent.png"], "/tmp") assert result == [] @@ -591,6 +630,7 @@ class TestArchiveScreenshotsAsWebp: def test_archive_missing_file(self): from src.plugins.llm_analysis.service import ScreenshotService + result = ScreenshotService._archive_screenshots_as_webp(["/nonexistent.png"], "/tmp") assert len(result) == 1 assert result[0]["webp_path"] is None @@ -641,11 +681,13 @@ class TestCleanupTempFiles: def test_cleanup_missing_file(self): from src.plugins.llm_analysis.service import ScreenshotService + ScreenshotService._cleanup_temp_files(["/nonexistent.png"]) # ── LLMClient Tests ── + class TestLLMClientInit: """Verify LLMClient initialization.""" @@ -667,11 +709,11 @@ class TestLLMClientInit: assert client.api_key == "sk-test-key" def test_init_openrouter_headers(self): - with patch.dict(os.environ, {"OPENROUTER_SITE_URL": "https://example.com", - "OPENROUTER_APP_NAME": "TestApp"}): + with patch.dict(os.environ, {"OPENROUTER_SITE_URL": "https://example.com", "OPENROUTER_APP_NAME": "TestApp"}): from src.plugins.llm_analysis.service import LLMClient - with patch('src.plugins.llm_analysis.service.httpx.AsyncClient'): - with patch('src.plugins.llm_analysis.service.AsyncOpenAI'): + + with patch("src.plugins.llm_analysis.service.httpx.AsyncClient"): + with patch("src.plugins.llm_analysis.service.AsyncOpenAI"): client = LLMClient( provider_type=LLMProviderType.OPENROUTER, api_key="sk-test", @@ -682,8 +724,9 @@ class TestLLMClientInit: def test_init_kilo_headers(self): from src.plugins.llm_analysis.service import LLMClient - with patch('src.plugins.llm_analysis.service.httpx.AsyncClient'): - with patch('src.plugins.llm_analysis.service.AsyncOpenAI'): + + with patch("src.plugins.llm_analysis.service.httpx.AsyncClient"): + with patch("src.plugins.llm_analysis.service.AsyncOpenAI"): client = LLMClient( provider_type=LLMProviderType.KILO, api_key="sk-test", @@ -694,8 +737,9 @@ class TestLLMClientInit: def _make_client(self, api_key="sk-test"): from src.plugins.llm_analysis.service import LLMClient - with patch('src.plugins.llm_analysis.service.httpx.AsyncClient'): - with patch('src.plugins.llm_analysis.service.AsyncOpenAI'): + + with patch("src.plugins.llm_analysis.service.httpx.AsyncClient"): + with patch("src.plugins.llm_analysis.service.AsyncOpenAI"): return LLMClient( provider_type=LLMProviderType.OPENAI, api_key=api_key, @@ -707,46 +751,21 @@ class TestLLMClientInit: class TestLLMClientSslVerify: """Verify _get_ssl_verify.""" - def test_default_disabled(self): + def test_returns_ssl_context(self): + """_get_ssl_verify always returns SSLContext (no env-based disable).""" from src.plugins.llm_analysis.service import LLMClient + from src.core.ssl import system_ssl_context + + result = LLMClient._get_ssl_verify() + assert isinstance(result, ssl.SSLContext) + + def test_ignores_llm_ssl_verify_env(self): + """LLM_SSL_VERIFY env is no longer read — central ssl helper is used.""" + from src.plugins.llm_analysis.service import LLMClient + with patch.dict(os.environ, {"LLM_SSL_VERIFY": "false"}): - assert LLMClient._get_ssl_verify() is False - - def test_disabled_zero(self): - from src.plugins.llm_analysis.service import LLMClient - with patch.dict(os.environ, {"LLM_SSL_VERIFY": "0"}): - assert LLMClient._get_ssl_verify() is False - - def test_disabled_no(self): - from src.plugins.llm_analysis.service import LLMClient - with patch.dict(os.environ, {"LLM_SSL_VERIFY": "no"}): - assert LLMClient._get_ssl_verify() is False - - def test_disabled_off(self): - from src.plugins.llm_analysis.service import LLMClient - with patch.dict(os.environ, {"LLM_SSL_VERIFY": "off"}): - assert LLMClient._get_ssl_verify() is False - - def test_enabled_returns_context(self): - from src.plugins.llm_analysis.service import LLMClient - with patch.dict(os.environ, {}, clear=True): result = LLMClient._get_ssl_verify() - assert isinstance(result, (ssl.SSLContext, bool)) - - def test_ca_dir_missing(self): - """When /etc/ssl/certs doesn't exist, still returns SSLContext.""" - from src.plugins.llm_analysis.service import LLMClient - with patch.dict(os.environ, {}, clear=True): - with patch('os.path.isdir', return_value=False): - result = LLMClient._get_ssl_verify() - assert isinstance(result, ssl.SSLContext) - - def test_unknown_value_defaults_enabled(self): - """Unknown env values treated as 'true' -> enabled.""" - from src.plugins.llm_analysis.service import LLMClient - with patch.dict(os.environ, {"LLM_SSL_VERIFY": "maybe"}): - result = LLMClient._get_ssl_verify() - assert isinstance(result, (ssl.SSLContext, bool)) + assert isinstance(result, ssl.SSLContext), "must not return False" class TestFormatConnectionError: @@ -754,12 +773,14 @@ class TestFormatConnectionError: def test_single_exception(self): from src.plugins.llm_analysis.service import LLMClient + result = LLMClient._format_connection_error(ValueError("test error")) assert "ValueError" in result assert "test error" in result def test_chained_exception(self): from src.plugins.llm_analysis.service import LLMClient + inner = ConnectionError("connection refused") outer = RuntimeError("API call failed") outer.__cause__ = inner @@ -770,6 +791,7 @@ class TestFormatConnectionError: def test_deep_chain(self): from src.plugins.llm_analysis.service import LLMClient + inner = ValueError("inner") middle = TypeError("middle") outer = RuntimeError("outer") @@ -786,30 +808,35 @@ class TestSupportsJsonResponseFormat: def test_free_model_disabled(self): from src.plugins.llm_analysis.service import LLMClient + client = MagicMock() client.default_model = "gpt-4o:free" assert LLMClient._supports_json_response_format(client) is False def test_stepfun_disabled(self): from src.plugins.llm_analysis.service import LLMClient + client = MagicMock() client.default_model = "step-1v" assert LLMClient._supports_json_response_format(client) is False def test_stepfun_slash_disabled(self): from src.plugins.llm_analysis.service import LLMClient + client = MagicMock() client.default_model = "stepfun/some-model" assert LLMClient._supports_json_response_format(client) is False def test_normal_model_enabled(self): from src.plugins.llm_analysis.service import LLMClient + client = MagicMock() client.default_model = "gpt-4o" assert LLMClient._supports_json_response_format(client) is True def test_empty_model_default_enabled(self): from src.plugins.llm_analysis.service import LLMClient + client = MagicMock() client.default_model = "" assert LLMClient._supports_json_response_format(client) is True @@ -820,6 +847,7 @@ class TestDeduplicateIssues: def test_deduplicates_by_severity_message_location(self): from src.plugins.llm_analysis.service import LLMClient + client = MagicMock() issues = [ {"severity": "HIGH", "message": "Chart missing", "location": "chart1"}, @@ -831,12 +859,14 @@ class TestDeduplicateIssues: def test_empty_issues(self): from src.plugins.llm_analysis.service import LLMClient + client = MagicMock() assert LLMClient._deduplicate_issues(client, []) == [] def test_issues_with_none_location(self): """Edge: None location treated as empty string.""" from src.plugins.llm_analysis.service import LLMClient + client = MagicMock() issues = [ {"severity": "HIGH", "message": "Error", "location": None}, @@ -848,6 +878,7 @@ class TestDeduplicateIssues: def test_issues_without_location(self): """Edge: missing location key.""" from src.plugins.llm_analysis.service import LLMClient + client = MagicMock() issues = [ {"severity": "HIGH", "message": "No loc"}, @@ -862,6 +893,7 @@ class TestEstimatePayloadSize: def test_estimate_basic(self): from src.plugins.llm_analysis.service import LLMClient + result = LLMClient._estimate_payload_size(["img1.png"], 1000, 128000) assert result["estimated_tokens"] > 0 assert "pct_of_limit" in result @@ -869,16 +901,19 @@ class TestEstimatePayloadSize: def test_large_payload_exceeds(self): from src.plugins.llm_analysis.service import LLMClient + result = LLMClient._estimate_payload_size(["img1.png"] * 100, 100000, 128000) assert result["exceeds_limit"] is True def test_small_payload_ok(self): from src.plugins.llm_analysis.service import LLMClient + result = LLMClient._estimate_payload_size([], 100, 128000) assert result["exceeds_limit"] is False def test_no_images(self): from src.plugins.llm_analysis.service import LLMClient + result = LLMClient._estimate_payload_size([], 4000, 128000) assert result["estimated_tokens"] == 1000 # 4000/4 @@ -888,42 +923,46 @@ class TestMergeChunkResults: def test_merge_takes_worst_status(self): from src.plugins.llm_analysis.service import LLMClient + client = MagicMock() chunks = [ {"status": "PASS", "summary": "All good", "issues": []}, {"status": "FAIL", "summary": "Broken", "issues": [{"severity": "HIGH", "message": "Error"}]}, ] - with patch.object(LLMClient, '_deduplicate_issues', return_value=[{"severity": "HIGH", "message": "Error"}]): + with patch.object(LLMClient, "_deduplicate_issues", return_value=[{"severity": "HIGH", "message": "Error"}]): result = LLMClient._merge_chunk_results(client, chunks) assert result["status"] == "FAIL" assert result["chunk_count"] == 2 def test_merge_single_chunk(self): from src.plugins.llm_analysis.service import LLMClient + client = MagicMock() chunks = [{"status": "WARN", "summary": "Some issues", "issues": []}] - with patch.object(LLMClient, '_deduplicate_issues', return_value=[]): + with patch.object(LLMClient, "_deduplicate_issues", return_value=[]): result = LLMClient._merge_chunk_results(client, chunks) assert result["status"] == "WARN" assert result["chunk_count"] == 1 def test_merge_unknown_default(self): from src.plugins.llm_analysis.service import LLMClient + client = MagicMock() chunks = [{"summary": "No status", "issues": []}] - with patch.object(LLMClient, '_deduplicate_issues', return_value=[]): + with patch.object(LLMClient, "_deduplicate_issues", return_value=[]): result = LLMClient._merge_chunk_results(client, chunks) assert result["status"] == "UNKNOWN" def test_merge_worst_over_pass(self): """WARN < PASS so WARN wins.""" from src.plugins.llm_analysis.service import LLMClient + client = MagicMock() chunks = [ {"status": "PASS", "summary": "OK", "issues": []}, {"status": "WARN", "summary": "Warning", "issues": []}, ] - with patch.object(LLMClient, '_deduplicate_issues', return_value=[]): + with patch.object(LLMClient, "_deduplicate_issues", return_value=[]): result = LLMClient._merge_chunk_results(client, chunks) assert result["status"] == "WARN" @@ -935,17 +974,15 @@ class TestLLMClientAnalyze: async def test_analyze_dashboard_delegates_to_multimodal(self): from src.plugins.llm_analysis.service import LLMClient as RealClient - with patch('src.plugins.llm_analysis.service.httpx.AsyncClient'): - with patch('src.plugins.llm_analysis.service.AsyncOpenAI'): + with patch("src.plugins.llm_analysis.service.httpx.AsyncClient"): + with patch("src.plugins.llm_analysis.service.AsyncOpenAI"): real_client = RealClient( provider_type=LLMProviderType.OPENAI, api_key="sk-test", base_url="https://api.openai.com", default_model="gpt-4o", ) - real_client.analyze_dashboard_multimodal = AsyncMock( - return_value={"status": "PASS", "summary": "OK"} - ) + real_client.analyze_dashboard_multimodal = AsyncMock(return_value={"status": "PASS", "summary": "OK"}) result = await real_client.analyze_dashboard( screenshot_path="/tmp/test.png", @@ -965,8 +1002,8 @@ class TestLLMClientOptimizeImages: Image.new("RGB", (100, 100), (255, 0, 0)).save(png_path, "PNG") # Create a real client with mocked internals - with patch('src.plugins.llm_analysis.service.httpx.AsyncClient'): - with patch('src.plugins.llm_analysis.service.AsyncOpenAI'): + with patch("src.plugins.llm_analysis.service.httpx.AsyncClient"): + with patch("src.plugins.llm_analysis.service.AsyncOpenAI"): real_client = LLMClient( provider_type=LLMProviderType.OPENAI, api_key="sk-test", @@ -974,7 +1011,7 @@ class TestLLMClientOptimizeImages: default_model="gpt-4o-mini", ) # Patch the static method on the class - with patch.object(LLMClient, '_reduce_image_quality', return_value=("base64data", 1000)): + with patch.object(LLMClient, "_reduce_image_quality", return_value=("base64data", 1000)): result = real_client._optimize_images([png_path], 1024, 60) assert len(result) == 1 assert result[0] == "base64data" @@ -987,7 +1024,7 @@ class TestLLMClientOptimizeImages: Image.new("RGB", (100, 100)).save(png_path, "PNG") client = MagicMock() - with patch.object(LLMClient, '_reduce_image_quality', side_effect=Exception("corrupt")): + with patch.object(LLMClient, "_reduce_image_quality", side_effect=Exception("corrupt")): result = LLMClient._optimize_images(client, [png_path], 1024, 60) assert len(result) == 1 assert isinstance(result[0], str) @@ -1051,7 +1088,7 @@ class TestLLMClientCallLlmForImages: client = MagicMock() client.get_json_completion = AsyncMock(return_value={"status": "PASS"}) - with patch.object(LLMClient, '_call_llm_for_images') as mock_call: + with patch.object(LLMClient, "_call_llm_for_images") as mock_call: mock_call.return_value = {"status": "PASS"} result = await LLMClient._call_llm_for_images(client, ["base64img"], "Analyze") assert result is not None @@ -1064,8 +1101,8 @@ class TestLLMClientFetchModels: async def test_fetch_success(self): from src.plugins.llm_analysis.service import LLMClient - with patch('src.plugins.llm_analysis.service.httpx.AsyncClient'): - with patch('src.plugins.llm_analysis.service.AsyncOpenAI') as MockOpenAI: + with patch("src.plugins.llm_analysis.service.httpx.AsyncClient"): + with patch("src.plugins.llm_analysis.service.AsyncOpenAI") as MockOpenAI: mock_client = MagicMock() MockOpenAI.return_value = mock_client mock_response = MagicMock() @@ -1090,8 +1127,8 @@ class TestLLMClientFetchModels: async def test_fetch_failure_raises(self): from src.plugins.llm_analysis.service import LLMClient - with patch('src.plugins.llm_analysis.service.httpx.AsyncClient'): - with patch('src.plugins.llm_analysis.service.AsyncOpenAI') as MockOpenAI: + with patch("src.plugins.llm_analysis.service.httpx.AsyncClient"): + with patch("src.plugins.llm_analysis.service.AsyncOpenAI") as MockOpenAI: mock_client = MagicMock() MockOpenAI.return_value = mock_client mock_client.models.list = AsyncMock(side_effect=ConnectionError("API unavailable")) @@ -1114,8 +1151,8 @@ class TestLLMClientTestRuntimeConnection: async def test_runtime_connection_success(self): from src.plugins.llm_analysis.service import LLMClient - with patch('src.plugins.llm_analysis.service.httpx.AsyncClient'): - with patch('src.plugins.llm_analysis.service.AsyncOpenAI'): + with patch("src.plugins.llm_analysis.service.httpx.AsyncClient"): + with patch("src.plugins.llm_analysis.service.AsyncOpenAI"): real_client = LLMClient( provider_type=LLMProviderType.OPENAI, api_key="sk-test", @@ -1145,14 +1182,14 @@ class TestGetJsonCompletion: """Edge: JSON embedded in ```json code block parsed.""" from src.plugins.llm_analysis.service import LLMClient - with patch('src.plugins.llm_analysis.service.httpx.AsyncClient'): - with patch('src.plugins.llm_analysis.service.AsyncOpenAI') as MockOpenAI: + with patch("src.plugins.llm_analysis.service.httpx.AsyncClient"): + with patch("src.plugins.llm_analysis.service.AsyncOpenAI") as MockOpenAI: mock_client = MagicMock() MockOpenAI.return_value = mock_client # Mock response content with JSON in code block mock_choice = MagicMock() - mock_choice.message.content = "```json\n{\"status\": \"PASS\"}\n```" + mock_choice.message.content = '```json\n{"status": "PASS"}\n```' mock_response = MagicMock() mock_response.choices = [mock_choice] mock_client.chat.completions.create = AsyncMock(return_value=mock_response) @@ -1172,13 +1209,13 @@ class TestGetJsonCompletion: """Edge: JSON in ``` code block (no json marker).""" from src.plugins.llm_analysis.service import LLMClient - with patch('src.plugins.llm_analysis.service.httpx.AsyncClient'): - with patch('src.plugins.llm_analysis.service.AsyncOpenAI') as MockOpenAI: + with patch("src.plugins.llm_analysis.service.httpx.AsyncClient"): + with patch("src.plugins.llm_analysis.service.AsyncOpenAI") as MockOpenAI: mock_client = MagicMock() MockOpenAI.return_value = mock_client mock_choice = MagicMock() - mock_choice.message.content = "```\n{\"status\": \"WARN\"}\n```" + mock_choice.message.content = '```\n{"status": "WARN"}\n```' mock_response = MagicMock() mock_response.choices = [mock_choice] mock_client.chat.completions.create = AsyncMock(return_value=mock_response) @@ -1198,8 +1235,8 @@ class TestGetJsonCompletion: """Negative: null content raises RuntimeError.""" from src.plugins.llm_analysis.service import LLMClient - with patch('src.plugins.llm_analysis.service.httpx.AsyncClient'): - with patch('src.plugins.llm_analysis.service.AsyncOpenAI') as MockOpenAI: + with patch("src.plugins.llm_analysis.service.httpx.AsyncClient"): + with patch("src.plugins.llm_analysis.service.AsyncOpenAI") as MockOpenAI: mock_client = MagicMock() MockOpenAI.return_value = mock_client @@ -1224,8 +1261,8 @@ class TestGetJsonCompletion: """Negative: empty choices raises RuntimeError.""" from src.plugins.llm_analysis.service import LLMClient - with patch('src.plugins.llm_analysis.service.httpx.AsyncClient'): - with patch('src.plugins.llm_analysis.service.AsyncOpenAI') as MockOpenAI: + with patch("src.plugins.llm_analysis.service.httpx.AsyncClient"): + with patch("src.plugins.llm_analysis.service.AsyncOpenAI") as MockOpenAI: mock_client = MagicMock() MockOpenAI.return_value = mock_client mock_response = MagicMock() @@ -1270,7 +1307,9 @@ class TestAnalyzeDashboardMultimodal: client._deduplicate_issues.return_value = [] result = await LLMClient.analyze_dashboard_multimodal( - client, [png_path], ["log1"], + client, + [png_path], + ["log1"], prompt_template="Analyze: {logs}", ) assert result["status"] == "PASS" @@ -1287,14 +1326,16 @@ class TestAnalyzeDashboardMultimodal: client = MagicMock() client._optimize_images.side_effect = [ ["img1_high"], # first call (high quality) - ["img1_low"], # second call (reduced quality) + ["img1_low"], # second call (reduced quality) ] client._estimate_payload_size.return_value = {"exceeds_limit": True, "pct_of_limit": 85} client._call_llm_for_images = AsyncMock(return_value={"status": "WARN", "summary": "Reduced", "issues": []}) client._deduplicate_issues.return_value = [] result = await LLMClient.analyze_dashboard_multimodal( - client, [png_path], ["log1"], + client, + [png_path], + ["log1"], prompt_template="Analyze: {logs}", ) assert result["status"] == "WARN" @@ -1319,7 +1360,9 @@ class TestAnalyzeDashboardMultimodal: paths.append(p) result = await LLMClient.analyze_dashboard_multimodal( - client, paths, ["log"], + client, + paths, + ["log"], max_images=2, ) # With 5 images and max_images=2, this creates 3 chunks @@ -1342,7 +1385,9 @@ class TestAnalyzeDashboardMultimodal: client._deduplicate_issues.return_value = [] result = await LLMClient.analyze_dashboard_multimodal( - client, [png_path], ["log"], + client, + [png_path], + ["log"], ) assert result["status"] == "UNKNOWN" @@ -1363,9 +1408,7 @@ class TestAnalyzeDashboardTextBatch: from src.plugins.llm_analysis.service import LLMClient client = MagicMock() - client.get_json_completion = AsyncMock(return_value={ - "dashboards": [{"dashboard_id": "1", "status": "PASS", "summary": "OK", "issues": []}] - }) + client.get_json_completion = AsyncMock(return_value={"dashboards": [{"dashboard_id": "1", "status": "PASS", "summary": "OK", "issues": []}]}) result = await LLMClient.analyze_dashboard_text_batch( client, @@ -1380,12 +1423,14 @@ class TestAnalyzeDashboardTextBatch: from src.plugins.llm_analysis.service import LLMClient client = MagicMock() - client.get_json_completion = AsyncMock(return_value={ - "dashboards": [ - {"dashboard_id": "1", "status": "PASS", "summary": "OK", "issues": []}, - {"dashboard_id": "2", "status": "FAIL", "summary": "Broken", "issues": [{"severity": "HIGH", "message": "err"}]}, - ] - }) + client.get_json_completion = AsyncMock( + return_value={ + "dashboards": [ + {"dashboard_id": "1", "status": "PASS", "summary": "OK", "issues": []}, + {"dashboard_id": "2", "status": "FAIL", "summary": "Broken", "issues": [{"severity": "HIGH", "message": "err"}]}, + ] + } + ) result = await LLMClient.analyze_dashboard_text_batch( client, @@ -1400,11 +1445,13 @@ class TestAnalyzeDashboardTextBatch: # ── DatasetHealthChecker Tests ── + class TestDatasetHealthChecker: """Verify DatasetHealthChecker.""" def test_import(self): from src.plugins.llm_analysis.service import DatasetHealthChecker + assert DatasetHealthChecker is not None @pytest.mark.asyncio @@ -1498,13 +1545,10 @@ class TestDatasetHealthChecker: mock_client = MagicMock() mock_client.network.request.return_value = {"result": [{"val": 1}]} - mock_client.get_dataset.return_value = { - "result": {"table_name": "test", "database": {"database_name": "db", "backend": "postgresql"}} - } + mock_client.get_dataset.return_value = {"result": {"table_name": "test", "database": {"database_name": "db", "backend": "postgresql"}}} chart_list = [ - {"slice_id": 1, "slice_name": "Chart 1", "datasource_id": 101, - "viz_type": "table", "params": {"metrics": ["count"]}, "datasource_type": "table"}, + {"slice_id": 1, "slice_name": "Chart 1", "datasource_id": 101, "viz_type": "table", "params": {"metrics": ["count"]}, "datasource_type": "table"}, ] checker = DatasetHealthChecker(mock_client) @@ -1520,13 +1564,10 @@ class TestDatasetHealthChecker: mock_client = MagicMock() mock_client.network.request.return_value = {"result": []} - mock_client.get_dataset.return_value = { - "result": {"table_name": "test", "database": {"database_name": "db", "backend": "postgresql"}} - } + mock_client.get_dataset.return_value = {"result": {"table_name": "test", "database": {"database_name": "db", "backend": "postgresql"}}} chart_list = [ - {"slice_id": 1, "slice_name": "Chart 1", "datasource_id": 101, - "viz_type": "table", "params": '{"metrics": ["count"]}', "datasource_type": "table"}, + {"slice_id": 1, "slice_name": "Chart 1", "datasource_id": 101, "viz_type": "table", "params": '{"metrics": ["count"]}', "datasource_type": "table"}, ] checker = DatasetHealthChecker(mock_client) @@ -1547,6 +1588,7 @@ class TestDatasetHealthChecker: # ── RedactionService Tests ── + class TestRedactionService: """Verify RedactionService.""" @@ -1572,6 +1614,7 @@ class TestRedactionService: def test_redact_logs_empty(self): from src.plugins.llm_analysis.service import RedactionService + assert RedactionService.redact_logs([]) == [] def test_redact_raw_response(self): @@ -1610,4 +1653,6 @@ class TestRedactionService: safe = '{"status": "PASS", "summary": "OK"}' redacted = RedactionService.redact_raw_response(safe) assert redacted == safe + + # #endregion Test.LLMAnalysisService diff --git a/backend/tests/plugins/translate/test_llm_async_http.py b/backend/tests/plugins/translate/test_llm_async_http.py index e7172585..252aac4a 100644 --- a/backend/tests/plugins/translate/test_llm_async_http.py +++ b/backend/tests/plugins/translate/test_llm_async_http.py @@ -6,8 +6,10 @@ # @TEST_EDGE: empty_content -> ValueError import os +import ssl import sys from pathlib import Path + sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / "src")) import pytest @@ -27,35 +29,16 @@ from src.plugins.translate._llm_async_http import ( class TestGetVerify: """_get_verify — SSL verification context.""" - def test_verify_true_default(self): - """Default (true) returns SSLContext.""" - with patch.dict(os.environ, {"LLM_SSL_VERIFY": "true"}, clear=True): - result = _get_verify() - assert isinstance(result, bool) is False # It's an SSLContext + def test_returns_ssl_context(self): + """Always returns SSLContext (no env-based disable).""" + result = _get_verify() + assert isinstance(result, ssl.SSLContext) - def test_verify_false(self): - """False returns False.""" - with patch.dict(os.environ, {"LLM_SSL_VERIFY": "false"}, clear=True): + def test_ignores_llm_ssl_verify_env(self): + """LLM_SSL_VERIFY env is no longer read — central ssl helper is used.""" + with patch.dict(os.environ, {"LLM_SSL_VERIFY": "false"}): result = _get_verify() - assert result is False - - def test_verify_off(self): - """'off' returns False.""" - with patch.dict(os.environ, {"LLM_SSL_VERIFY": "off"}, clear=True): - result = _get_verify() - assert result is False - - def test_verify_0(self): - """'0' returns False.""" - with patch.dict(os.environ, {"LLM_SSL_VERIFY": "0"}, clear=True): - result = _get_verify() - assert result is False - - def test_verify_no(self): - """'no' returns False.""" - with patch.dict(os.environ, {"LLM_SSL_VERIFY": "no"}, clear=True): - result = _get_verify() - assert result is False + assert isinstance(result, ssl.SSLContext), "must not return False" class TestGetHttpClient: @@ -101,12 +84,13 @@ class TestCallOpenaiCompatible: } mock_response.text = '{"choices": [{"finish_reason": "stop", "message": {"content": "Hello, world!"}}]}' - with patch("src.plugins.translate._llm_async_http._do_http_request", - AsyncMock(return_value=(mock_response, mock_response.text))): - with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", - AsyncMock()): + with patch("src.plugins.translate._llm_async_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))): + with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", AsyncMock()): content, finish_reason = await call_openai_compatible( - "https://api.openai.com", "sk-test", "gpt-4o", "translate this", + "https://api.openai.com", + "sk-test", + "gpt-4o", + "translate this", ) assert content == "Hello, world!" assert finish_reason == "stop" @@ -120,13 +104,14 @@ class TestCallOpenaiCompatible: mock_response.json.return_value = {"choices": []} mock_response.text = '{"choices": []}' - with patch("src.plugins.translate._llm_async_http._do_http_request", - AsyncMock(return_value=(mock_response, mock_response.text))): - with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", - AsyncMock()): + with patch("src.plugins.translate._llm_async_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))): + with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", AsyncMock()): with pytest.raises(ValueError, match="no choices"): await call_openai_compatible( - "https://api.openai.com", "sk-test", "gpt-4o", "translate this", + "https://api.openai.com", + "sk-test", + "gpt-4o", + "translate this", ) @pytest.mark.asyncio @@ -145,13 +130,14 @@ class TestCallOpenaiCompatible: } mock_response.text = '{"choices": [{"finish_reason": "stop", "message": {"content": ""}}]}' - with patch("src.plugins.translate._llm_async_http._do_http_request", - AsyncMock(return_value=(mock_response, mock_response.text))): - with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", - AsyncMock()): + with patch("src.plugins.translate._llm_async_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))): + with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", AsyncMock()): with pytest.raises(ValueError, match="empty content"): await call_openai_compatible( - "https://api.openai.com", "sk-test", "gpt-4o", "translate this", + "https://api.openai.com", + "sk-test", + "gpt-4o", + "translate this", ) @pytest.mark.asyncio @@ -165,13 +151,14 @@ class TestCallOpenaiCompatible: } mock_response.text = '{"choices": [null]}' - with patch("src.plugins.translate._llm_async_http._do_http_request", - AsyncMock(return_value=(mock_response, mock_response.text))): - with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", - AsyncMock()): + with patch("src.plugins.translate._llm_async_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))): + with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", AsyncMock()): with pytest.raises(ValueError, match="LLM response processing failed"): await call_openai_compatible( - "https://api.openai.com", "sk-test", "gpt-4o", "translate this", + "https://api.openai.com", + "sk-test", + "gpt-4o", + "translate this", ) @pytest.mark.asyncio @@ -190,13 +177,14 @@ class TestCallOpenaiCompatible: } mock_response.text = '{"choices": [{"finish_reason": "stop", "message": {"refusal": "I cannot answer that", "content": null}}]}' - with patch("src.plugins.translate._llm_async_http._do_http_request", - AsyncMock(return_value=(mock_response, mock_response.text))): - with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", - AsyncMock()): + with patch("src.plugins.translate._llm_async_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))): + with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", AsyncMock()): with pytest.raises(ValueError, match="refused"): await call_openai_compatible( - "https://api.openai.com", "sk-test", "gpt-4o", "translate this", + "https://api.openai.com", + "sk-test", + "gpt-4o", + "translate this", ) @pytest.mark.asyncio @@ -206,17 +194,20 @@ class TestCallOpenaiCompatible: mock_response.is_success = False mock_response.status_code = 500 mock_response.raise_for_status.side_effect = httpx.HTTPStatusError( - "Server error", request=MagicMock(), response=mock_response, + "Server error", + request=MagicMock(), + response=mock_response, ) mock_response.text = "Internal Server Error" - with patch("src.plugins.translate._llm_async_http._do_http_request", - AsyncMock(return_value=(mock_response, mock_response.text))): - with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", - AsyncMock()): + with patch("src.plugins.translate._llm_async_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))): + with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", AsyncMock()): with pytest.raises(httpx.HTTPStatusError): await call_openai_compatible( - "https://api.openai.com", "sk-test", "gpt-4o", "translate this", + "https://api.openai.com", + "sk-test", + "gpt-4o", + "translate this", ) @pytest.mark.asyncio @@ -231,14 +222,16 @@ class TestCallOpenaiCompatible: } mock_response.text = '{"choices": [{"finish_reason": "stop", "message": {"content": "hi"}}]}' - with patch("src.plugins.translate._llm_async_http._do_http_request", - AsyncMock(return_value=(mock_response, mock_response.text))): - with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", - AsyncMock()): + with patch("src.plugins.translate._llm_async_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))): + with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", AsyncMock()): # disable_reasoning=False, so response_format should be set content, _ = await call_openai_compatible( - "https://api.openai.com", "sk-test", "gpt-4o", "test", - provider_type="openai", disable_reasoning=False, + "https://api.openai.com", + "sk-test", + "gpt-4o", + "test", + provider_type="openai", + disable_reasoning=False, ) assert content == "hi" @@ -254,12 +247,13 @@ class TestCallOpenaiCompatible: } mock_response.text = '{"choices": [{"finish_reason": "stop", "message": {"content": "hi"}}]}' - with patch("src.plugins.translate._llm_async_http._do_http_request", - AsyncMock(return_value=(mock_response, mock_response.text))): - with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", - AsyncMock()): + with patch("src.plugins.translate._llm_async_http._do_http_request", AsyncMock(return_value=(mock_response, mock_response.text))): + with patch("src.plugins.translate._llm_async_http._handle_response_format_fallback", AsyncMock()): content, _ = await call_openai_compatible( - "https://api.openai.com", "sk-test", "gpt-4o", "test", + "https://api.openai.com", + "sk-test", + "gpt-4o", + "test", disable_reasoning=True, ) assert content == "hi" @@ -275,8 +269,7 @@ class TestDoHttpRequest: mock_response.status_code = 200 mock_response.text = "OK" - with patch("src.plugins.translate._llm_async_http._get_http_client", - AsyncMock()) as mock_get: + with patch("src.plugins.translate._llm_async_http._get_http_client", AsyncMock()) as mock_get: mock_client = AsyncMock() mock_get.return_value = mock_client mock_client.post.return_value = mock_response @@ -301,14 +294,12 @@ class TestDoHttpRequest: mock_200.status_code = 200 mock_200.text = "OK" - with patch("src.plugins.translate._llm_async_http._get_http_client", - AsyncMock()) as mock_get: + with patch("src.plugins.translate._llm_async_http._get_http_client", AsyncMock()) as mock_get: mock_client = AsyncMock() mock_get.return_value = mock_client mock_client.post.side_effect = [mock_429, mock_200] - with patch("src.plugins.translate._llm_async_http.asyncio.sleep", - AsyncMock()): + with patch("src.plugins.translate._llm_async_http.asyncio.sleep", AsyncMock()): response, text = await _do_http_request( "https://api.openai.com/chat/completions", {"Authorization": "Bearer test"}, @@ -330,14 +321,12 @@ class TestDoHttpRequest: mock_200.status_code = 200 mock_200.text = "OK" - with patch("src.plugins.translate._llm_async_http._get_http_client", - AsyncMock()) as mock_get: + with patch("src.plugins.translate._llm_async_http._get_http_client", AsyncMock()) as mock_get: mock_client = AsyncMock() mock_get.return_value = mock_client mock_client.post.side_effect = [mock_429, mock_200] - with patch("src.plugins.translate._llm_async_http.asyncio.sleep", - AsyncMock()) as mock_sleep: + with patch("src.plugins.translate._llm_async_http.asyncio.sleep", AsyncMock()) as mock_sleep: response, _ = await _do_http_request( "https://api.openai.com/chat/completions", {"Authorization": "Bearer test"}, @@ -355,14 +344,12 @@ class TestDoHttpRequest: mock_429.headers = {} mock_429.text = "Rate limited" - with patch("src.plugins.translate._llm_async_http._get_http_client", - AsyncMock()) as mock_get: + with patch("src.plugins.translate._llm_async_http._get_http_client", AsyncMock()) as mock_get: mock_client = AsyncMock() mock_get.return_value = mock_client mock_client.post.return_value = mock_429 # Always 429 - with patch("src.plugins.translate._llm_async_http.asyncio.sleep", - AsyncMock()): + with patch("src.plugins.translate._llm_async_http.asyncio.sleep", AsyncMock()): response, _ = await _do_http_request( "https://api.openai.com/chat/completions", {"Authorization": "Bearer test"}, @@ -383,14 +370,12 @@ class TestDoHttpRequest: mock_200.status_code = 200 mock_200.text = "OK" - with patch("src.plugins.translate._llm_async_http._get_http_client", - AsyncMock()) as mock_get: + with patch("src.plugins.translate._llm_async_http._get_http_client", AsyncMock()) as mock_get: mock_client = AsyncMock() mock_get.return_value = mock_client mock_client.post.side_effect = [mock_429, mock_200] - with patch("src.plugins.translate._llm_async_http.asyncio.sleep", - AsyncMock()) as mock_sleep: + with patch("src.plugins.translate._llm_async_http.asyncio.sleep", AsyncMock()) as mock_sleep: response, _ = await _do_http_request( "https://api.openai.com/chat/completions", {"Authorization": "Bearer test"}, @@ -421,15 +406,17 @@ class TestHandleResponseFormatFallback: payload = {"model": "gpt-4o", "response_format": {"type": "json_object"}} - with patch("src.plugins.translate._llm_async_http._get_http_client", - AsyncMock()) as mock_get: + with patch("src.plugins.translate._llm_async_http._get_http_client", AsyncMock()) as mock_get: mock_client = AsyncMock() mock_get.return_value = mock_client mock_client.post.return_value = mock_200 await _handle_response_format_fallback( - mock_400, mock_400.text, payload, - "https://api.openai.com", {}, + mock_400, + mock_400.text, + payload, + "https://api.openai.com", + {}, ) # Verify response_format was popped assert "response_format" not in payload @@ -446,14 +433,16 @@ class TestHandleResponseFormatFallback: payload = {"model": "gpt-4o", "response_format": {"type": "json_object"}} - with patch("src.plugins.translate._llm_async_http._get_http_client", - AsyncMock()) as mock_get: + with patch("src.plugins.translate._llm_async_http._get_http_client", AsyncMock()) as mock_get: mock_client = AsyncMock() mock_get.return_value = mock_client await _handle_response_format_fallback( - mock_500, mock_500.text, payload, - "https://api.openai.com", {}, + mock_500, + mock_500.text, + payload, + "https://api.openai.com", + {}, ) # Payload unchanged assert "response_format" in payload @@ -470,15 +459,19 @@ class TestHandleResponseFormatFallback: payload = {"model": "gpt-4o", "response_format": {"type": "json_object"}} - with patch("src.plugins.translate._llm_async_http._get_http_client", - AsyncMock()) as mock_get: + with patch("src.plugins.translate._llm_async_http._get_http_client", AsyncMock()) as mock_get: mock_client = AsyncMock() mock_get.return_value = mock_client await _handle_response_format_fallback( - mock_400, mock_400.text, payload, - "https://api.openai.com", {}, + mock_400, + mock_400.text, + payload, + "https://api.openai.com", + {}, ) assert "response_format" in payload mock_client.post.assert_not_called() + + # #endregion Test.LLMAsyncHttpClient diff --git a/docker-compose.enterprise-clean.yml b/docker-compose.enterprise-clean.yml index cf70a4c9..f5fd359a 100644 --- a/docker-compose.enterprise-clean.yml +++ b/docker-compose.enterprise-clean.yml @@ -56,10 +56,6 @@ services: FEATURES__HEALTH_MONITOR: ${FEATURES__HEALTH_MONITOR:-true} # Путь до сертификатов внутри контейнера — всегда /opt/certs CERTS_PATH: /opt/certs - # URL CA-сертификатов для LLM-провайдеров (скачка на старте, DER→PEM конвертация) - LLM_CA_CERT_URLS: ${LLM_CA_CERT_URLS:-} - # Отключить SSL verify для LLM (когда корп. сертификаты не установлены в контейнере) - LLM_SSL_VERIFY: ${LLM_SSL_VERIFY:-true} ports: - "${BACKEND_HOST_PORT:-8001}:8000" volumes: @@ -102,11 +98,14 @@ services: LLM_BASE_URL: ${LLM_BASE_URL:-https://api.openai.com/v1} LLM_MODEL: ${LLM_MODEL:-gpt-4o} FASTAPI_URL: http://backend:8000 + CERTS_PATH: /opt/certs AUTH_SECRET_KEY: ${AUTH_SECRET_KEY:?Set AUTH_SECRET_KEY in .env.enterprise-clean} SERVICE_JWT: ${SERVICE_JWT:-agent-service-secret} DATABASE_URL: postgresql+psycopg2://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env.enterprise-clean}@${POSTGRES_HOST:?Set POSTGRES_HOST in .env.enterprise-clean}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-ss_tools} GRADIO_SERVER_PORT: 7860 ports: - "${AGENT_HOST_PORT:-7860}:7860" + volumes: + - ${CERTS_PATH:-./certs}:/opt/certs:ro # #endregion docker.compose.enterprise-clean diff --git a/docker-compose.yml b/docker-compose.yml index 2c256f46..6425b18f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,12 +37,11 @@ services: INITIAL_ADMIN_PASSWORD: ${INITIAL_ADMIN_PASSWORD:-} FEATURES__DATASET_REVIEW: ${FEATURES__DATASET_REVIEW:-true} FEATURES__HEALTH_MONITOR: ${FEATURES__HEALTH_MONITOR:-true} - LLM_CA_CERT_URLS: ${LLM_CA_CERT_URLS:-} - LLM_SSL_VERIFY: ${LLM_SSL_VERIFY:-true} ports: - "${BACKEND_HOST_PORT:-8001}:8000" volumes: - ./storage:/app/storage + - ${CERTS_PATH:-./certs}:/opt/certs:ro agent: build: @@ -65,6 +64,8 @@ services: GRADIO_SERVER_PORT: 7860 ports: - "7860" # internal only, proxied through nginx + volumes: + - ${CERTS_PATH:-./certs}:/opt/certs:ro frontend: build: diff --git a/docker/Dockerfile.agent b/docker/Dockerfile.agent index 9137b1d1..fd4ff985 100644 --- a/docker/Dockerfile.agent +++ b/docker/Dockerfile.agent @@ -16,9 +16,9 @@ FROM python:3.11-slim WORKDIR /app -# Install system deps for pdfplumber + psycopg (v3 needs libpq) +# Install system deps for pdfplumber + psycopg (v3 needs libpq) + certs RUN apt-get update && apt-get install -y --no-install-recommends \ - libgl1 libglib2.0-0 libpq5 && rm -rf /var/lib/apt/lists/* + libgl1 libglib2.0-0 libpq5 ca-certificates openssl && rm -rf /var/lib/apt/lists/* # Python dependencies — agent runtime (gradio, langgraph, httpx, pdfplumber, ...) # Без sentence-transformers — embedding router безопасно отключается при ImportError. @@ -47,10 +47,15 @@ COPY backend/src/core/logger.py /app/backend/src/core/logger.py COPY backend/src/core/ws_log_handler.py /app/backend/src/core/ws_log_handler.py COPY backend/src/agent/ /app/backend/src/agent/ +# Entrypoint for corporate CA cert installation +COPY docker/certs.sh /app/docker/certs.sh +COPY docker/agent.entrypoint.sh /app/docker/agent.entrypoint.sh +RUN chmod +x /app/docker/agent.entrypoint.sh /app/docker/certs.sh + # Gradio server ENV GRADIO_SERVER_NAME=0.0.0.0 ENV GRADIO_SERVER_PORT=7860 WORKDIR /app/backend -CMD ["python", "-m", "src.agent.run"] +ENTRYPOINT ["/app/docker/agent.entrypoint.sh"] # #endregion docker.agent.Dockerfile diff --git a/docker/agent.entrypoint.sh b/docker/agent.entrypoint.sh new file mode 100644 index 00000000..10a82e8b --- /dev/null +++ b/docker/agent.entrypoint.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# #region docker.agent.entrypoint [C:3] [TYPE Module] [SEMANTICS docker,agent,entrypoint,certs] +# @BRIEF Agent container entrypoint — install corporate CA certs, then start agent. +# @LAYER Infrastructure +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CERTS_PATH="${CERTS_PATH:-/opt/certs}" + +if [ -f "$SCRIPT_DIR/certs.sh" ]; then + source "$SCRIPT_DIR/certs.sh" + install_all_certs +else + echo "[agent-entry] certs.sh not found — skipping corporate CA installation" +fi + +exec python -m src.agent.run +# #endregion docker.agent.entrypoint diff --git a/docker/backend.entrypoint.sh b/docker/backend.entrypoint.sh index 4486e18f..a3d881f7 100755 --- a/docker/backend.entrypoint.sh +++ b/docker/backend.entrypoint.sh @@ -497,8 +497,9 @@ except Exception as exc: # MAIN # ====================================================================== -install_certificates -install_llm_ca_certs +source "$(dirname "$0")/certs.sh" + +install_all_certs install_ca_to_nss # ── C1: Wait for database before any migration logic ── diff --git a/docker/certs.sh b/docker/certs.sh new file mode 100644 index 00000000..0295ae0c --- /dev/null +++ b/docker/certs.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# #region docker.certs [C:3] [TYPE Module] [SEMANTICS docker,certs,ssl,ca,trust] +# @BRIEF Shared certificate installation functions for all containers. +# @LAYER Infrastructure +# @PRE CERTS_PATH points to mounted /opt/certs directory (may be empty or absent). +# @POST Corporate CA certificates installed into system trust store and NSS DB. +# @INVARIANT server.crt, server.key, server.p12, *.key, *.p12, *.pfx are never imported as CA. + +set -euo pipefail + +# ── normalize_cert – PEM/DER detection + conversion ────────────────── + +normalize_cert() { + local input="$1" + local output="$2" + if openssl x509 -in "$input" -inform PEM -noout 2>/dev/null; then + cp "$input" "$output" + return 0 + fi + if openssl x509 -in "$input" -inform DER -noout 2>/dev/null; then + openssl x509 -in "$input" -inform DER -out "$output" -outform PEM 2>/dev/null + return 0 + fi + return 1 +} + +# ── install_all_certs – single unified entry point ─────────────────── + +install_all_certs() { + local certs_dir="${CERTS_PATH:-/opt/certs}" + if [ ! -d "$certs_dir" ]; then + echo "[certs] CERTS_PATH=${certs_dir} не существует – пропускаем" + return 0 + fi + + local target="/usr/local/share/ca-certificates/custom" + mkdir -p "$target" + + echo "[certs] Устанавливаем корпоративные сертификаты из ${certs_dir}..." + local installed=0 skipped=0 invalid=0 + + for f in "$certs_dir"/*; do + [ -f "$f" ] || continue + local name + name="$(basename "$f")" + + # Skip non-CA server/private files + case "$(echo "$name" | tr '[:upper:]' '[:lower:]')" in + server.crt|server.key|server.pem|*.key|*.p12|*.pfx) + echo "[certs] ⏭️ пропускаем: ${name} (server/private файл)" + skipped=$((skipped + 1)) + continue + ;; + esac + + # Accept .crt, .pem, .cer, .der + case "$(echo "$name" | tr '[:upper:]' '[:lower:]')" in + *.crt|*.pem|*.cer|*.der) + # valid cert extension + ;; + *) + echo "[certs] ⚠️ неизвестный формат: ${name} – пропускаем" + skipped=$((skipped + 1)) + continue + ;; + esac + + local out_name="${name%.*}.crt" + local out_file="${target}/${out_name}" + + if normalize_cert "$f" "$out_file"; then + echo "[certs] ✅ ${name} (→ ${out_name})" + installed=$((installed + 1)) + else + echo "[certs] ❌ невалидный сертификат: ${name}" + invalid=$((invalid + 1)) + fi + done + + if [ "$installed" -eq 0 ]; then + echo "[certs] Нет CA-сертификатов для установки" + return 0 + fi + + echo "[certs] Установлено: ${installed}, пропущено: ${skipped}, невалидно: ${invalid}" + + # Update system CA store + if command -v update-ca-certificates &>/dev/null; then + echo "[certs] Обновляем системное хранилище CA-сертификатов..." + update-ca-certificates --fresh 2>&1 | sed 's/^/[certs] /' + fi + + # Create hash symlinks for OpenSSL capath + echo "[certs] Создаём хеш-симлинки..." + local sym_count=0 + for cert_file in "$target"/*.crt; do + [ -f "$cert_file" ] || continue + local cert_name + cert_name="$(basename "$cert_file" .crt)" + local hash_val + hash_val="$(openssl x509 -in "$cert_file" -noout -hash 2>/dev/null || true)" + if [ -n "$hash_val" ]; then + local suffix=0 + local link_path="/etc/ssl/certs/${hash_val}.${suffix}" + while [ -L "$link_path" ]; do + if [ "$(readlink "$link_path")" = "$cert_file" ]; then + break 2 + fi + suffix=$((suffix + 1)) + link_path="/etc/ssl/certs/${hash_val}.${suffix}" + done + if [ ! -L "$link_path" ]; then + ln -sf "$cert_file" "$link_path" + sym_count=$((sym_count + 1)) + fi + fi + done + echo "[certs] Создано хеш-симлинок: ${sym_count}" +} + +# ── install_ca_to_nss – NSS DB for Chromium/Playwright ────────────── + +install_ca_to_nss() { + if ! command -v certutil &>/dev/null; then + return 0 + fi + local target="/usr/local/share/ca-certificates/custom" + if [ ! -d "$target" ] || [ -z "$(ls -A "$target" 2>/dev/null)" ]; then + return 0 + fi + local nssdb="${HOME}/.pki/nssdb" + mkdir -p "$nssdb" + local nss_count=0 + echo "[certs] Импортируем сертификаты в NSS DB..." + for cert_file in "$target"/*.crt; do + [ -f "$cert_file" ] || continue + local cert_name + cert_name="$(basename "$cert_file" .crt)" + certutil -A -d "sql:${nssdb}" -t "C,," -n "custom-${cert_name}" -i "$cert_file" 2>/dev/null && nss_count=$((nss_count + 1)) || true + done + echo "[certs] Импортировано в NSS: ${nss_count}" +} + +# ── skip_server_files – remove server certs from CA dir ────────────── + +skip_server_files_filter() { + # Used by frontend to skip server.crt from being installed as CA + local certs_dir="${1:-/opt/certs}" + [ -d "$certs_dir" ] || return 0 + for f in "$certs_dir"/*.crt "$certs_dir"/*.pem; do + [ -f "$f" ] || continue + case "$(basename "$f" | tr '[:upper:]' '[:lower:]')" in + server.crt|server.key) continue ;; + esac + echo "$f" + done +} +# #endregion docker.certs diff --git a/scripts/diag_container.py b/scripts/diag_container.py index 1c552b29..a82fecbd 100644 --- a/scripts/diag_container.py +++ b/scripts/diag_container.py @@ -46,12 +46,7 @@ def section(title: str) -> None: def check_env() -> None: section("1. Environment variables") - for var in ( - "ENCRYPTION_KEY", - "AUTH_SECRET_KEY", - "LLM_SSL_VERIFY", - "LLM_CA_CERT_URLS", - ): + for var in ("ENCRYPTION_KEY", "AUTH_SECRET_KEY", "CERTS_PATH"): val = os.getenv(var, "") if not val: warn(f"{var} is NOT set") @@ -64,25 +59,36 @@ def check_env() -> None: fp = hashlib.sha256(os.getenv("ENCRYPTION_KEY", "").encode()).hexdigest()[:8] print(f"\n Current key fingerprint: {BOLD}sha256:{fp}{RESET}") - if os.getenv("LLM_SSL_VERIFY", "").lower() in ("false", "0", "no", "off"): - fail( - "LLM_SSL_VERIFY is DISABLED — SSL verification bypassed.\n" - " This masks the root cause. Remove LLM_SSL_VERIFY=false\n" - " and fix cert installation instead." - ) - else: - ok( - "LLM_SSL_VERIFY is ENABLED (capath approach).\n" - " If LLM calls fail, the certificate for the target server\n" - " is not in /etc/ssl/certs/." - ) + +# ── Section 2: CERTS_PATH inventory ────────────────────────────────── -# ── Section 2: System CA store ────────────────────────────────────── +def check_certs_inventory() -> None: + section("2. CERTS_PATH inventory (/opt/certs)") + + from src.core.ssl import cert_dir_inventory + + inv = cert_dir_inventory("/opt/certs") + if not inv["trust_candidates"] and not inv["server_files"]: + warn("CERTS_PATH is empty or not mounted — no corporate CA certs found") + print( + " Put .crt/.pem/.cer/.der CA files in ./certs/ on the host and restart containers." + ) + return + + for name in inv["trust_candidates"]: + ok(f"Trust candidate: {Path(name).name}") + for name in inv["server_files"]: + ok(f"Server file (skipped as CA): {Path(name).name}") + for name in inv["invalid"]: + warn(f"Unrecognized file (skipped): {Path(name).name}") + + +# ── Section 3: System CA store ────────────────────────────────────── def check_system_certs() -> None: - section("2. System CA store (/etc/ssl/certs/)") + section("3. System CA store (/etc/ssl/certs/)") bundle = Path("/etc/ssl/certs/ca-certificates.crt") if not bundle.exists(): @@ -96,78 +102,38 @@ def check_system_certs() -> None: ).stdout.strip() ok(f"ca-certificates.crt exists, {cert_count} certificates in bundle") - certs_dir = Path("/etc/ssl/certs") - custom_pems = list(Path("/usr/local/share/ca-certificates").rglob("*.crt")) - llm_pems = list(Path("/usr/local/share/ca-certificates/llm").rglob("*.crt")) - + custom_pems = list(Path("/usr/local/share/ca-certificates/custom").rglob("*.crt")) if custom_pems: names = [p.name for p in custom_pems] ok(f"Installed CA certs: {', '.join(names)}") else: - warn("No custom .crt files in /usr/local/share/ca-certificates/") + warn("No custom .crt files in /usr/local/share/ca-certificates/custom/") - if llm_pems: - names = [p.name for p in llm_pems] - ok(f"LLM CA certs: {', '.join(names)}") - else: - warn( - "No LLM CA certs downloaded.\n" - " LLM_CA_CERT_URLS is not set — set it in .env.enterprise-clean\n" - " or place the CA .crt in ./certs/ on the host." - ) - - # Check hash symlinks + certs_dir = Path("/etc/ssl/certs") hash_links = [p for p in certs_dir.iterdir() if p.is_symlink()] - llm_links = [l for l in hash_links if "llm" in str(Path(os.readlink(str(l))))] - if llm_links: - ok(f"Hash symlinks for LLM certs: {len(llm_links)}") + custom_links = [l for l in hash_links if "custom" in str(Path(os.readlink(str(l))))] + if custom_links: + ok(f"Hash symlinks for custom certs: {len(custom_links)}") else: - warn("No hash symlinks pointing to LLM cert dir — capath may not find them") + warn("No hash symlinks for custom certs — capath may not find them") -# ── Section 3: OpenSSL connectivity ────────────────────────────────── +# ── Section 4: OpenSSL connectivity ────────────────────────────────── def check_openssl(target: str) -> None: - section(f"3. OpenSSL connectivity ({target})") + section(f"4. OpenSSL connectivity ({target})") - # cafile approach (expected to FAIL with code 20 per ADR-0009 Finding 7) - cafile = "/etc/ssl/certs/ca-certificates.crt" - r = subprocess.run( - [ - "openssl", - "s_client", - "-connect", - target, - "-servername", - target.split(":")[0], - "-CAfile", - cafile, - ], - input="", - capture_output=True, - text=True, - timeout=15, - ) - if "Verify return code: 0" in r.stderr or "Verify return code: 0" in r.stdout: - ok(f"cafile: verify OK (unexpected — usually fails per ADR-0009 Finding 7)") - elif "Verify return code: 20" in (r.stderr + r.stdout): - warn( - f"cafile: verify code 20 (EXPECTED per ADR-0009 — intermediate CA ignored)" - ) - else: - fail(f"cafile: unexpected output, returncode={r.returncode}") - - # capath approach (should succeed) capath = "/etc/ssl/certs/" + host, port = _parse_target(target) r = subprocess.run( [ "openssl", "s_client", "-connect", - target, + f"{host}:{port}", "-servername", - target.split(":")[0], + host, "-CApath", capath, ], @@ -181,21 +147,31 @@ def check_openssl(target: str) -> None: else: fail( f"capath: verify FAILED.\n" - f" The CA certificate for {target} is NOT in /etc/ssl/certs/.\n" - f" Add it via LLM_CA_CERT_URLS or ./certs/ on the host." + f" The CA certificate for {host} is NOT in /etc/ssl/certs/.\n" + f" Put the issuing/root CA .crt into CERTS_PATH (./certs)\n" + f" and restart the container." ) -# ── Section 4: Python SSL context ──────────────────────────────────── +def _parse_target(target: str) -> tuple[str, int]: + if "://" in target: + target = target.split("://")[1] + if ":" in target: + host, port = target.rsplit(":", 1) + return host, int(port) + return target, 443 + + +# ── Section 5: Python SSL context ──────────────────────────────────── def check_python_ssl(target: str) -> None: - section(f"4. Python SSL context ({target})") + section(f"5. Python SSL context ({target})") - # capath approach (what _get_ssl_verify() uses) - ctx = ssl.create_default_context(capath="/etc/ssl/certs/") - host, port = target.split(":") if ":" in target else (target, 443) - port = int(port) + from src.core.ssl import system_ssl_context + + ctx = system_ssl_context() + host, port = _parse_target(target) try: with ctx.wrap_socket(__import__("socket").socket(), server_hostname=host) as s: @@ -208,16 +184,17 @@ def check_python_ssl(target: str) -> None: fail(f"SSLContext(capath): FAILED — {e}") -# ── Section 5: httpx connectivity ──────────────────────────────────── +# ── Section 6: httpx connectivity ──────────────────────────────────── def check_httpx(target: str) -> None: - section(f"5. httpx connectivity ({target})") + section(f"6. httpx connectivity ({target})") try: import httpx + from src.core.ssl import httpx_verify - ctx = ssl.create_default_context(capath="/etc/ssl/certs/") + ctx = httpx_verify() url = f"https://{target}" if not target.startswith("http") else target r = httpx.get(url, verify=ctx, timeout=10, follow_redirects=True) ok(f"httpx(capath): HTTP {r.status_code}") @@ -227,11 +204,11 @@ def check_httpx(target: str) -> None: fail(f"httpx(capath): FAILED — {e}") -# ── Section 6: Encryption health ───────────────────────────────────── +# ── Section 7: Encryption health ───────────────────────────────────── def check_encryption_health() -> None: - section("6. Encryption key health (internal)") + section("7. Encryption key health (internal)") sys.path.insert(0, "/app/backend") try: @@ -246,17 +223,15 @@ def check_encryption_health() -> None: ok(f"LLM providers found: {len(providers)}") for p in providers: if not p.api_key: - warn(f" {p.name}: api_key is EMPTY — no key stored") + warn(f" {p.name}: api_key is EMPTY") continue if not is_fernet_token(p.api_key): - warn( - f" {p.name}: api_key is NOT a fernet token — stored unencrypted?" - ) + warn(f" {p.name}: api_key is NOT a fernet token") continue try: mgr.decrypt(p.api_key) ok(f" {p.name}: api_key decrypts OK (healthy)") - except Exception as e: + except Exception: fail( f" {p.name}: api_key decrypt FAILED — " f"encrypted with different ENCRYPTION_KEY" @@ -291,6 +266,7 @@ def main() -> None: print(f"Target: {target}\n") check_env() + check_certs_inventory() check_system_certs() check_openssl(target) check_python_ssl(target) @@ -299,16 +275,11 @@ def main() -> None: section("Summary") print( - "\nExpected (per ADR-0009):\n" - " - openssl CAfile: code 20 FAIL (intermediate CA ignored)\n" - " - openssl CApath: code 0 OK\n" - " - Python SSLContext(capath): OK\n" - " - httpx(capath): HTTP 200 OK\n" - "\nIf CApath/httpx failures:\n" - " → LLM_CA_CERT_URLS not set → certs not downloaded\n" - " → set LLM_CA_CERT_URLS or add .crt to ./certs/\n" + "\nIf cert verification failures:\n" + " → Put the issuing/root CA for target into CERTS_PATH (./certs)\n" + " → Restart affected containers\n" + " → Rerun this diagnostic\n" ) - print( "\nIf ENCRYPTION_KEY health failures:\n" " → Stored secrets encrypted with old key\n"