refactor(ssl): centralize SSL trust management, remove LLM_SSL_VERIFY

Centralized SSL via one contract: CERTS_PATH=/opt/certs mounted into all containers.

Backend:
  - NEW: backend/src/core/ssl.py — system_ssl_context(), httpx_verify(),
    cert_dir_inventory()
  - LLMClient._get_ssl_verify() → delegates to core.ssl
  - _llm_async_http._get_verify() → delegates to core.ssl
  - Removed LLM_SSL_VERIFY env reading from all runtime code

Docker:
  - NEW: docker/certs.sh — shared cert installer (PEM/DER/cer to .crt conversion,
    update-ca-certificates, hash symlinks, NSS import)
  - NEW: docker/agent.entrypoint.sh — agent entrypoint with cert installation
  - backend.entrypoint.sh → uses certs.sh instead of install_llm_ca_certs
  - Dockerfile.agent → adds ca-certificates, openssl, entrypoint

Compose:
  - Removed LLM_CA_CERT_URLS and LLM_SSL_VERIFY from all compose files
  - Added CERTS_PATH volume mount to agent (dev + enterprise)
  - Added certs volume mount to backend/agent in dev compose

Env examples:
  - Removed LLM_SSL_VERIFY, LLM_CA_CERT_URLS from .env.example,
    .env.enterprise-clean.example, .env.current.example, .env.master.example,
    backend/.env.example
  - Enhanced CERTS_PATH comments with accepted formats

Diagnostics:
  - diag_container.py: removed LLM_* checks, added CERTS_PATH inventory,
    uses core.ssl for context creation

Tests:
  - Updated test_llm_analysis_service, test_llm_async_http,
    test_client_headers to verify centralized ssl context (no env disable)
  - 4/4 SSL tests pass
This commit is contained in:
2026-07-06 21:00:28 +03:00
parent 43e3f4135e
commit 8fd23f7ea1
16 changed files with 1200 additions and 1048 deletions

102
backend/src/core/ssl.py Normal file
View File

@@ -0,0 +1,102 @@
# #region CoreSslTrust [C:4] [TYPE Module] [SEMANTICS ssl,tls,trust,certs,security]
# @BRIEF Centralized SSL/TLS trust context for all Python HTTP clients.
# @LAYER Infrastructure
# @INVARIANT Never returns verify=False from environment configuration.
# The application does not disable TLS verification via env var.
# All trust anchors come from the system CA store populated
# at container startup via CERTS_PATH (/opt/certs) mount.
# @RATIONALE Previously LLM_SSL_VERIFY, LLM_CA_CERT_URLS, and per-client
# _get_verify / _get_ssl_verify functions were scattered across
# llm_analysis, translate, and agent code. This centralizes
# them into one contract: system CA store is the single source
# of trust, populated once per container at startup.
# @REJECTED Per-client SSL verify toggle was rejected — caused operators
# to manage LLM TLS separately from Superset/Git/backend TLS,
# and allowed LLM_SSL_VERIFY=false to become a permanent
# workaround instead of fixing cert installation.
import os
import ssl
from pathlib import Path
DEFAULT_CAPATH = "/etc/ssl/certs"
# ── Public API ────────────────────────────────────────────────────────
def system_ssl_context() -> ssl.SSLContext:
"""Return an SSLContext that trusts the system CA store.
Prefers capath over cafile because OpenSSL 3.x ignores intermediate
CA certificates in flat bundle files (ADR-0009 Finding 7).
Falls back to default context if capath directory is not available
(rare edge case, e.g. minimal containers without ca-certificates).
"""
if os.path.isdir(DEFAULT_CAPATH):
return ssl.create_default_context(capath=DEFAULT_CAPATH)
return ssl.create_default_context()
def httpx_verify() -> ssl.SSLContext:
"""Return SSL verification context for httpx clients.
Alias for system_ssl_context() to provide a clear API for the
main HTTP library used across the project (httpx).
"""
return system_ssl_context()
def describe_context(ctx: ssl.SSLContext | bool | None) -> str:
"""Return a diagnostic description of the SSL verification state.
Does NOT leak key material or secret values.
"""
if ctx is False:
return "DISABLED (verify=False — should not happen in production)"
if ctx is None:
return "DEFAULT (certifi)"
if isinstance(ctx, ssl.SSLContext):
capath = getattr(ctx, "capath", None) or getattr(ctx, "_capath", None)
cafile = getattr(ctx, "cafile", None) or getattr(ctx, "_cafile", None)
parts = []
if cafile:
parts.append(f"cafile={cafile}")
if capath:
parts.append(f"capath={capath}")
parts.append(f"verify_mode={'CERT_REQUIRED' if ctx.verify_mode == ssl.CERT_REQUIRED else ctx.verify_mode}")
return "SSLContext(" + ", ".join(parts) + ")"
return f"unknown({type(ctx).__name__})"
def cert_dir_inventory(certs_path: str = "/opt/certs") -> dict:
"""Inventory /opt/certs for operator diagnostics.
Returns counts of recognized trust candidates, server/private files,
and unrecognized files.
"""
result: dict[str, list[str]] = {
"trust_candidates": [],
"server_files": [],
"invalid": [],
}
base = Path(certs_path)
if not base.is_dir():
return result
for f in sorted(base.iterdir()):
if not f.is_file():
continue
name = f.name.lower()
if name in ("server.crt", "server.key", "server.pem") or any(name.endswith(ext) for ext in (".key", ".p12", ".pfx")):
result["server_files"].append(str(f))
continue
if any(name.endswith(ext) for ext in (".crt", ".pem", ".cer", ".der")):
result["trust_candidates"].append(str(f))
else:
result["invalid"].append(str(f))
return result
# #endregion CoreSslTrust

View File

@@ -38,6 +38,7 @@ SCREENSHOT_SERVICE_TIMEOUT_MS = 120000
LLM_HTTP_TIMEOUT_S = 120 # seconds (httpx client timeout)
DEFAULT_USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
# #region ScreenshotService [C:4] [TYPE Class] [SEMANTICS llm,screenshot,playwright]
# @defgroup LLMAnalysis Module group.
# @BRIEF Handles capturing screenshots of Superset dashboards.
@@ -48,6 +49,7 @@ class ScreenshotService:
# @PRE env is a valid Environment object.
def __init__(self, env: Environment):
self.env = env
# #endregion ScreenshotService.__init__
# #region ScreenshotService._find_first_visible_locator [TYPE Function]
@@ -65,6 +67,7 @@ class ScreenshotService:
except Exception:
continue
return None
# #endregion ScreenshotService._find_first_visible_locator
# #region ScreenshotService._iter_login_roots [TYPE Function]
@@ -81,6 +84,7 @@ class ScreenshotService:
except Exception:
pass
return roots
# #endregion ScreenshotService._iter_login_roots
# #region ScreenshotService._extract_hidden_login_fields [TYPE Function]
@@ -102,6 +106,7 @@ class ScreenshotService:
except Exception:
continue
return hidden_fields
# #endregion ScreenshotService._extract_hidden_login_fields
# #region ScreenshotService._extract_csrf_token [TYPE Function]
@@ -111,6 +116,7 @@ class ScreenshotService:
async def _extract_csrf_token(self, page) -> str:
hidden_fields = await self._extract_hidden_login_fields(page)
return str(hidden_fields.get("csrf_token") or "").strip()
# #endregion ScreenshotService._extract_csrf_token
# #region ScreenshotService._response_looks_like_login_page [TYPE Function]
@@ -130,6 +136,7 @@ class ScreenshotService:
'name="csrf_token"',
]
return sum(marker in normalized for marker in markers) >= 3
# #endregion ScreenshotService._response_looks_like_login_page
# #region ScreenshotService._redirect_looks_authenticated [TYPE Function]
@@ -141,6 +148,7 @@ class ScreenshotService:
if not normalized:
return True
return "/login" not in normalized
# #endregion ScreenshotService._redirect_looks_authenticated
# #region ScreenshotService._submit_login_via_form_post [TYPE Function]
@@ -184,11 +192,7 @@ class ScreenshotService:
response_url = str(getattr(response, "url", "") or "")
response_status = int(getattr(response, "status", 0) or 0)
response_headers = dict(getattr(response, "headers", {}) or {})
redirect_location = str(
response_headers.get("location")
or response_headers.get("Location")
or ""
).strip()
redirect_location = str(response_headers.get("location") or response_headers.get("Location") or "").strip()
redirect_statuses = {301, 302, 303, 307, 308}
if response_status in redirect_statuses:
redirect_authenticated = self._redirect_looks_authenticated(redirect_location)
@@ -206,6 +210,7 @@ class ScreenshotService:
payload={"status": response_status, "url": response_url, "login_markup": looks_like_login_page, "snippet": text_snippet},
)
return not looks_like_login_page
# #endregion ScreenshotService._submit_login_via_form_post
# #region ScreenshotService._find_login_field_locator [TYPE Function]
@@ -246,6 +251,7 @@ class ScreenshotService:
return locator
return None
# #endregion ScreenshotService._find_login_field_locator
# #region ScreenshotService._find_submit_locator [TYPE Function]
@@ -266,6 +272,7 @@ class ScreenshotService:
if locator:
return locator
return None
# #endregion ScreenshotService._find_submit_locator
# #region ScreenshotService._goto_resilient [TYPE Function]
@@ -289,6 +296,7 @@ class ScreenshotService:
error=str(primary_error),
)
return await page.goto(url, wait_until=fallback_wait_until, timeout=timeout)
# #endregion ScreenshotService._goto_resilient
# #region ScreenshotService._wait_for_charts_stabilized [TYPE Function] [C:2]
@@ -302,7 +310,8 @@ class ScreenshotService:
# Short initial delay for rendering pipeline to start
await asyncio.sleep(0.5)
try:
await page.wait_for_function("""() => {
await page.wait_for_function(
"""() => {
const charts = document.querySelectorAll('.chart-container canvas, .slice_container svg, .grid-content canvas');
if (charts.length === 0) return true;
return Array.from(charts).some(c => {
@@ -313,9 +322,12 @@ class ScreenshotService:
}
return false;
});
}""", timeout=timeout_ms)
}""",
timeout=timeout_ms,
)
except Exception:
logger.explore("Chart stabilization wait timed out, proceeding anyway", error="Timed out waiting for charts to render")
# #endregion ScreenshotService._wait_for_charts_stabilized
# #region ScreenshotService._wait_for_resize_rendered [TYPE Function] [C:2]
@@ -327,15 +339,20 @@ class ScreenshotService:
async def _wait_for_resize_rendered(self, page, chart_count_before: dict, timeout_ms: int = 10000):
"""Wait for charts to re-render after viewport resize, with polling."""
try:
await page.wait_for_function("""(preCounts) => {
await page.wait_for_function(
"""(preCounts) => {
const currentCharts = document.querySelectorAll('.chart-container, .slice_container').length;
const currentCanvases = document.querySelectorAll('canvas').length;
const currentSvgs = document.querySelectorAll('.chart-container svg, .slice_container svg').length;
// At least one chart element must be present
return currentCharts > 0 && (currentCanvases > 0 || currentSvgs > 0);
}""", arg=chart_count_before, timeout=timeout_ms)
}""",
arg=chart_count_before,
timeout=timeout_ms,
)
except Exception:
logger.explore("Re-render wait timed out after viewport resize, proceeding anyway", error="Timed out waiting for charts to re-render after resize")
# #endregion ScreenshotService._wait_for_resize_rendered
# #region ScreenshotService._save_debug_screenshot [TYPE Function] [C:1]
@@ -351,6 +368,7 @@ class ScreenshotService:
return debug_path
except Exception:
return None
# #endregion ScreenshotService._save_debug_screenshot
# #region ScreenshotService._launch_and_login [TYPE Function] [C:4]
@@ -424,21 +442,13 @@ class ScreenshotService:
if username_locator is not None:
await username_locator.fill(self.env.username)
password_locator = (
await self._find_login_field_locator(page, "password")
if username_locator is not None
else None
)
password_locator = await self._find_login_field_locator(page, "password") if username_locator is not None else None
if username_locator is not None and not password_locator:
raise RuntimeError("Could not find password input field on login page")
if password_locator is not None:
await password_locator.fill(self.env.password)
submit_locator = (
await self._find_submit_locator(page)
if username_locator is not None
else None
)
submit_locator = await self._find_submit_locator(page) if username_locator is not None else None
if username_locator is not None and not submit_locator:
raise RuntimeError("Could not find submit button on login page")
if submit_locator is not None:
@@ -451,11 +461,7 @@ class ScreenshotService:
pass
if not used_direct_form_login and "/login" in page.url:
error_msg = (
await page.locator(".alert-danger, .error-message").text_content()
if await page.locator(".alert-danger, .error-message").count() > 0
else "Unknown error"
)
error_msg = await page.locator(".alert-danger, .error-message").text_content() if await page.locator(".alert-danger, .error-message").count() > 0 else "Unknown error"
raise RuntimeError(f"Login failed: {error_msg}")
except Exception as e:
raise RuntimeError(f"Login failed: {e!s}")
@@ -533,6 +539,7 @@ class ScreenshotService:
await self._wait_for_charts_stabilized(page)
logger.reflect("Login and navigation to dashboard successful", payload={"dashboard_id": dashboard_id})
return browser, context, page
# #endregion ScreenshotService._launch_and_login
# #region ScreenshotService.capture_dashboard_chunks [TYPE Function] [C:4]
@@ -716,6 +723,7 @@ class ScreenshotService:
return results
finally:
await browser.close()
# #endregion ScreenshotService.capture_dashboard_chunks
# #region ScreenshotService.capture_dashboard [TYPE Function] [C:4]
@@ -767,6 +775,7 @@ class ScreenshotService:
# 4. Return JPEGs for LLM — caller cleans up after analysis
return jpeg_paths, archive_results
# #endregion ScreenshotService.capture_dashboard
# #region ScreenshotService._convert_screenshots_for_llm [TYPE Function] [C:2]
@@ -802,6 +811,7 @@ class ScreenshotService:
logger.explore("Failed to convert screenshot for LLM", payload={"png_path": png_path}, error=str(e))
return jpeg_paths
# #endregion ScreenshotService._convert_screenshots_for_llm
# #region ScreenshotService._archive_screenshots_as_webp [TYPE Function] [C:2]
@@ -836,6 +846,7 @@ class ScreenshotService:
results.append({"original": png_path, "webp_path": None})
return results
# #endregion ScreenshotService._archive_screenshots_as_webp
# #region ScreenshotService._cleanup_temp_files [TYPE Function] [C:1]
@@ -849,9 +860,13 @@ class ScreenshotService:
os.remove(path)
except Exception as e:
logger.explore("Failed to delete temporary file", payload={"path": path}, error=str(e))
# #endregion ScreenshotService._cleanup_temp_files
# #endregion ScreenshotService
# #region LLMClient [C:4] [TYPE Class] [SEMANTICS llm,client,provider,openai]
# @defgroup LLMAnalysis Module group.
# @BRIEF Wrapper for LLM provider APIs.
@@ -886,10 +901,7 @@ class LLMClient:
# Some OpenAI-compatible gateways are strict about auth header naming.
default_headers = {"Authorization": f"Bearer {self.api_key}"}
if self.provider_type == LLMProviderType.OPENROUTER:
default_headers["HTTP-Referer"] = (
os.getenv("OPENROUTER_SITE_URL", "").strip()
or os.getenv("APP_BASE_URL", "").strip()
)
default_headers["HTTP-Referer"] = os.getenv("OPENROUTER_SITE_URL", "").strip() or os.getenv("APP_BASE_URL", "").strip()
default_headers["X-Title"] = os.getenv("OPENROUTER_APP_NAME", "").strip() or ""
if self.provider_type == LLMProviderType.KILO:
default_headers["Authentication"] = f"Bearer {self.api_key}"
@@ -915,6 +927,7 @@ class LLMClient:
default_headers=default_headers,
http_client=http_client,
)
# #endregion LLMClient.__init__
# #region LLMClient._get_ssl_verify [TYPE Function]
@@ -933,14 +946,10 @@ class LLMClient:
# из единого bundle-файла. Только capath с хеш-симлинками даёт code 0.
@staticmethod
def _get_ssl_verify() -> ssl.SSLContext | bool:
raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
if raw in ("false", "0", "no", "off"):
return False
ca_dir = "/etc/ssl/certs"
if os.path.isdir(ca_dir):
return ssl.create_default_context(capath=ca_dir)
# fallback: если директории нет (редко), используем дефолтный
return ssl.create_default_context()
from ...core.ssl import system_ssl_context
return system_ssl_context()
# #endregion LLMClient._get_ssl_verify
# #region LLMClient._format_connection_error [TYPE Function]
@@ -954,6 +963,7 @@ class LLMClient:
parts.append(f" └─ {type(cause).__name__}: {cause!s}")
cause = cause.__cause__ or cause.__context__
return "\n".join(parts)
# #endregion LLMClient._format_connection_error
# #region LLMClient._supports_json_response_format [TYPE Function]
@@ -971,6 +981,7 @@ class LLMClient:
if "stepfun/" in model or model.startswith("step-"):
return False
return True
# #endregion LLMClient._supports_json_response_format
# #region LLMClient.get_json_completion [TYPE Function]
@@ -993,12 +1004,7 @@ class LLMClient:
# For other exceptions, limit retries
return True
@retry(
stop=stop_after_attempt(5),
wait=wait_exponential(multiplier=2, min=5, max=60),
retry=retry_if_exception(_should_retry),
reraise=True
)
@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=2, min=5, max=60), retry=retry_if_exception(_should_retry), reraise=True)
async def get_json_completion(self, messages: list[dict[str, Any]]) -> dict[str, Any]:
with belief_scope("get_json_completion"):
response = None
@@ -1017,28 +1023,13 @@ class LLMClient:
)
if use_json_mode:
response = await self.client.chat.completions.create(
model=self.default_model,
messages=messages,
response_format={"type": "json_object"}
)
response = await self.client.chat.completions.create(model=self.default_model, messages=messages, response_format={"type": "json_object"})
else:
response = await self.client.chat.completions.create(
model=self.default_model,
messages=messages
)
response = await self.client.chat.completions.create(model=self.default_model, messages=messages)
except Exception as e:
if use_json_mode and (
"JSON mode is not enabled" in str(e)
or "json_object is not supported" in str(e).lower()
or "response_format" in str(e).lower()
or "400" in str(e)
):
if use_json_mode and ("JSON mode is not enabled" in str(e) or "json_object is not supported" in str(e).lower() or "response_format" in str(e).lower() or "400" in str(e)):
logger.explore("JSON mode failed or not supported, falling back to plain text", payload={"model": self.default_model}, error=str(e))
response = await self.client.chat.completions.create(
model=self.default_model,
messages=messages
)
response = await self.client.chat.completions.create(model=self.default_model, messages=messages)
else:
raise e
@@ -1051,7 +1042,7 @@ class LLMClient:
logger.explore("Rate limit hit on LLM call, retrying with backoff", payload={}, error=str(e))
# Extract retry_delay from error metadata if available
retry_delay = 5.0 # Default fallback
retry_delay = 5.0 # Default fallback
try:
# Based on logs, the raw response is in e.body or e.response.json()
# The logs show 'metadata': {'raw': '...'} which suggests a proxy or specific client wrapper
@@ -1065,13 +1056,13 @@ class LLMClient:
retry_delay = float(match.group(1))
else:
# Try to parse from response if it's a standard OpenAI-like error with body
if hasattr(e, 'body') and isinstance(e.body, dict):
if hasattr(e, "body") and isinstance(e.body, dict):
# Some providers put it in details
details = e.body.get('error', {}).get('details', [])
details = e.body.get("error", {}).get("details", [])
for detail in details:
if detail.get('@type') == 'type.googleapis.com/google.rpc.RetryInfo':
delay_str = detail.get('retryDelay', '5s')
retry_delay = float(delay_str.rstrip('s'))
if detail.get("@type") == "type.googleapis.com/google.rpc.RetryInfo":
delay_str = detail.get("retryDelay", "5s")
retry_delay = float(delay_str.rstrip("s"))
break
except Exception as parse_e:
logger.explore("Failed to parse retry delay from error response", payload={}, error=str(parse_e))
@@ -1089,7 +1080,7 @@ class LLMClient:
)
raise
if not response or not hasattr(response, 'choices') or not response.choices:
if not response or not hasattr(response, "choices") or not response.choices:
raise RuntimeError(f"Invalid LLM response: {response}")
content = response.choices[0].message.content
@@ -1111,6 +1102,7 @@ class LLMClient:
return json.loads(json_str)
else:
raise
# #endregion LLMClient.get_json_completion
# #region LLMClient.test_runtime_connection [TYPE Function]
@@ -1127,6 +1119,7 @@ class LLMClient:
}
]
return await self.get_json_completion(messages)
# #endregion LLMClient.test_runtime_connection
# #region LLMClient.fetch_models [TYPE Function]
@@ -1152,6 +1145,7 @@ class LLMClient:
error=str(e),
)
raise
# #endregion LLMClient.fetch_models
# #region LLMClient.analyze_dashboard [TYPE Function]
@@ -1173,6 +1167,7 @@ class LLMClient:
logs=logs,
prompt_template=prompt_template,
)
# #endregion LLMClient.analyze_dashboard
# #region LLMClient._reduce_image_quality [TYPE Function] [C:2]
@@ -1203,6 +1198,7 @@ class LLMClient:
img.save(buffer, format="JPEG", quality=image_quality, optimize=True)
raw = buffer.getvalue()
return base64.b64encode(raw).decode("utf-8"), len(raw)
# #endregion LLMClient._reduce_image_quality
# #region LLMClient._estimate_payload_size [TYPE Function] [C:2]
@@ -1230,6 +1226,7 @@ class LLMClient:
"exceeds_limit": exceeds_limit,
"pct_of_limit": round(total_tokens / model_context * 100, 1),
}
# #endregion LLMClient._estimate_payload_size
# #region LLMClient._deduplicate_issues [TYPE Function] [C:2]
@@ -1294,15 +1291,15 @@ class LLMClient:
# #region LLMClient._call_llm_for_images [TYPE Function] [C:2]
# @PURPOSE Send a single chunk of images to the LLM and return parsed result.
async def _call_llm_for_images(
self, encoded_images: list[str], prompt: str
) -> dict[str, Any]:
async def _call_llm_for_images(self, encoded_images: list[str], prompt: str) -> dict[str, Any]:
content: list[dict] = [{"type": "text", "text": prompt}]
for b64_img in encoded_images:
content.append({
"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{b64_img}"},
})
content.append(
{
"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{b64_img}"},
}
)
messages = [{"role": "user", "content": content}]
return await self.get_json_completion(messages)
@@ -1335,14 +1332,15 @@ class LLMClient:
encoded_images = self._optimize_images(screenshot_paths, max_width, image_quality)
log_text = "\n".join(logs)
tab_list_text = "\n".join(
f" Screenshot {i}: {label}" for i, label in enumerate(tab_labels or [])
) or "Screenshots are in order."
prompt = render_prompt(prompt_template, {
"logs": log_text,
"tab_list": tab_list_text,
"total_chunks": str(len(encoded_images)),
})
tab_list_text = "\n".join(f" Screenshot {i}: {label}" for i, label in enumerate(tab_labels or [])) or "Screenshots are in order."
prompt = render_prompt(
prompt_template,
{
"logs": log_text,
"tab_list": tab_list_text,
"total_chunks": str(len(encoded_images)),
},
)
# 2. Determine chunking
# Default to 8 images per chunk as a safe fallback when max_images is 0 or None
@@ -1362,9 +1360,7 @@ class LLMClient:
# well within the context window at normal quality.
else:
# Single batch: estimate payload and reduce quality if needed
estimate = self._estimate_payload_size(
screenshot_paths, len(prompt) + len(log_text)
)
estimate = self._estimate_payload_size(screenshot_paths, len(prompt) + len(log_text))
if estimate["exceeds_limit"] and image_quality > 30:
logger.reason(
"Reducing image quality to fit context window",
@@ -1373,10 +1369,7 @@ class LLMClient:
encoded_images = self._optimize_images(screenshot_paths, max_width, image_quality=30)
# 3. Split into chunks
chunks: list[list[str]] = [
encoded_images[i:i + chunk_size]
for i in range(0, n_total, chunk_size)
]
chunks: list[list[str]] = [encoded_images[i : i + chunk_size] for i in range(0, n_total, chunk_size)]
# 4. Call LLM — parallel for multiple chunks, single for one
try:
@@ -1390,11 +1383,13 @@ class LLMClient:
for i, cr in enumerate(chunk_results):
if isinstance(cr, Exception):
logger.explore("Multimodal analysis chunk failed", payload={"chunk_index": i + 1, "total_chunks": len(chunks)}, error=str(cr))
valid_results.append({
"status": "UNKNOWN",
"summary": f"Chunk {i + 1} failed: {cr!s}",
"issues": [],
})
valid_results.append(
{
"status": "UNKNOWN",
"summary": f"Chunk {i + 1} failed: {cr!s}",
"issues": [],
}
)
else:
valid_results.append(cr)
@@ -1408,6 +1403,7 @@ class LLMClient:
}
return result
# #endregion LLMClient.analyze_dashboard_multimodal
# #region LLMClient.analyze_dashboard_text_batch [TYPE Function] [C:3]
@@ -1449,27 +1445,22 @@ class LLMClient:
did = p.get("dashboard_id", "UNKNOWN")
top = p.get("topology", "")
first_line = top.split("\n")[0] if top else "(no topology)"
section = (
f'─── Dashboard {i + 1}: "{first_line}" (id: {did}) ───\n'
f"{top}\n\n"
f"Dataset health:\n{p.get('dataset_health', '')}\n\n"
f"Logs:\n{p.get('log_text', '')}"
)
section = f'─── Dashboard {i + 1}: "{first_line}" (id: {did}) ───\n{top}\n\nDataset health:\n{p.get("dataset_health", "")}\n\nLogs:\n{p.get("log_text", "")}'
sections.append(section)
full_prompt = prompt_template.replace("{total_dashboards}", str(len(payloads)))
full_prompt += (
"\n\nRespond with a JSON object containing EACH dashboard's results:\n"
'{"dashboards": [{"dashboard_id": "...", "status": "...", "summary": "...", "issues": [...]}]}\n\n'
)
full_prompt += '\n\nRespond with a JSON object containing EACH dashboard\'s results:\n{"dashboards": [{"dashboard_id": "...", "status": "...", "summary": "...", "issues": [...]}]}\n\n'
full_prompt += "\n---\n".join(sections)
messages = [{"role": "user", "content": full_prompt}]
return await self.get_json_completion(messages)
# #endregion LLMClient.analyze_dashboard_text_batch
# #endregion LLMClient
# #region DatasetHealthChecker [C:3] [TYPE Class]
# @defgroup LLMAnalysis Module group.
# @BRIEF Checks dataset accessibility and KXD connectivity via Superset API.
@@ -1486,6 +1477,7 @@ class DatasetHealthChecker:
# @POST self.client is ready for health checks.
def __init__(self, client: Any):
self.client = client
# #endregion DatasetHealthChecker.__init__
# #region DatasetHealthChecker._call_sync [TYPE Function]
@@ -1498,6 +1490,7 @@ class DatasetHealthChecker:
if asyncio.iscoroutinefunction(method):
return await method(*args, **kwargs)
return await asyncio.to_thread(method, *args, **kwargs)
# #endregion DatasetHealthChecker._call_sync
# #region DatasetHealthChecker.check_dataset_health [TYPE Function]
@@ -1542,6 +1535,7 @@ class DatasetHealthChecker:
"metadata_accessible": False,
"error": str(e),
}
# #endregion DatasetHealthChecker.check_dataset_health
# #region DatasetHealthChecker.check_chart_data [TYPE Function]
@@ -1561,6 +1555,7 @@ class DatasetHealthChecker:
row_count (int|None), error (str|None)
"""
import time
start = time.time()
try:
# Use the client's network layer for the chart data POST.
@@ -1571,7 +1566,8 @@ class DatasetHealthChecker:
network_request = self.client.network.request
result = await self._call_sync(
network_request,
"POST", "/chart/data",
"POST",
"/chart/data",
data=payload,
headers=headers,
)
@@ -1600,6 +1596,7 @@ class DatasetHealthChecker:
"row_count": None,
"error": str(e),
}
# #endregion DatasetHealthChecker.check_chart_data
# #region DatasetHealthChecker.check_dashboard_datasets [TYPE Function]
@@ -1634,11 +1631,7 @@ class DatasetHealthChecker:
for ds_id in sorted(unique_ds_ids):
result = await self.check_dataset_health(ds_id)
# Map affected charts
affected_charts = [
{"chart_id": c.get("slice_id"), "chart_name": c.get("slice_name", f"chart_{c.get('slice_id')}")}
for c in chart_list
if c.get("datasource_id") == ds_id
]
affected_charts = [{"chart_id": c.get("slice_id"), "chart_name": c.get("slice_name", f"chart_{c.get('slice_id')}")} for c in chart_list if c.get("datasource_id") == ds_id]
result["affected_charts"] = affected_charts
dataset_results.append(result)
@@ -1668,9 +1661,13 @@ class DatasetHealthChecker:
"datasets": dataset_results,
"chart_data": chart_data_results,
}
# #endregion DatasetHealthChecker.check_dashboard_datasets
# #endregion DatasetHealthChecker
# #region RedactionService [C:2] [TYPE Module]
# @defgroup LLMAnalysis Module group.
# @BRIEF Redacts PII, credentials, and sensitive data from logs and LLM responses.
@@ -1705,6 +1702,7 @@ class RedactionService:
line = re.sub(pattern, replacement, line, flags=re.IGNORECASE)
redacted.append(line)
return redacted
# #endregion RedactionService.redact_logs
# #region RedactionService.redact_raw_response [TYPE Function] [C:2]
@@ -1717,7 +1715,10 @@ class RedactionService:
for pattern, replacement in RedactionService.PATTERNS:
raw = re.sub(pattern, replacement, raw, flags=re.IGNORECASE)
return raw
# #endregion RedactionService.redact_raw_response
# #endregion RedactionService
# #endregion LLMAnalysisService

View File

@@ -46,10 +46,11 @@ DEFAULT_MAX_TOKENS: int = 8192
# строки в verify=, требует SSLContext.
# @POST Returns ssl.SSLContext with capath when enabled, False when disabled.
def _get_verify() -> ssl.SSLContext | bool:
raw = os.getenv("LLM_SSL_VERIFY", "true").strip().lower()
if raw in ("false", "0", "no", "off"):
return False
return ssl.create_default_context(capath="/etc/ssl/certs/")
from ...core.ssl import httpx_verify
return httpx_verify()
# #endregion _get_verify
@@ -61,14 +62,14 @@ async def _get_http_client() -> httpx.AsyncClient:
if _http_client is None:
ssl_verify = _get_verify()
if ssl_verify is False:
log("LLMAsyncHttpClient", "EXPLORE",
"TLS verification disabled via LLM_SSL_VERIFY=false",
error="TLS verification disabled — traffic to LLM provider is unencrypted")
log("LLMAsyncHttpClient", "EXPLORE", "TLS verification disabled via LLM_SSL_VERIFY=false", error="TLS verification disabled — traffic to LLM provider is unencrypted")
_http_client = httpx.AsyncClient(
verify=ssl_verify,
timeout=httpx.Timeout(180.0),
)
return _http_client
# #endregion _get_http_client
@@ -100,13 +101,7 @@ async def call_openai_compatible(
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
}
system_content = (
"You are a database content translation assistant. "
"Translate the provided text accurately, preserving data semantics. "
"Respond directly with ONLY the JSON result. "
"Do NOT include any reasoning, thinking, chain-of-thought, analysis, "
"or explanation. Output ONLY valid JSON."
)
system_content = "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics. Respond directly with ONLY the JSON result. Do NOT include any reasoning, thinking, chain-of-thought, analysis, or explanation. Output ONLY valid JSON."
payload: dict[str, Any] = {
"model": model,
@@ -127,43 +122,43 @@ async def call_openai_compatible(
payload["reasoning_effort"] = "none"
payload["max_tokens"] = max_tokens
logger.reason(
f"LLM request url={_sanitize_url(base_url)} model={payload.get('model')} "
f"provider_type={provider_type} "
f"response_format={'yes' if 'response_format' in payload else 'no'} "
f"prompt_len={len(prompt)}"
)
logger.reason(f"LLM request url={_sanitize_url(base_url)} model={payload.get('model')} provider_type={provider_type} response_format={'yes' if 'response_format' in payload else 'no'} prompt_len={len(prompt)}")
response, response_text = await _do_http_request(url, headers, payload)
await _handle_response_format_fallback(response, response_text, payload, url, headers)
if not response.is_success:
logger.explore(
f"LLM API error status={response.status_code} "
f"model={payload.get('model')} "
f"body={response_text[:2000]}"
)
logger.explore(f"LLM API error status={response.status_code} model={payload.get('model')} body={response_text[:2000]}")
response.raise_for_status()
data = response.json()
choices = data.get("choices", [])
if not choices:
logger.explore("LLM returned no choices", extra={
"src": "executor", "response_keys": list(data.keys()),
"response_preview": str(data)[:2000],
})
logger.explore(
"LLM returned no choices",
extra={
"src": "executor",
"response_keys": list(data.keys()),
"response_preview": str(data)[:2000],
},
)
raise ValueError("LLM returned no choices")
try:
finish_reason = choices[0].get("finish_reason") or "none"
msg = choices[0].get("message") or {}
except (TypeError, AttributeError) as e:
logger.explore("TypeError processing LLM response choices", extra={
"src": "executor_diag", "error": str(e),
"choices_0_type": type(choices[0]).__name__ if choices else "N/A",
"choices_0_repr": repr(choices[0])[:2000] if choices else "N/A",
"data_type": type(data).__name__, "data_preview": str(data)[:2000],
})
logger.explore(
"TypeError processing LLM response choices",
extra={
"src": "executor_diag",
"error": str(e),
"choices_0_type": type(choices[0]).__name__ if choices else "N/A",
"choices_0_repr": repr(choices[0])[:2000] if choices else "N/A",
"data_type": type(data).__name__,
"data_preview": str(data)[:2000],
},
)
raise ValueError(f"LLM response processing failed: {e}")
# Log provider token usage for batch sizing calibration
@@ -183,28 +178,36 @@ async def call_openai_compatible(
refusal = msg.get("refusal") if isinstance(msg, dict) else None
if refusal:
logger.explore("LLM refused to respond", extra={
"src": "executor", "refusal": str(refusal)[:500], "finish_reason": finish_reason,
})
logger.explore(
"LLM refused to respond",
extra={
"src": "executor",
"refusal": str(refusal)[:500],
"finish_reason": finish_reason,
},
)
raise ValueError(f"LLM refused to respond: {refusal}")
content = msg.get("content") if isinstance(msg, dict) else ""
if not content and isinstance(msg, dict):
content = msg.get("content") or ""
logger.reason(
f"LLM response finish_reason={finish_reason} content_len={len(content)} "
f"msg_keys={list(msg.keys()) if isinstance(msg, dict) else []}"
)
logger.reason(f"LLM response finish_reason={finish_reason} content_len={len(content)} msg_keys={list(msg.keys()) if isinstance(msg, dict) else []}")
if not content:
logger.explore("LLM returned empty content", extra={
"src": "executor", "finish_reason": finish_reason,
"msg_keys": list(msg.keys()) if isinstance(msg, dict) else [],
"response_preview": str(data)[:2000],
})
logger.explore(
"LLM returned empty content",
extra={
"src": "executor",
"finish_reason": finish_reason,
"msg_keys": list(msg.keys()) if isinstance(msg, dict) else [],
"response_preview": str(data)[:2000],
},
)
raise ValueError("LLM returned empty content")
return content, finish_reason
# #endregion call_openai_compatible
@@ -224,34 +227,34 @@ async def _do_http_request(url: str, headers: dict, payload: dict) -> tuple[http
try:
wait = int(retry_after)
except (ValueError, TypeError):
wait = 2 ** _retry_count_429
wait = 2**_retry_count_429
else:
wait = 2 ** _retry_count_429
logger.explore(f"Rate limited (429), retry {_retry_count_429}/{_max_retry_429} after {wait}s",
extra={"src": "executor", "retry_after": retry_after, "wait": wait})
wait = 2**_retry_count_429
logger.explore(f"Rate limited (429), retry {_retry_count_429}/{_max_retry_429} after {wait}s", extra={"src": "executor", "retry_after": retry_after, "wait": wait})
await asyncio.sleep(wait)
if _retry_count_429 >= _max_retry_429:
break
else:
break
return response, response_text
# #endregion _do_http_request
# #region _handle_response_format_fallback [C:1] [TYPE Function]
async def _handle_response_format_fallback(
response: httpx.Response, response_text: str, payload: dict, url: str, headers: dict,
response: httpx.Response,
response_text: str,
payload: dict,
url: str,
headers: dict,
) -> None:
"""Handle 400 errors from structured_outputs not being supported."""
_patterns = ("response_format", "structured_outputs", "structured", "json_object")
if (
not response.is_success
and response.status_code == 400
and any(p in (response_text or "").lower() for p in _patterns)
):
if not response.is_success and response.status_code == 400 and any(p in (response_text or "").lower() for p in _patterns):
client = await _get_http_client()
logger.explore("Structured outputs not supported, retrying without response_format",
extra={"src": "executor"})
logger.explore("Structured outputs not supported, retrying without response_format", extra={"src": "executor"})
payload.pop("response_format", None)
new_response = await client.post(url, headers=headers, json=payload)
# Mutate the original response object with new data
@@ -259,5 +262,7 @@ async def _handle_response_format_fallback(
response._content = new_response.content
response.encoding = new_response.encoding
response.headers = new_response.headers
# #endregion _handle_response_format_fallback
# #endregion LLMAsyncHttpClient