refactor(agent): extract agent+shared into standalone packages with full GRACE semantic markup

- Move agent code from backend/src/agent/ to agent/src/ss_tools/agent/
- Extract shared stdlib-only utilities to shared/src/ss_tools/shared/
- Add #region/#endregion contracts to all ~140 functions (INV_1 compliance)
- Update docker files, entrypoint, build scripts for new package layout
- Backend now imports ss_tools.shared._llm_health (no gradio/langchain deps)
- Add specs for 036-039 feature plans
This commit is contained in:
2026-07-07 15:18:24 +03:00
parent ce368429f7
commit b95df37cd5
75 changed files with 2711 additions and 1861 deletions

17
shared/pyproject.toml Normal file
View File

@@ -0,0 +1,17 @@
[build-system]
requires = ["setuptools>=69", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "ss-tools-shared"
version = "0.1.0"
description = "Shared lightweight utilities for superset-tools backend and agent"
requires-python = ">=3.11"
dependencies = []
[project.optional-dependencies]
llm = ["openai>=1.0.0", "httpx>=0.28.1"]
[tool.setuptools.packages.find]
where = ["src"]
include = ["ss_tools.shared*"]

View File

@@ -0,0 +1,8 @@
# #region ss_tools.shared [C:1] [TYPE Package] [SEMANTICS shared,package,init]
# @BRIEF Shared lightweight utilities for superset-tools backend and agent.
# Contains: cot_logger (stdlib-only trace propagation),
# ssl (stdlib-only SSL context),
# logger (lightweight JSON logger, no pydantic),
# _llm_health (LLM health probe, openai+httpx).
# @INVARIANT No dependency on FastAPI, SQLAlchemy, Gradio, LangChain.
# #endregion ss_tools.shared

View File

@@ -0,0 +1,123 @@
# #region AgentChat.LlmHealth [C:3] [TYPE Module] [SEMANTICS agent-chat,llm,health,status]
# @defgroup Shared Shared lightweight utilities for backend and agent.
# @BRIEF LLM provider health check with in-memory cache (30s TTL).
# Uses openai+httpx only (no langchain) — works in both backend and agent containers.
# @LAYER Infrastructure
# @RELATION CALLED_BY -> [api/routes/agent_status.py]
# @RELATION CALLED_BY -> [agent/app.py]
# @RATIONALE agent/app.py imports gradio at module level. The backend container
# does not have gradio installed. The /api/agent/llm-status endpoint needs
# _check_llm_provider_health but must not trigger gradio import.
# Additionally, langchain_openai/langchain_core are only in the agent container
# (requirements-agent.txt), not the backend (requirements-backend.txt).
# This module uses only openai+httpx (available in both containers).
# @POST Returns status string: 'ok' | 'unavailable' | 'timeout' | 'auth_error'.
# @SIDE_EFFECT Makes a probe request to the LLM provider; caches result in module memory.
# @INVARIANT No dependency on Gradio, LangChain, FastAPI, SQLAlchemy, pydantic.
import os
import time
from typing import Any
import httpx
from openai import (
APIConnectionError,
APITimeoutError,
AuthenticationError,
RateLimitError,
)
from .logger import logger
# ── LLM provider health cache ─────────────────────────────────────────
_llm_status: dict[str, Any] = {
"status": "ok",
"last_error": "",
"last_check_ts": 0.0,
}
_LLM_CHECK_CACHE_TTL = 30 # seconds between health checks
_LLM_LAST_ERROR_KEY = "last_llm_error"
_LLM_LAST_ERROR_TS_KEY = "last_llm_error_ts"
# #region AgentChat.LlmHealth.Check [C:2] [TYPE Function] [SEMANTICS agent-chat,llm,health,check]
async def _check_llm_provider_health() -> str:
"""Check LLM provider connectivity. Cached for _LLM_CHECK_CACHE_TTL seconds."""
now = time.time()
if now - _llm_status["last_check_ts"] < _LLM_CHECK_CACHE_TTL:
return _llm_status["status"]
# Fetch LLM config from backend's own API (same as agent container does)
try:
fastapi_url = os.getenv("FASTAPI_URL", "http://localhost:8000")
async with httpx.AsyncClient(timeout=5) as client:
resp = await client.get(f"{fastapi_url}/api/agent/llm-config")
if resp.status_code != 200:
_llm_status["status"] = "unavailable"
_llm_status["last_error"] = f"LLM config endpoint returned {resp.status_code}"
_llm_status["last_check_ts"] = time.time()
return "unavailable"
config = resp.json()
if not config or not config.get("configured"):
_llm_status["status"] = "unavailable"
_llm_status["last_error"] = "No LLM provider configured"
_llm_status["last_check_ts"] = time.time()
return "unavailable"
except Exception as exc:
_llm_status["status"] = "unavailable"
_llm_status["last_error"] = f"Failed to fetch LLM config: {exc}"
_llm_status["last_check_ts"] = time.time()
return "unavailable"
# Probe LLM API using AsyncOpenAI (available in both backend and agent containers)
try:
from openai import AsyncOpenAI
from .ssl import system_ssl_context
api = AsyncOpenAI(
api_key=config.get("api_key", ""),
base_url=config.get("base_url", "https://api.openai.com/v1"),
http_client=httpx.AsyncClient(
verify=system_ssl_context(),
timeout=10,
),
)
await api.chat.completions.create(
model=config.get("default_model", "gpt-4o-mini"),
max_tokens=1,
messages=[{"role": "user", "content": "ping"}],
)
_llm_status["status"] = "ok"
_llm_status["last_error"] = ""
_llm_status["last_check_ts"] = time.time()
return "ok"
except (APIConnectionError, httpx.ConnectError):
_llm_status["status"] = "unavailable"
_llm_status["last_error"] = "Connection refused"
_llm_status["last_check_ts"] = time.time()
return "unavailable"
except (APITimeoutError, httpx.ReadTimeout):
_llm_status["status"] = "timeout"
_llm_status["last_error"] = "Request timed out"
_llm_status["last_check_ts"] = time.time()
return "timeout"
except AuthenticationError:
_llm_status["status"] = "auth_error"
_llm_status["last_error"] = "Invalid API key"
_llm_status["last_check_ts"] = time.time()
return "auth_error"
except RateLimitError:
_llm_status["status"] = "unavailable"
_llm_status["last_error"] = "Rate limited"
_llm_status["last_check_ts"] = time.time()
return "unavailable"
except Exception as exc:
if "usage_metadata" in str(exc) or "total_tokens" in str(exc):
_llm_status["status"] = "ok"
_llm_status["last_error"] = ""
_llm_status["last_check_ts"] = time.time()
return "ok"
logger.explore("LLM health check failed", error=str(exc), extra={"src": "AgentChat.LlmHealth.Check"})
return _llm_status["status"] # return cached status on unexpected errors
# #endregion AgentChat.LlmHealth.Check
# #endregion AgentChat.LlmHealth

View File

@@ -0,0 +1,127 @@
# #region CotLoggerModule [C:4] [TYPE Module] [SEMANTICS logging,cot,json,trace,structured]
# @defgroup Shared Shared lightweight utilities for backend and agent.
# @BRIEF Structured JSON logger implementing the decision audit logging protocol.
# Uses ContextVar for trace_id and span_id propagation across async contexts.
# Provides log(), seed_trace_id(), push_span(), pop_span().
# @LAYER Core
# @PRE Python 3.7+ (ContextVar available).
# @POST JSON log records written to the 'cot' Python logger.
# @SIDE_EFFECT Writes structured JSON to the 'cot' Python logger.
# @DATA_CONTRACT Log call -> Single-line JSON to logging.StreamHandler/file.
# @INVARIANT No dependency on FastAPI, SQLAlchemy, Gradio, LangChain, pydantic.
# Pure stdlib + standard logging.
from contextvars import ContextVar
import logging
from typing import Any
import uuid
# #region cot_trace_context [C:1] [TYPE Data] [SEMANTICS contextvar,trace_id,span_id,propagation]
_trace_id: ContextVar[str] = ContextVar("_trace_id", default="")
_span_id: ContextVar[str] = ContextVar("_span_id", default="")
# #endregion cot_trace_context
# #region cot_logger_instance [C:1] [TYPE Data] [SEMANTICS logger,instance]
cot_logger = logging.getLogger("cot")
# #endregion cot_logger_instance
__all__ = [
"get_trace_id",
"log",
"pop_span",
"push_span",
"seed_trace_id",
"set_trace_id",
]
# #region seed_trace_id [C:1] [TYPE Function] [SEMANTICS trace_id,uuid,contextvar,set]
def seed_trace_id() -> str:
"""Generate a new UUID4 trace ID and store it in the thread-local ContextVar."""
trace_id = str(uuid.uuid4())
_trace_id.set(trace_id)
return trace_id
# #endregion seed_trace_id
# #region set_trace_id [C:1] [TYPE Function] [SEMANTICS trace_id,contextvar,set,public]
def set_trace_id(trace_id: str) -> None:
"""Set a specific trace_id into the ContextVar (e.g. from an incoming header)."""
_trace_id.set(trace_id)
# #endregion set_trace_id
# #region get_trace_id [C:1] [TYPE Function] [SEMANTICS trace_id,contextvar,get,public]
def get_trace_id() -> str:
"""Get the current trace_id from the thread-local ContextVar."""
return _trace_id.get()
# #endregion get_trace_id
# #region push_span [C:1] [TYPE Function] [SEMANTICS span_id,contextvar,stack]
def push_span(span: str) -> str:
"""Push a new span ID onto the context and return the previous span ID."""
prev = _span_id.get()
_span_id.set(span)
return prev
# #endregion push_span
# #region pop_span [C:1] [TYPE Function] [SEMANTICS span_id,contextvar,restore]
def pop_span(prev: str) -> None:
"""Restore a previous span ID into the ContextVar."""
_span_id.set(prev)
# #endregion pop_span
# #region cot_log_function [C:2] [TYPE Function] [SEMANTICS log,json,structured,marker]
def log(
src: str,
marker: str,
intent: str,
payload: dict[str, Any] | None = None,
error: str | None = None,
level: str | None = None,
) -> None:
"""Emit a single-line structured JSON log record through the 'cot' logger."""
if level is None:
level = "WARNING" if marker == "EXPLORE" else "INFO"
extra: dict[str, Any] = {
"marker": marker,
"intent": intent,
"src": src,
"span_id": _span_id.get() or None,
}
if payload is not None:
extra["payload"] = payload
if error is not None:
extra["error"] = error
log_func = {
"WARNING": cot_logger.warning,
"ERROR": cot_logger.error,
"DEBUG": cot_logger.debug,
}.get(level, cot_logger.info)
log_func(intent, extra=extra)
# #endregion cot_log_function
# #region MarkerLogger [C:2] [TYPE Class] [SEMANTICS logger,proxy,marker,syntactic-sugar]
class MarkerLogger:
"""Thin proxy over the cot_logger.log() function."""
def __init__(self, module_name: str) -> None:
self._module_name = module_name
def reason(self, intent: str, *, payload: dict[str, Any] | None = None) -> None:
log(self._module_name, "REASON", intent, payload=payload)
def reflect(self, intent: str, *, payload: dict[str, Any] | None = None) -> None:
log(self._module_name, "REFLECT", intent, payload=payload)
def explore(
self,
intent: str,
*,
payload: dict[str, Any] | None = None,
error: str | None = None,
) -> None:
log(self._module_name, "EXPLORE", intent, payload=payload, error=error)
# #endregion MarkerLogger
# #endregion CotLoggerModule

View File

@@ -0,0 +1,147 @@
# #region SharedLoggerModule [C:3] [TYPE Module] [SEMANTICS logging,json,formatter,structured]
# @BRIEF Lightweight JSON logger for agent container.
# Provides the same .reason()/.reflect()/.explore() API as backend's
# src.core.logger, but without pydantic, WebSocketLogHandler, or
# RotatingFileHandler dependencies.
# @INVARIANT No dependency on FastAPI, SQLAlchemy, Gradio, LangChain, pydantic.
# @INVARIANT Every log.info/error/warning call produces single-line JSON.
# @DATA_CONTRACT Log record format:
# {ts, level, trace_id, src, marker, intent, span_id?, payload?, error?}
import json
import logging
import sys
import types
from datetime import UTC, datetime
from .cot_logger import _span_id, _trace_id
# #region CotJsonFormatter [C:2] [TYPE Class] [SEMANTICS logging,formatter,json,cot]
class CotJsonFormatter(logging.Formatter):
"""JSON formatter matching the backend decision audit protocol.
Reads structured data from the LogRecord's extra attributes.
"""
def format(self, record):
marker = getattr(record, 'marker', None)
intent = getattr(record, 'intent', None)
payload = getattr(record, 'payload', None)
error = getattr(record, 'error', None)
src = getattr(record, 'src', None) or record.name
if not marker:
marker = "REASON"
if not intent:
intent = record.getMessage()
log_obj = {
"ts": datetime.now(UTC).isoformat(timespec="milliseconds"),
"level": record.levelname,
"trace_id": _trace_id.get() or "no-trace",
"src": src,
"marker": marker,
"intent": intent,
}
span_id = _span_id.get()
if span_id:
log_obj["span_id"] = span_id
if payload is not None:
log_obj["payload"] = payload
if error is not None:
log_obj["error"] = error
return json.dumps(log_obj, ensure_ascii=False, default=str)
# #endregion CotJsonFormatter
# #region shared_logger_setup [C:2] [TYPE Function] [SEMANTICS logging,setup,instance]
def _setup_logger() -> logging.Logger:
"""Create and configure the shared JSON logger.
Guards against duplicate handlers: if the logger already has handlers
(e.g., configured by backend's src.core.logger), skip setup.
"""
log = logging.getLogger("superset_tools_app")
# If already configured (e.g. by backend src.core.logger), skip
if log.handlers:
return log
log.setLevel(logging.INFO)
log.propagate = False
formatter = CotJsonFormatter()
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setFormatter(formatter)
log.addHandler(console_handler)
return log
logger = _setup_logger()
# #endregion shared_logger_setup
# #region explore [C:2] [TYPE Function] [SEMANTICS log,explore,marker,warning]
def explore(self, msg, *args, **kwargs):
"""Log an EXPLORE marker (WARNING level) with structured extra data."""
user_extra = kwargs.pop('extra', {})
error_val = kwargs.pop('error', None)
payload = kwargs.pop('payload', None)
extra: dict = {'marker': 'EXPLORE', 'intent': msg}
if error_val is not None:
extra['error'] = error_val
if not payload and len(args) == 1 and isinstance(args[0], dict):
payload = args[0]
args = ()
if payload is not None:
extra['payload'] = payload
extra.update(user_extra)
self.warning(msg, extra=extra, **kwargs)
# #endregion explore
# #region reason [C:2] [TYPE Function] [SEMANTICS log,reason,marker,info]
def reason(self, msg, *args, **kwargs):
"""Log a REASON marker (INFO level) with structured extra data."""
user_extra = kwargs.pop('extra', {})
error_val = kwargs.pop('error', None)
payload = kwargs.pop('payload', None)
extra: dict = {'marker': 'REASON', 'intent': msg}
if error_val is not None:
extra['error'] = error_val
if not payload and len(args) == 1 and isinstance(args[0], dict):
payload = args[0]
args = ()
if payload is not None:
extra['payload'] = payload
extra.update(user_extra)
self.info(msg, extra=extra, **kwargs)
# #endregion reason
# #region reflect [C:2] [TYPE Function] [SEMANTICS log,reflect,marker,info]
def reflect(self, msg, *args, **kwargs):
"""Log a REFLECT marker (INFO level) with structured extra data."""
user_extra = kwargs.pop('extra', {})
error_val = kwargs.pop('error', None)
payload = kwargs.pop('payload', None)
extra: dict = {'marker': 'REFLECT', 'intent': msg}
if error_val is not None:
extra['error'] = error_val
if not payload and len(args) == 1 and isinstance(args[0], dict):
payload = args[0]
args = ()
if payload is not None:
extra['payload'] = payload
extra.update(user_extra)
self.info(msg, extra=extra, **kwargs)
# #endregion reflect
# Monkey-patch the convenience methods onto the logger instance
logger.explore = types.MethodType(explore, logger)
logger.reason = types.MethodType(reason, logger)
logger.reflect = types.MethodType(reflect, logger)
# #endregion SharedLoggerModule

View File

@@ -0,0 +1,87 @@
# #region CoreSslTrust [C:4] [TYPE Module] [SEMANTICS ssl,tls,trust,certs,security]
# @BRIEF Centralized SSL/TLS trust context for all Python HTTP clients.
# @LAYER Infrastructure
# @INVARIANT Never returns verify=False from environment configuration.
# The application does not disable TLS verification via env var.
# All trust anchors come from the system CA store populated
# at container startup via CERTS_PATH (/opt/certs) mount.
# @INVARIANT No dependency on FastAPI, SQLAlchemy, Gradio, LangChain, pydantic.
import os
import ssl
from pathlib import Path
DEFAULT_CAPATH = "/etc/ssl/certs"
# #region CoreSslTrust.SystemContext [C:1] [TYPE Function] [SEMANTICS ssl,context,system]
# @ingroup Shared
# @BRIEF Return an SSLContext that trusts the system CA store.
def system_ssl_context() -> ssl.SSLContext:
"""Return an SSLContext that trusts the system CA store."""
if os.path.isdir(DEFAULT_CAPATH):
return ssl.create_default_context(capath=DEFAULT_CAPATH)
return ssl.create_default_context()
# #endregion CoreSslTrust.SystemContext
# #region CoreSslTrust.HttpxVerify [C:1] [TYPE Function] [SEMANTICS ssl,httpx,verify]
# @ingroup Shared
# @BRIEF Return SSL verification context for httpx clients.
def httpx_verify() -> ssl.SSLContext:
"""Return SSL verification context for httpx clients."""
return system_ssl_context()
# #endregion CoreSslTrust.HttpxVerify
# #region CoreSslTrust.DescribeContext [C:2] [TYPE Function] [SEMANTICS ssl,diagnostics,describe]
# @ingroup Shared
# @BRIEF Return a diagnostic description of the SSL verification state.
def describe_context(ctx: ssl.SSLContext | bool | None) -> str:
"""Return a diagnostic description of the SSL verification state."""
if ctx is False:
return "DISABLED (verify=False — should not happen in production)"
if ctx is None:
return "DEFAULT (certifi)"
if isinstance(ctx, ssl.SSLContext):
bundle = Path(DEFAULT_CAPATH) / "ca-certificates.crt"
cert_count = 0
if bundle.exists():
cert_count = bundle.read_bytes().count(b"-----BEGIN CERTIFICATE-----")
mode = "CERT_REQUIRED" if ctx.verify_mode == ssl.CERT_REQUIRED else str(ctx.verify_mode)
return f"SSLContext(verify_mode={mode}, system_certs={cert_count})"
return f"unknown({type(ctx).__name__})"
# #endregion CoreSslTrust.DescribeContext
# #region CoreSslTrust.CertInventory [C:2] [TYPE Function] [SEMANTICS ssl,certs,inventory,diagnostics]
# @ingroup Shared
# @BRIEF Inventory /opt/certs for operator diagnostics.
def cert_dir_inventory(certs_path: str = "/opt/certs") -> dict:
"""Inventory /opt/certs for operator diagnostics."""
result: dict[str, list[str]] = {
"trust_candidates": [],
"server_files": [],
"invalid": [],
}
base = Path(certs_path)
if not base.is_dir():
return result
for f in sorted(base.iterdir()):
if not f.is_file():
continue
name = f.name.lower()
if name in ("server.crt", "server.key", "server.pem") or any(
name.endswith(ext) for ext in (".key", ".p12", ".pfx")
):
result["server_files"].append(str(f))
continue
if any(name.endswith(ext) for ext in (".crt", ".pem", ".cer", ".der")):
result["trust_candidates"].append(str(f))
else:
result["invalid"].append(str(f))
return result
# #endregion CoreSslTrust.CertInventory
# #endregion CoreSslTrust