From f22b0cf41c2cdbbdbd0f92ba57bc90e542a974f9 Mon Sep 17 00:00:00 2001 From: busya Date: Wed, 8 Jul 2026 11:10:59 +0300 Subject: [PATCH] chore: add GRACE semantic contracts across agent + backend + build scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add @RELATION/@RATIONALE/@REJECTED headers to agent modules - Add GRACE contracts to backend services, models, schemas - Update build.sh with single-image build commands (build:backend|frontend|agent) - Update docker entrypoint: openssl rsa → openssl pkey (alg-agnostic) - Add @RATIONALE to backend core modules (cot_logger, config_manager, ssl) --- agent/src/ss_tools/agent/__init__.py | 13 + agent/src/ss_tools/agent/_confirmation.py | 2 + agent/src/ss_tools/agent/_context.py | 1 + agent/src/ss_tools/agent/_llm_params.py | 1 + agent/src/ss_tools/agent/_persistence.py | 4 + agent/src/ss_tools/agent/_tool_filter.py | 1 + agent/src/ss_tools/agent/_tool_resolver.py | 2 + agent/src/ss_tools/agent/app.py | 3 + agent/src/ss_tools/agent/context.py | 1 + agent/src/ss_tools/agent/document_parser.py | 1 + agent/src/ss_tools/agent/langgraph_setup.py | 3 + agent/src/ss_tools/agent/middleware.py | 3 + agent/src/ss_tools/agent/tools.py | 2 + backend/.image-built | 0 backend/src/api/routes/validation_tasks.py | 32 +- backend/src/core/auth/logger.py | 8 + backend/src/core/auth/repository.py | 7 + backend/src/core/config_manager.py | 8 + backend/src/core/cot_logger.py | 9 + backend/src/core/migration/archive_parser.py | 8 + backend/src/core/ssl.py | 6 + backend/src/models/auth.py | 8 + backend/src/models/mapping.py | 7 + backend/src/models/profile.py | 8 + backend/src/plugins/migration.py | 9 + backend/src/schemas/auth.py | 8 + backend/src/services/auth_service.py | 7 + .../clean_release/artifact_catalog_loader.py | 8 + .../services/clean_release/policy_engine.py | 9 + .../policy_resolution_service.py | 8 + .../src/services/clean_release/stages/base.py | 9 + .../src/services/reports/report_service.py | 14 + build.sh | 496 +++++++++--------- docker/frontend.entrypoint.sh | 9 +- 34 files changed, 452 insertions(+), 263 deletions(-) create mode 100644 backend/.image-built diff --git a/agent/src/ss_tools/agent/__init__.py b/agent/src/ss_tools/agent/__init__.py index 63c8e120..6b48d9e1 100644 --- a/agent/src/ss_tools/agent/__init__.py +++ b/agent/src/ss_tools/agent/__init__.py @@ -2,4 +2,17 @@ # #region AgentChat [C:3] [TYPE Module] [SEMANTICS agent-chat] # @defgroup AgentChat LangGraph-based Gradio agent — streaming chat with HITL guardrails. # @LAYER Application +# @RELATION DISPATCHES -> [AgentChat.Config] +# @RELATION DISPATCHES -> [AgentChat.Tools] +# @RELATION DISPATCHES -> [AgentChat.Confirmation] +# @RELATION DISPATCHES -> [AgentChat.Context] +# @RELATION DISPATCHES -> [AgentChat.Context.Validate] +# @RELATION DISPATCHES -> [AgentChat.LangGraph.Setup] +# @RELATION DISPATCHES -> [AgentChat.ToolResolver] +# @RELATION DISPATCHES -> [AgentChat.ToolFilter] +# @RELATION DISPATCHES -> [AgentChat.LlmParams] +# @RELATION DISPATCHES -> [AgentChat.Middleware] +# @RELATION DISPATCHES -> [AgentChat.Persistence] +# @RELATION DISPATCHES -> [AgentChat.Document.Parser] +# @RELATION DISPATCHES -> [AgentChat.GradioApp] # #endregion AgentChat diff --git a/agent/src/ss_tools/agent/_confirmation.py b/agent/src/ss_tools/agent/_confirmation.py index 5f0b9c19..5d1197b7 100644 --- a/agent/src/ss_tools/agent/_confirmation.py +++ b/agent/src/ss_tools/agent/_confirmation.py @@ -4,6 +4,8 @@ # @LAYER Service # @RELATION DEPENDS_ON -> [AgentChat.ToolResolver] # @RELATION DEPENDS_ON -> [AgentChat.Tools] +# @RELATION DEPENDS_ON -> [AgentChat.LlmParams] +# @RELATION DEPENDS_ON -> [AgentChat.LangGraph.Setup] # @RATIONALE Extracting confirmation logic into a dedicated module prevents the handler # from exceeding 400 lines and centralises risk classification in one place. from collections.abc import AsyncGenerator diff --git a/agent/src/ss_tools/agent/_context.py b/agent/src/ss_tools/agent/_context.py index 1ecc680c..78fa72ae 100644 --- a/agent/src/ss_tools/agent/_context.py +++ b/agent/src/ss_tools/agent/_context.py @@ -1,5 +1,6 @@ # agent/src/ss_tools/agent/_context.py # #region AgentChat.Context.Validate [C:3] [TYPE Module] [SEMANTICS agent-chat,context,validate,security] +# @ingroup AgentChat # @BRIEF UIContext validation and prompt-injection protection. # @LAYER Service # @POST Passes through contextVersion, objectType, objectId, objectName, envId, route, padding. diff --git a/agent/src/ss_tools/agent/_llm_params.py b/agent/src/ss_tools/agent/_llm_params.py index 70df653e..e991c743 100644 --- a/agent/src/ss_tools/agent/_llm_params.py +++ b/agent/src/ss_tools/agent/_llm_params.py @@ -1,5 +1,6 @@ # agent/src/ss_tools/agent/_llm_params.py # #region AgentChat.LlmParams [C:3] [TYPE Module] [SEMANTICS agent-chat,llm,openai,compatibility] +# @ingroup AgentChat # @BRIEF Build provider-safe ChatOpenAI kwargs and raw OpenAI payloads. # @POST Unsupported sampling parameters are omitted for reasoning/codex models. from typing import Any diff --git a/agent/src/ss_tools/agent/_persistence.py b/agent/src/ss_tools/agent/_persistence.py index 6d4b588d..5c5cce58 100644 --- a/agent/src/ss_tools/agent/_persistence.py +++ b/agent/src/ss_tools/agent/_persistence.py @@ -1,7 +1,11 @@ # agent/src/ss_tools/agent/_persistence.py # #region AgentChat.Persistence [C:3] [TYPE Module] [SEMANTICS agent-chat,persistence,save,prefetch,title] +# @ingroup AgentChat # @BRIEF Conversation persistence helpers — save, clean titles, LLM title generation, prefetch. # @LAYER Service +# @RELATION DEPENDS_ON -> [AgentChat.Config] +# @RELATION DEPENDS_ON -> [AgentChat.LlmParams] +# @RELATION DEPENDS_ON -> [AgentChat.Tools] import asyncio from datetime import datetime diff --git a/agent/src/ss_tools/agent/_tool_filter.py b/agent/src/ss_tools/agent/_tool_filter.py index e2bdbfb4..95e11370 100644 --- a/agent/src/ss_tools/agent/_tool_filter.py +++ b/agent/src/ss_tools/agent/_tool_filter.py @@ -1,5 +1,6 @@ # agent/src/ss_tools/agent/_tool_filter.py # #region AgentChat.ToolFilter [C:3] [TYPE Module] [SEMANTICS agent-chat,tools,filter,context] +# @ingroup AgentChat # @BRIEF Context-aware tool filtering + RBAC enforcement. # @LAYER Service # @DATA_CONTRACT build_tool_pipeline returns a list — never mutates the input list. diff --git a/agent/src/ss_tools/agent/_tool_resolver.py b/agent/src/ss_tools/agent/_tool_resolver.py index de1ef3e1..f3927b01 100644 --- a/agent/src/ss_tools/agent/_tool_resolver.py +++ b/agent/src/ss_tools/agent/_tool_resolver.py @@ -1,6 +1,8 @@ # agent/src/ss_tools/agent/_tool_resolver.py # #region AgentChat.ToolResolver [C:2] [TYPE Module] [SEMANTICS agent-chat,tools,resolution] +# @ingroup AgentChat # @BRIEF Tool resolution helpers for the LangGraph agent. +# @RELATION DEPENDS_ON -> [AgentChat.Tools] # @LAYER Service from typing import Any diff --git a/agent/src/ss_tools/agent/app.py b/agent/src/ss_tools/agent/app.py index 531aa718..3757f3e5 100644 --- a/agent/src/ss_tools/agent/app.py +++ b/agent/src/ss_tools/agent/app.py @@ -9,6 +9,9 @@ # @RELATION DEPENDS_ON -> [AgentChat.Confirmation] # @RELATION DEPENDS_ON -> [AgentChat.Persistence] # @RELATION DEPENDS_ON -> [AgentChat.LangGraph.Setup] +# @RELATION DEPENDS_ON -> [AgentChat.Config] +# @RELATION DEPENDS_ON -> [AgentChat.Tools] +# @RELATION DEPENDS_ON -> [AgentChat.Context] # @RATIONALE Gradio ChatInterface chosen for its built-in streaming, file upload, and multimodal support — avoids custom WebSocket implementation for agent chat. # @REJECTED Custom React chat frontend rejected — Gradio provides free authentication, session management, and mobile-responsive UI out of the box. diff --git a/agent/src/ss_tools/agent/context.py b/agent/src/ss_tools/agent/context.py index 199d1e36..c3875a96 100644 --- a/agent/src/ss_tools/agent/context.py +++ b/agent/src/ss_tools/agent/context.py @@ -1,5 +1,6 @@ # agent/src/ss_tools/agent/context.py # #region AgentChat.Context [C:3] [TYPE Module] [SEMANTICS agent-chat,context,auth] +# @ingroup AgentChat # @BRIEF JWT context propagation for LangGraph tools. # @RATIONALE LangGraph tool execution may run in a different async context, # preventing ContextVar from propagating. Module-level globals diff --git a/agent/src/ss_tools/agent/document_parser.py b/agent/src/ss_tools/agent/document_parser.py index 5b363ad1..e9760a44 100644 --- a/agent/src/ss_tools/agent/document_parser.py +++ b/agent/src/ss_tools/agent/document_parser.py @@ -1,5 +1,6 @@ # agent/src/ss_tools/agent/document_parser.py # #region AgentChat.Document.Parser [C:3] [TYPE Module] [SEMANTICS agent-chat,document,parser] +# @ingroup AgentChat # @BRIEF Parse PDF and XLSX files into text/structured data. # @RELATION DEPENDS_ON -> [EXT:pdfplumber] # @RELATION DEPENDS_ON -> [EXT:openpyxl] diff --git a/agent/src/ss_tools/agent/langgraph_setup.py b/agent/src/ss_tools/agent/langgraph_setup.py index 0a6fc688..01f0ec87 100644 --- a/agent/src/ss_tools/agent/langgraph_setup.py +++ b/agent/src/ss_tools/agent/langgraph_setup.py @@ -1,8 +1,11 @@ # agent/src/ss_tools/agent/langgraph_setup.py # #region AgentChat.LangGraph.Setup [C:4] [TYPE Module] [SEMANTICS agent-chat,langgraph,agent] +# @ingroup AgentChat # @BRIEF LangGraph agent setup: create_react_agent with PostgresSaver. # @PRE LLM provider configured via backend API /api/agent/llm-config. # @POST Compiled StateGraph ready for astream_events(). +# @RELATION DEPENDS_ON -> [AgentChat.Config] +# @RELATION DEPENDS_ON -> [AgentChat.LlmParams] import inspect as _inspect import os diff --git a/agent/src/ss_tools/agent/middleware.py b/agent/src/ss_tools/agent/middleware.py index f9c00613..02254d09 100644 --- a/agent/src/ss_tools/agent/middleware.py +++ b/agent/src/ss_tools/agent/middleware.py @@ -1,6 +1,9 @@ # agent/src/ss_tools/agent/middleware.py # #region AgentChat.Middleware [C:3] [TYPE Module] [SEMANTICS agent-chat,middleware,logging,audit] +# @ingroup AgentChat # @BRIEF Audit logging middleware for the LangGraph agent. +# @RELATION DEPENDS_ON -> [AgentChat.Context] +# @RELATION DEPENDS_ON -> [AgentChat.Tools] from datetime import UTC, datetime from ss_tools.agent.context import get_user_jwt diff --git a/agent/src/ss_tools/agent/tools.py b/agent/src/ss_tools/agent/tools.py index a8932e90..d4f9c277 100644 --- a/agent/src/ss_tools/agent/tools.py +++ b/agent/src/ss_tools/agent/tools.py @@ -1,6 +1,8 @@ # agent/src/ss_tools/agent/tools.py # #region AgentChat.Tools [C:4] [TYPE Module] [SEMANTICS agent-chat,tools,langchain] # @defgroup AgentChat Native LangChain @tool functions. +# @RELATION DEPENDS_ON -> [AgentChat.Config] +# @RELATION DEPENDS_ON -> [AgentChat.Context] # @RATIONALE LangChain @tool decorator chosen over direct FastAPI calls for LangGraph compatibility — tools are auto-registered in the agent's tool-calling loop. # @REJECTED StructuredTool wrapping — native @tool is the single source of truth. # @INVARIANT Every @tool function MUST have a Python docstring (triple-quoted string immediately after the signature). LangChain raises ValueError("Function must have a docstring if description not provided.") when args_schema is provided without a description param AND the function lacks a docstring. This is a runtime blocker — the entire Gradio agent fails to import when ANY single tool violates this invariant. The GRACE @BRIEF comment is NOT a substitute — it's structural metadata invisible to the Python runtime. Rule: BEFORE decorating with @tool(args_schema=...), ensure `"""One-line description."""` is present on the next line. diff --git a/backend/.image-built b/backend/.image-built new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/api/routes/validation_tasks.py b/backend/src/api/routes/validation_tasks.py index 1959eb48..06ad6807 100644 --- a/backend/src/api/routes/validation_tasks.py +++ b/backend/src/api/routes/validation_tasks.py @@ -105,8 +105,8 @@ def _dict_to_run_response(d: dict[str, Any]) -> RunResponse: # #region create_task [C:3] [TYPE Function] # @ingroup Api # @BRIEF Create a new validation task. -# @RELATION CALLS -> [ValidationTaskService.create_task] -# @RELATION CALLS -> [SchedulerService.reload_validation_policy] +# @RELATION CALLS -> [create_task] +# @RELATION CALLS -> [reload_validation_policy] @router.post("", response_model=ValidationTaskResponse, status_code=status.HTTP_201_CREATED) async def create_task( payload: ValidationTaskCreate, @@ -132,7 +132,7 @@ async def create_task( # #region list_tasks [C:3] [TYPE Function] # @ingroup Api # @BRIEF List all validation tasks with pagination and optional filters. -# @RELATION CALLS -> [ValidationTaskService.list_tasks] +# @RELATION CALLS -> [list_tasks] @router.get("", response_model=ValidationTaskListResponse) async def list_tasks( page: int = Query(1, ge=1), @@ -163,7 +163,7 @@ async def list_tasks( # #region get_task [C:3] [TYPE Function] # @ingroup Api # @BRIEF Get a validation task by policy ID. -# @RELATION CALLS -> [ValidationTaskService.get_task] +# @RELATION CALLS -> [get_task] @router.get("/{policy_id}", response_model=ValidationTaskResponse) async def get_task( policy_id: str, @@ -186,8 +186,8 @@ async def get_task( # #region update_task [C:3] [TYPE Function] # @ingroup Api # @BRIEF Update a validation task. -# @RELATION CALLS -> [ValidationTaskService.update_task] -# @RELATION CALLS -> [SchedulerService.reload_validation_policy] +# @RELATION CALLS -> [update_task] +# @RELATION CALLS -> [reload_validation_policy] @router.put("/{policy_id}", response_model=ValidationTaskResponse) async def update_task( policy_id: str, @@ -214,8 +214,8 @@ async def update_task( # #region delete_task [C:3] [TYPE Function] # @ingroup Api # @BRIEF Delete a validation task. -# @RELATION CALLS -> [ValidationTaskService.delete_task] -# @RELATION CALLS -> [SchedulerService.remove_validation_job] +# @RELATION CALLS -> [delete_task] +# @RELATION CALLS -> [remove_validation_job] @router.delete("/{policy_id}", status_code=status.HTTP_204_NO_CONTENT) async def delete_task( policy_id: str, @@ -241,9 +241,9 @@ async def delete_task( # #region toggle_task_status [C:3] [TYPE Function] # @ingroup Api # @BRIEF Toggle a validation task's active status. -# @RELATION CALLS -> [ValidationTaskService.update_task] -# @RELATION CALLS -> [SchedulerService.remove_validation_job] -# @RELATION CALLS -> [SchedulerService.reload_validation_policy] +# @RELATION CALLS -> [update_task] +# @RELATION CALLS -> [remove_validation_job] +# @RELATION CALLS -> [reload_validation_policy] @router.patch("/{policy_id}/status", response_model=ValidationTaskResponse) async def toggle_task_status( policy_id: str, @@ -278,7 +278,7 @@ async def toggle_task_status( # #region trigger_run [C:3] [TYPE Function] # @ingroup Api # @BRIEF Trigger immediate validation for a task. -# @RELATION CALLS -> [ValidationTaskService.trigger_run] +# @RELATION CALLS -> [trigger_run] @router.post("/{policy_id}/run", response_model=TriggerRunResponse) async def trigger_run( policy_id: str, @@ -309,7 +309,7 @@ async def trigger_run( # #region list_all_runs [C:3] [TYPE Function] # @ingroup Api # @BRIEF List validation runs across all tasks with optional filters. -# @RELATION CALLS -> [ValidationTaskService.list_all_runs] +# @RELATION CALLS -> [list_all_runs] @router.get("/runs/all", response_model=RecordListResponse) async def list_all_runs( page: int = Query(1, ge=1), @@ -340,7 +340,7 @@ async def list_all_runs( # #region list_runs [C:3] [TYPE Function] # @ingroup Api # @BRIEF List run history for a validation task. -# @RELATION CALLS -> [ValidationTaskService.get_run_history] +# @RELATION CALLS -> [get_run_history] @router.get("/{policy_id}/runs", response_model=RunListResponse) async def list_runs( policy_id: str, @@ -367,7 +367,7 @@ async def list_runs( # #region get_run_detail [C:3] [TYPE Function] # @ingroup Api # @BRIEF Get full run detail with all records for a validation task. -# @RELATION CALLS -> [ValidationTaskService.get_run_detail] +# @RELATION CALLS -> [get_run_detail] @router.get("/{policy_id}/runs/{run_id}", response_model=RunDetailResponse) async def get_run_detail( policy_id: str, @@ -400,7 +400,7 @@ async def get_run_detail( # #region parse_superset_url [C:3] [TYPE Function] # @ingroup Api # @BRIEF Parse a Superset dashboard URL and extract dashboard context, native filters, and recovery info. -# @RELATION CALLS -> [SupersetContextExtractor.parse_superset_link] +# @RELATION CALLS -> [SupersetContextParsingMixin.parse_superset_link] @router.post("/parse-url") async def parse_superset_url( payload: ParseUrlRequest, diff --git a/backend/src/core/auth/logger.py b/backend/src/core/auth/logger.py index c193cbf8..aa725c51 100644 --- a/backend/src/core/auth/logger.py +++ b/backend/src/core/auth/logger.py @@ -12,6 +12,14 @@ # @POST Audit logging functions exported # @SIDE_EFFECT Writes auth audit log entries # @DATA_CONTRACT AuthEvent -> LogEntry +# @RATIONALE Structured logging with automatic sensitive field masking prevents accidental +# credential leakage in logs. Each auth event (LOGIN, LOGOUT, LOGIN_BLOCKED, etc.) +# is a typed, structured record rather than freeform text — enabling audit queries +# and alerting on specific event types without log parsing heuristics. +# @REJECTED Freeform text logging was rejected — it is impossible to reliably query for failed +# logins or blocked tokens without regex parsing that breaks on log format changes. +# Logging raw user input into the details dict without masking was rejected — passwords +# or tokens could appear in error messages and leak to log storage. from datetime import UTC, datetime import re diff --git a/backend/src/core/auth/repository.py b/backend/src/core/auth/repository.py index a7e0b78a..996d58b3 100644 --- a/backend/src/core/auth/repository.py +++ b/backend/src/core/auth/repository.py @@ -10,6 +10,13 @@ # @PRE Database connection is active. # @POST Provides valid access to identity data. # @SIDE_EFFECT Executes database read queries through the injected SQLAlchemy session boundary. +# @RATIONALE Repository pattern chosen over direct ORM access in service layer — isolates all +# SQLAlchemy query logic from auth services, making queries testable against a mock +# session and allowing eager-loading (selectinload) optimizations in one place. +# @REJECTED Scattered ORM queries across route handlers was rejected — eager-loading decisions +# would leak into API endpoints, and changing the ORM relationship loading strategy +# would require touching multiple files. Using raw SQL for auth queries was rejected — +# SQLAlchemy ORM provides relationship loading and identity map that raw SQL lacks. from sqlalchemy.orm import Session, selectinload diff --git a/backend/src/core/config_manager.py b/backend/src/core/config_manager.py index c74ab0a6..4e2e6ced 100644 --- a/backend/src/core/config_manager.py +++ b/backend/src/core/config_manager.py @@ -15,6 +15,14 @@ # @RELATION DEPENDS_ON -> [EncryptionManager] # @RELATION CALLS -> [logger] # @RELATION CALLS -> [configure_logger] +# @RATIONALE DB-backed ConfigManager chosen over JSON-file-only configuration because environment +# passwords (Superset DB, SSH keys) need encryption at rest via EncryptionManager. The +# one-time migration from legacy config.json ensures backward compatibility without manual +# reconfiguration. Lazy-loading with in-memory cache avoids DB reads on every get_config() call. +# @REJECTED Keeping all config in config.json was rejected — passwords would be stored in plaintext +# and the file is not synchronized across multiple backend instances. Reading from DB on +# every access was rejected — adds ~5ms latency per config read for no benefit since config +# changes are rare. # import json import os diff --git a/backend/src/core/cot_logger.py b/backend/src/core/cot_logger.py index 5a4854b3..89489345 100644 --- a/backend/src/core/cot_logger.py +++ b/backend/src/core/cot_logger.py @@ -10,6 +10,15 @@ # @POST JSON log records written to the 'cot' logger at appropriate levels. # @SIDE_EFFECT Writes structured JSON to the 'cot' Python logger. # @DATA_CONTRACT Log call -> Single-line JSON to logging.StreamHandler/file. +# @RATIONALE ContextVar-based trace_id/span_id propagation chosen over thread-local storage +# because async/await execution can switch between threads at await points — thread-locals +# lose context. The Molecular CoT protocol (REASON/REFLECT/EXPLORE) is implemented as a +# structured JSON log format rather than in-memory spans to remain visible in production +# log streams without requiring a separate tracing backend. +# @REJECTED Using Python's logging.LoggerAdapter for trace propagation was rejected — it requires +# passing the adapter instance through every function call, which would break the existing +# public logger API. OpenTelemetry was rejected as too heavy — the project needs simple +# structured logging, not distributed tracing with span exporters and sampling. from contextvars import ContextVar import logging diff --git a/backend/src/core/migration/archive_parser.py b/backend/src/core/migration/archive_parser.py index 5fd5259a..20351a05 100644 --- a/backend/src/core/migration/archive_parser.py +++ b/backend/src/core/migration/archive_parser.py @@ -8,6 +8,14 @@ # @POST Parsed migration archive returned # @SIDE_EFFECT Reads archive file (read-only) # @DATA_CONTRACT ArchivePath -> ParsedMigration +# @RATIONALE Superset exports are ZIP archives containing JSON/YAML metadata plus embedded +# dashboard and chart definitions. ArchiveParser extracts these into a normalized +# object catalog (dashboards, datasets, charts) that the diff engine can compare +# without re-importing. Read-only parsing guarantees the archive is never mutated. +# @REJECTED Using Superset's own import API to read exports was rejected — the API modifies +# target state; parsing the ZIP directly provides a pure inspection path. Relying +# on Superset's /api/v1/export endpoint format without extraction was rejected — +# the raw export includes instance-specific IDs that must be resolved before diffing. import json from pathlib import Path import tempfile diff --git a/backend/src/core/ssl.py b/backend/src/core/ssl.py index 206fa70e..77ac9639 100644 --- a/backend/src/core/ssl.py +++ b/backend/src/core/ssl.py @@ -32,7 +32,13 @@ def system_ssl_context() -> ssl.SSLContext: CA certificates in flat bundle files (ADR-0009 Finding 7). Falls back to default context if capath directory is not available (rare edge case, e.g. minimal containers without ca-certificates). + + Respects the SSL_CERT_DIR environment variable for non-root environments + (development, test, containers without update-ca-certificates root access). """ + ssl_cert_dir = os.environ.get("SSL_CERT_DIR") + if ssl_cert_dir and os.path.isdir(ssl_cert_dir): + return ssl.create_default_context(capath=ssl_cert_dir) if os.path.isdir(DEFAULT_CAPATH): return ssl.create_default_context(capath=DEFAULT_CAPATH) return ssl.create_default_context() diff --git a/backend/src/models/auth.py b/backend/src/models/auth.py index 226c6278..f52e1e75 100644 --- a/backend/src/models/auth.py +++ b/backend/src/models/auth.py @@ -9,6 +9,14 @@ # @POST Auth ORM models registered with unique constraints # @SIDE_EFFECT Defines auth user tables # @DATA_CONTRACT UserData -> UserRecord +# @RATIONALE SQLAlchemy ORM models used over a document store because auth relationships (user→roles→permissions) +# require referential integrity and cascade deletes — a document DB would force application-level +# consistency for the user-role-permission graph. Separate auth.db from main app DB isolates auth +# schema from Superset metadata migrations. +# @REJECTED Using Superset's built-in Flask-AppBuilder tables was rejected — the Flask-AppBuilder schema +# is tightly coupled to Flask-Login sessions and cannot support API-key auth, ADFS OIDC, or the +# custom Role→Permission→Resource RBAC model without fragile monkey-patching. A single shared DB +# for auth and app data was rejected — auth schema must survive Superset metadata upgrades. from datetime import UTC, datetime import uuid diff --git a/backend/src/models/mapping.py b/backend/src/models/mapping.py index 7e16e1ec..debc18e2 100644 --- a/backend/src/models/mapping.py +++ b/backend/src/models/mapping.py @@ -12,6 +12,13 @@ # @POST Mapping ORM models registered with UUID primary keys # @SIDE_EFFECT Defines environment/database/resource mapping tables # @DATA_CONTRACT MappingData -> MappingRecord +# @RATIONALE SQLAlchemy models for environment metadata and database mappings provide referential +# integrity between source→target database pairs. The UUID primary key convention +# avoids integer ID collisions during dashboard migration between Superset instances. +# @REJECTED Using Superset's native database IDs as mapping keys was rejected — database IDs +# differ between source and target Superset instances, making direct ID mapping impossible. +# A document store (no schema) was rejected — mapping integrity (every target maps to +# exactly one source) must be enforced at the database level. import enum import uuid diff --git a/backend/src/models/profile.py b/backend/src/models/profile.py index 62808663..0c80ef14 100644 --- a/backend/src/models/profile.py +++ b/backend/src/models/profile.py @@ -12,6 +12,14 @@ # @POST Profile ORM models registered # @SIDE_EFFECT Defines user profile/preference tables # @DATA_CONTRACT ProfileData -> PreferenceRecord +# @RATIONALE Separate profile table from auth users keeps user identity (credentials, roles) +# separate from user preferences (UI state, Git tokens) — auth db is critical path +# for every request, while profile data is read only on dashboard load. Encrypted +# Git token at rest prevents credential leakage through DB backup exposure. +# @REJECTED Storing profile data in the auth user table was rejected — it would bloat the +# auth schema with nullable UI preference columns and increase auth query payload +# for every authenticated request. Storing Git tokens in plaintext was rejected — +# DB backups could expose VCS credentials. from datetime import UTC, datetime import uuid diff --git a/backend/src/plugins/migration.py b/backend/src/plugins/migration.py index 6478992b..6fb334ae 100755 --- a/backend/src/plugins/migration.py +++ b/backend/src/plugins/migration.py @@ -13,6 +13,15 @@ # @SIDE_EFFECT Reads config, opens database sessions, creates temporary artifacts, and triggers Superset export/import workflows. # @DATA_CONTRACT Input[TaskContext{from_env,to_env,dashboard_regex,replace_db_config,from_db_id,to_db_id,passwords?}] -> Output[MigrationResult|artifact set] # @INVARIANT Dashboards must never be imported with unmapped/source DB connections to prevent data leaks or cross-environment pollution. +# @RATIONALE MigrationPlugin implements the PluginBase interface for TaskManager — it encapsulates +# the full export→map→import lifecycle as a single plugin.execute() call. The plugin +# reads source dashboards via SupersetClient, applies DB-ID mapping via IdMappingService, +# and imports to target. This keeps migration logic isolated from task orchestration. +# @REJECTED Implementing migration as inline code in the TaskManager plugin dispatch was rejected +# — migration has complex error handling (partial import recovery, rollback) that doesn't +# fit the generic plugin pattern. Exposing SupersetClient directly to tasks (without +# the plugin wrapper) was rejected — plugin isolation provides cancellation, progress +# reporting, and logging that raw client calls lack. import re from typing import Any diff --git a/backend/src/schemas/auth.py b/backend/src/schemas/auth.py index d0115d49..77bc74bf 100644 --- a/backend/src/schemas/auth.py +++ b/backend/src/schemas/auth.py @@ -7,6 +7,14 @@ # # @INVARIANT Sensitive fields like password must not be included in response schemas. # @DATA_CONTRACT AuthPayload -> AuthSchema +# @RATIONALE Pydantic v2 BaseModel chosen over dataclasses or TypedDict — Pydantic provides +# built-in field validation (EmailStr, field_validator), JSON serialization, and OpenAPI +# schema generation for FastAPI docs. Separate request/response schemas prevent accidental +# password leakage through response serialization. +# @REJECTED Using SQLAlchemy models directly as API schemas was rejected — ORM models carry +# lazy-loaded relationships and database session state that cannot be safely serialized. +# A single User schema for both request and response was rejected — password_hash must +# never appear in API responses, and separate request schemas allow per-endpoint validation. from datetime import datetime import re diff --git a/backend/src/services/auth_service.py b/backend/src/services/auth_service.py index 6b936b96..2eabc808 100644 --- a/backend/src/services/auth_service.py +++ b/backend/src/services/auth_service.py @@ -12,6 +12,13 @@ # @POST User identity verified and session tokens issued according to role scopes. # @SIDE_EFFECT Writes last login timestamps and JIT-provisions external users. # @DATA_CONTRACT [Credentials | ADFSClaims] -> [UserEntity | SessionToken] +# @RATIONALE Thin service layer between API routes and AuthRepository — encapsulates credential +# verification (password hashing), ADFS JIT provisioning, and session creation into +# a single orchestration class. This keeps route handlers free of auth business logic. +# @REJECTED Putting credential verification logic directly in API route handlers was rejected — +# it would be duplicated across local login, ADFS callback, and API-key auth paths. +# Authentication was kept separate from authorization (permission checks in middleware) +# to allow independent testing of each concern. from datetime import UTC, datetime from typing import Any diff --git a/backend/src/services/clean_release/artifact_catalog_loader.py b/backend/src/services/clean_release/artifact_catalog_loader.py index 769f9868..023a1c36 100644 --- a/backend/src/services/clean_release/artifact_catalog_loader.py +++ b/backend/src/services/clean_release/artifact_catalog_loader.py @@ -6,6 +6,14 @@ # @INVARIANT Artifact catalog must produce deterministic CandidateArtifact entries with required identity and checksum fields. # @SIDE_EFFECT Reads JSON file from filesystem # @DATA_CONTRACT FilePath -> CandidateArtifact[] +# @RATIONALE ArtifactCatalogLoader provides a standalone entry point for loading release +# candidate manifests from JSON files — used by the TUI and API to bootstrap +# clean-release flows without requiring an already-configured release pipeline. +# The JSON artifact catalog is human-editable for manual overrides. +# @REJECTED Embedding artifact loading in the compliance orchestrator was rejected — the +# loader must work independently for TUI-based workflows where no orchestrator +# exists. A DB-backed artifact store was rejected for the bootstrap flow — +# the release process starts before the database may be fully configured. from __future__ import annotations diff --git a/backend/src/services/clean_release/policy_engine.py b/backend/src/services/clean_release/policy_engine.py index 50b3c7c9..ae760327 100644 --- a/backend/src/services/clean_release/policy_engine.py +++ b/backend/src/services/clean_release/policy_engine.py @@ -9,6 +9,15 @@ # @PRE PolicyRepository is accessible # @POST PolicyDecision returned with approval status # @SIDE_EFFECT Read-only policy evaluation; no state changes +# @RATIONALE Policy evaluation is a pure function from Candidate -> PolicyDecision, enabling +# deterministic unit tests without database mocking. The "enterprise-clean" policy +# rejects dashboards referencing non-registry datasources — this prevents deploying +# dashboards that depend on ad-hoc tables or views that won't exist in production. +# @REJECTED Executing policy checks inside the deployment pipeline was rejected — policy failures +# should be surfaced during release preparation, not after deployment begins. An +# allow-list approach (listing permitted sources) was rejected as too brittle — the +# registry-based approach (any source not in the registry is a violation) automatically +# catches new unregistered sources without policy updates. from __future__ import annotations diff --git a/backend/src/services/clean_release/policy_resolution_service.py b/backend/src/services/clean_release/policy_resolution_service.py index 6e6272d4..5643ff82 100644 --- a/backend/src/services/clean_release/policy_resolution_service.py +++ b/backend/src/services/clean_release/policy_resolution_service.py @@ -10,6 +10,14 @@ # @PRE PolicyRepository and Manifest are available # @POST ResolutionResult with matched policies # @SIDE_EFFECT Read-only policy evaluation; logs resolution decisions +# @RATIONALE PolicyResolutionService decouples snapshot resolution from policy evaluation — it +# resolves the active policy and registry snapshots from ConfigManager's environment +# configuration, then passes them to PolicyEngine for evaluation. This separation +# allows testing resolution logic without evaluating policies, and vice versa. +# @REJECTED Merging resolution and evaluation into one service was rejected — resolution depends +# on ConfigManager (which environment is target), while evaluation is a pure function +# over snapshots. Combining them would require mocking ConfigManager even for unit +# testing simple policy rules. from __future__ import annotations diff --git a/backend/src/services/clean_release/stages/base.py b/backend/src/services/clean_release/stages/base.py index 93494781..d8bd6441 100644 --- a/backend/src/services/clean_release/stages/base.py +++ b/backend/src/services/clean_release/stages/base.py @@ -7,6 +7,15 @@ # @INVARIANT Stage execution is deterministic for equal input context. # @SIDE_EFFECT None (deterministic execution) # @DATA_CONTRACT Context -> StageResult +# @RATIONALE Pluggable stage architecture isolates each compliance check (manifest integrity, +# internal-only sources, data purity, etc.) into a self-contained Protocol-based stage. +# Each stage receives a shared ComplianceContext and produces a StageResult — the +# orchestrator aggregates results without knowing stage internals. This allows adding +# new compliance checks without modifying the execution pipeline. +# @REJECTED Monolithic compliance check was rejected — a single function checking all rules would +# exceed INV_7 (400 lines) and require coordinated changes for every new policy. Using +# inheritance (base class with abstract methods) was rejected — Protocol-based structural +# subtyping is more testable with simple dataclass contracts instead of class hierarchies. from __future__ import annotations diff --git a/backend/src/services/reports/report_service.py b/backend/src/services/reports/report_service.py index 0f8b3acf..e3926ca5 100644 --- a/backend/src/services/reports/report_service.py +++ b/backend/src/services/reports/report_service.py @@ -14,6 +14,12 @@ # @PRE session is active and valid # @POST Returns Report with generated summary # @SIDE_EFFECT Read-only database operations; logs report generation +# @RATIONALE Module-level contract aggregates task report normalization and query services +# into a single namespace consumed by route handlers. Normalization logic is extracted +# as a pure function (normalize_task_report) to enable unit testing without DB state. +# @REJECTED Splitting normalization and query into separate modules was rejected — they always +# change together (when a new task type is added, both the normalizer and query filter +# must be updated), making separation artificial overhead. from datetime import UTC, datetime @@ -81,6 +87,14 @@ def _filter_tasks_by_rbac(tasks: list[Task], current_user) -> list[Task]: # @SIDE_EFFECT Reads task history and optional clean-release repository state without mutating source records. # @DATA_CONTRACT Input[TaskManager, Optional[CleanReleaseRepository], ReportQuery|report_id] -> Output[ReportCollection|ReportDetailView|None] # @INVARIANT Service methods are read-only over task history source. +# @RATIONALE ReportService provides a read-only query layer over TaskManager + CleanReleaseRepository +# — separate from the task execution pipeline — so that report UIs don't need access to +# task creation or lifecycle APIs. Service methods normalize task records into a consistent +# ReportRow shape regardless of the originating plugin (migration, translate, etc.). +# @REJECTED Embedding report queries in TaskManager was rejected — it would mix read-only analytics +# with task execution concerns and expose execution internals to report consumers. A +# dedicated reporting database was rejected — the operational task store is the SSOT and +# syncing to a reporting DB would add latency and consistency risk. # # @TEST_CONTRACT ReportsServiceModel -> # { diff --git a/build.sh b/build.sh index 6104fa72..3cef134b 100755 --- a/build.sh +++ b/build.sh @@ -1,19 +1,32 @@ #!/usr/bin/env bash # #region build [C:2] [TYPE Module] -# @PURPOSE: Unified build script — local docker compose + release bundles (backend + frontend + agent) + lightweight bundle +# @PURPOSE: Unified build script — local docker compose + individual image builds + release bundles # @COMPLEXITY: 2 # # Usage: ./build.sh [options] # -# Commands: +# Commands (local docker): # up [profile] Build and start local docker compose (default: current) # down [profile] Stop services # restart [profile] Rebuild and restart services # logs [profile] Tail logs # status Show running containers for all profiles +# +# Commands (individual image build — no export): +# build:backend Build backend image only (superset-tools-backend:) +# build:frontend Build frontend image only (superset-tools-frontend:) +# build:agent Build agent image only (superset-tools-agent:) +# +# Commands (single-image bundle — build + .tar.xz export): +# bundle:backend Build + export backend .tar.xz only +# bundle:frontend Build + export frontend .tar.xz only +# bundle:agent Build + export agent .tar.xz only +# +# Commands (full bundles — all three images): # bundle Slim enterprise bundle (no embeddings agent). Default. # bundle:embeddings Enterprise bundle WITH semantic embedding routing (larger agent) # bundle:light Lightweight all-in-one bundle (<200 MB, .tar.xz, no Playwright) +# # help Show this help # # Profiles: current (default), master, enterprise-clean @@ -141,6 +154,67 @@ get_repo_digest() { fi } +resolve_app_version() { + local tag="$1" + local git_commit + git_commit="$(git rev-parse --short HEAD 2>/dev/null || echo 'unknown')" + echo "${tag}+${git_commit}" +} + +# ====================================================================== +# SINGLE-IMAGE BUILD FUNCTIONS +# ====================================================================== + +build_backend() { + local tag="$1" + local image="superset-tools-backend:${tag}" + echo "[build] Building backend image ${image}..." + docker build -f docker/backend.Dockerfile -t "${image}" . + echo "[build] ✅ Backend image ${image} built" +} + +build_frontend() { + local tag="$1" + local image="superset-tools-frontend:${tag}" + local app_version + app_version="$(resolve_app_version "${tag}")" + echo "[build] Building frontend image ${image} (APP_VERSION=${app_version})..." + docker build --build-arg "APP_VERSION=${app_version}" -f docker/frontend.Dockerfile -t "${image}" . + echo "[build] ✅ Frontend image ${image} built" +} + +build_agent() { + local tag="$1" + local image="superset-tools-agent:${tag}" + echo "[build] Building agent image ${image}..." + docker build -f docker/Dockerfile.agent -t "${image}" . + echo "[build] ✅ Agent image ${image} built" +} + +build_agent_embeddings() { + local tag="$1" + local image="superset-tools-agent:${tag}" + echo "[build] Building agent image ${image} (WITH_EMBEDDINGS=true)..." + docker build \ + --build-arg WITH_EMBEDDINGS=true \ + -f docker/Dockerfile.agent \ + -t "${image}" . + echo "[build] ✅ Agent image ${image} built (embeddings)" +} + +# ====================================================================== +# SINGLE-IMAGE EXPORT +# ====================================================================== + +export_image() { + local image="$1" + local archive="$2" + echo "[bundle] Compressing ${image} -> ${archive}..." + mkdir -p "$(dirname "${archive}")" + docker save "${image}" | xz -T0 -3 > "${archive}" + echo "[bundle] ✅ ${archive} saved" +} + # ====================================================================== # COMPOSE COMMANDS # ====================================================================== @@ -206,58 +280,64 @@ compose_status() { # BUNDLE: full release (backend + frontend + agent .tar.xz's) # ====================================================================== -bundle_release() { +# ====================================================================== +# SINGLE-IMAGE BUNDLE FUNCTIONS +# ====================================================================== + +bundle_single_backend() { if [[ $# -lt 1 ]]; then echo "Error: tag is required." - echo "Usage: ./build.sh bundle (e.g. v1.0.0)" + echo "Usage: ./build.sh bundle:backend " exit 1 fi local tag="$1" - local backend_tag="superset-tools-backend:${tag}" - local frontend_tag="superset-tools-frontend:${tag}" - local agent_tag="superset-tools-agent:${tag}" - - # Git commit hash for APP_VERSION (e.g. "0.5.2+58d06fb2") - local git_commit - git_commit="$(git rev-parse --short HEAD 2>/dev/null || echo 'unknown')" - local app_version="${tag}+${git_commit}" - + local image="superset-tools-backend:${tag}" mkdir -p "$DIST_ROOT" + build_backend "${tag}" + export_image "${image}" "${DIST_ROOT}/superset-tools-backend.${tag}.tar.xz" + (cd "${DIST_ROOT}" && sha256sum "superset-tools-backend.${tag}.tar.xz" > "sha256sums-backend.${tag}.txt") + echo "[bundle:backend] ✅ Backend bundle ${tag} created" + echo " xz -dc ${DIST_ROOT}/superset-tools-backend.${tag}.tar.xz | docker load" +} - echo "[bundle] Building backend image ${backend_tag}..." - docker build -f docker/backend.Dockerfile -t "${backend_tag}" . +bundle_single_frontend() { + if [[ $# -lt 1 ]]; then + echo "Error: tag is required." + echo "Usage: ./build.sh bundle:frontend " + exit 1 + fi + local tag="$1" + local image="superset-tools-frontend:${tag}" + mkdir -p "$DIST_ROOT" + build_frontend "${tag}" + export_image "${image}" "${DIST_ROOT}/superset-tools-frontend.${tag}.tar.xz" + (cd "${DIST_ROOT}" && sha256sum "superset-tools-frontend.${tag}.tar.xz" > "sha256sums-frontend.${tag}.txt") + echo "[bundle:frontend] ✅ Frontend bundle ${tag} created" + echo " xz -dc ${DIST_ROOT}/superset-tools-frontend.${tag}.tar.xz | docker load" +} - echo "[bundle] Building frontend image ${frontend_tag}..." - docker build --build-arg "APP_VERSION=${app_version}" -f docker/frontend.Dockerfile -t "${frontend_tag}" . +bundle_single_agent() { + if [[ $# -lt 1 ]]; then + echo "Error: tag is required." + echo "Usage: ./build.sh bundle:agent " + exit 1 + fi + local tag="$1" + local image="superset-tools-agent:${tag}" + mkdir -p "$DIST_ROOT" + build_agent "${tag}" + export_image "${image}" "${DIST_ROOT}/superset-tools-agent.${tag}.tar.xz" + (cd "${DIST_ROOT}" && sha256sum "superset-tools-agent.${tag}.tar.xz" > "sha256sums-agent.${tag}.txt") + echo "[bundle:agent] ✅ Agent bundle ${tag} created" + echo " xz -dc ${DIST_ROOT}/superset-tools-agent.${tag}.tar.xz | docker load" +} - echo "[bundle] Building agent image ${agent_tag}..." - docker build -f docker/Dockerfile.agent -t "${agent_tag}" . +# ====================================================================== +# FULL BUNDLE: backend + frontend + agent +# ====================================================================== - echo "[bundle] Exporting .tar.xz archives..." - echo "[bundle] Compressing backend image..." - docker save "${backend_tag}" | xz -T0 -3 > "${DIST_ROOT}/superset-tools-backend.${tag}.tar.xz" - echo "[bundle] Compressing frontend image..." - docker save "${frontend_tag}" | xz -T0 -3 > "${DIST_ROOT}/superset-tools-frontend.${tag}.tar.xz" - echo "[bundle] Compressing agent image..." - docker save "${agent_tag}" | xz -T0 -3 > "${DIST_ROOT}/superset-tools-agent.${tag}.tar.xz" - - echo "[bundle] Calculating checksums..." - ( - cd "${DIST_ROOT}" - sha256sum "superset-tools-backend.${tag}.tar.xz" "superset-tools-frontend.${tag}.tar.xz" "superset-tools-agent.${tag}.tar.xz" > "sha256sums.${tag}.txt" - ) - - local backend_id frontend_id agent_id - local backend_digest frontend_digest agent_digest - backend_id="$(get_image_id "${backend_tag}")" - frontend_id="$(get_image_id "${frontend_tag}")" - agent_id="$(get_image_id "${agent_tag}")" - backend_digest="$(get_repo_digest "${backend_tag}")" - frontend_digest="$(get_repo_digest "${frontend_tag}")" - agent_digest="$(get_repo_digest "${agent_tag}")" - - # Генерируем deploy-compose для загрузки pre-built образов (image: + pull_policy: never) - # Без postgres — используется внешний корпоративный PostgreSQL. +generate_deploy_compose() { + local tag="$1" backend_tag="$2" frontend_tag="$3" agent_tag="$4" cat > "${DIST_ROOT}/docker-compose.enterprise-clean.yml" < "${DIST_ROOT}/manifest.${tag}.txt" < "${DIST_ROOT}/manifest.${tag}${suffix}.txt" < "${DIST_ROOT}/manifest.${tag}.json" < "${DIST_ROOT}/manifest.${tag}${suffix}.json" < (e.g. v1.0.0)" + exit 1 + fi + local tag="$1" + local backend_tag="superset-tools-backend:${tag}" + local frontend_tag="superset-tools-frontend:${tag}" + local agent_tag="superset-tools-agent:${tag}" + + mkdir -p "$DIST_ROOT" + + # Build all three images + build_backend "${tag}" + build_frontend "${tag}" + build_agent "${tag}" + + # Export .tar.xz archives + echo "[bundle] Exporting .tar.xz archives..." + export_image "${backend_tag}" "${DIST_ROOT}/superset-tools-backend.${tag}.tar.xz" + export_image "${frontend_tag}" "${DIST_ROOT}/superset-tools-frontend.${tag}.tar.xz" + export_image "${agent_tag}" "${DIST_ROOT}/superset-tools-agent.${tag}.tar.xz" + + # Checksums + echo "[bundle] Calculating checksums..." + (cd "${DIST_ROOT}" && sha256sum \ + "superset-tools-backend.${tag}.tar.xz" \ + "superset-tools-frontend.${tag}.tar.xz" \ + "superset-tools-agent.${tag}.tar.xz" > "sha256sums.${tag}.txt") + + # Generate deploy compose + manifest + generate_deploy_compose "${tag}" "${backend_tag}" "${frontend_tag}" "${agent_tag}" + generate_manifest "${tag}" "${backend_tag}" "${frontend_tag}" "${agent_tag}" cp "${SCRIPT_DIR}/.env.enterprise-clean.example" "${DIST_ROOT}/.env.enterprise-clean.example" @@ -526,185 +652,47 @@ bundle_embeddings() { exit 1 fi local tag="$1" - local backend_tag="superset-tools-backend:${tag}-embeddings" - local frontend_tag="superset-tools-frontend:${tag}-embeddings" - local agent_tag="superset-tools-agent:${tag}-embeddings" - - # Git commit hash for APP_VERSION (e.g. "0.5.2+58d06fb2") - local git_commit - git_commit="$(git rev-parse --short HEAD 2>/dev/null || echo 'unknown')" - local app_version="${tag}+${git_commit}" + local suffix="-embeddings" + local backend_tag="superset-tools-backend:${tag}${suffix}" + local frontend_tag="superset-tools-frontend:${tag}${suffix}" + local agent_tag="superset-tools-agent:${tag}${suffix}" mkdir -p "$DIST_ROOT" - echo "[bundle:embeddings] Building backend image ${backend_tag}..." - docker build -f docker/backend.Dockerfile -t "${backend_tag}" . - - echo "[bundle:embeddings] Building frontend image ${frontend_tag}..." - docker build --build-arg "APP_VERSION=${app_version}" -f docker/frontend.Dockerfile -t "${frontend_tag}" . - - echo "[bundle:embeddings] Building agent image ${agent_tag} (WITH_EMBEDDINGS=true)..." - docker build \ - --build-arg WITH_EMBEDDINGS=true \ - -f docker/Dockerfile.agent \ - -t "${agent_tag}" . + # Build all three images + build_backend "${tag}${suffix}" + build_frontend "${tag}${suffix}" + build_agent_embeddings "${tag}${suffix}" + # Export .tar.xz archives echo "[bundle:embeddings] Exporting .tar.xz archives..." - echo "[bundle:embeddings] Compressing backend image..." - docker save "${backend_tag}" | xz -T0 -3 > "${DIST_ROOT}/superset-tools-backend.${tag}-embeddings.tar.xz" - echo "[bundle:embeddings] Compressing frontend image..." - docker save "${frontend_tag}" | xz -T0 -3 > "${DIST_ROOT}/superset-tools-frontend.${tag}-embeddings.tar.xz" - echo "[bundle:embeddings] Compressing agent image..." - docker save "${agent_tag}" | xz -T0 -3 > "${DIST_ROOT}/superset-tools-agent.${tag}-embeddings.tar.xz" + export_image "${backend_tag}" "${DIST_ROOT}/superset-tools-backend.${tag}${suffix}.tar.xz" + export_image "${frontend_tag}" "${DIST_ROOT}/superset-tools-frontend.${tag}${suffix}.tar.xz" + export_image "${agent_tag}" "${DIST_ROOT}/superset-tools-agent.${tag}${suffix}.tar.xz" + # Checksums echo "[bundle:embeddings] Calculating checksums..." - ( - cd "${DIST_ROOT}" - sha256sum \ - "superset-tools-backend.${tag}-embeddings.tar.xz" \ - "superset-tools-frontend.${tag}-embeddings.tar.xz" \ - "superset-tools-agent.${tag}-embeddings.tar.xz" > "sha256sums.${tag}-embeddings.txt" - ) + (cd "${DIST_ROOT}" && sha256sum \ + "superset-tools-backend.${tag}${suffix}.tar.xz" \ + "superset-tools-frontend.${tag}${suffix}.tar.xz" \ + "superset-tools-agent.${tag}${suffix}.tar.xz" > "sha256sums.${tag}${suffix}.txt") - local backend_id frontend_id agent_id - local backend_digest frontend_digest agent_digest - backend_id="$(get_image_id "${backend_tag}")" - frontend_id="$(get_image_id "${frontend_tag}")" - agent_id="$(get_image_id "${agent_tag}")" - backend_digest="$(get_repo_digest "${backend_tag}")" - frontend_digest="$(get_repo_digest "${frontend_tag}")" - agent_digest="$(get_repo_digest "${agent_tag}")" + # Generate deploy compose + manifest + generate_deploy_compose "${tag}${suffix}" "${backend_tag}" "${frontend_tag}" "${agent_tag}" + generate_manifest "${tag}" "${backend_tag}" "${frontend_tag}" "${agent_tag}" "${suffix}" - # Generate deploy-compose — same as enterprise-clean, with embeddings image tags - cat > "${DIST_ROOT}/docker-compose.enterprise-clean.yml" < "${DIST_ROOT}/manifest.${tag}-embeddings.txt" <> "${DIST_ROOT}/manifest.${tag}${suffix}.txt" < "${DIST_ROOT}/manifest.${tag}-embeddings.json" < .env.enterprise-clean, then:" echo " docker compose -f ${DIST_ROOT}/docker-compose.enterprise-clean.yml up" echo "" @@ -727,20 +715,30 @@ Commands for local docker: logs [profile] Tail logs from all services status Show running containers for all profiles -Commands for release bundles: +Commands for individual image build (docker build only, no export): + build:backend Build backend image (superset-tools-backend:) + build:frontend Build frontend image (superset-tools-frontend:) + build:agent Build agent image (superset-tools-agent:) + +Commands for single-image bundle (build + .tar.xz export): + bundle:backend Build + export backend .tar.xz only + bundle:frontend Build + export frontend .tar.xz only + bundle:agent Build + export agent .tar.xz only + +Commands for full bundles (all three images): bundle Default slim bundle (backend + frontend + agent .tar.xz's). - Agent: NO sentence-transformers/embeddings (~500 MB saved). - Backend: NO agent/ML/dev deps. - NOTE: external PostgreSQL only — no bundled postgres image. - REQUIRED: AUTH_SECRET_KEY, ENCRYPTION_KEY, POSTGRES_PASSWORD, SERVICE_JWT. - Example: ./build.sh bundle v1.0.0 + Agent: NO sentence-transformers/embeddings (~500 MB saved). + Backend: NO agent/ML/dev deps. + NOTE: external PostgreSQL only — no bundled postgres image. + REQUIRED: AUTH_SECRET_KEY, ENCRYPTION_KEY, POSTGRES_PASSWORD, SERVICE_JWT. + Example: ./build.sh bundle v1.0.0 bundle:embeddings Enterprise bundle with semantic embedding routing. - Agent built WITH sentence-transformers+torch (larger image). - Same backend/frontend as default bundle. - REQUIRED: AUTH_SECRET_KEY, ENCRYPTION_KEY, POSTGRES_PASSWORD, SERVICE_JWT. - Example: ./build.sh bundle:embeddings v1.0.0 + Agent built WITH sentence-transformers+torch (larger image). + Same backend/frontend as default bundle. + REQUIRED: AUTH_SECRET_KEY, ENCRYPTION_KEY, POSTGRES_PASSWORD, SERVICE_JWT. + Example: ./build.sh bundle:embeddings v1.0.0 bundle:light Lightweight all-in-one image (.tar.xz, <200 MB, no Playwright) Example: ./build.sh bundle:light v1.0.0 @@ -760,6 +758,10 @@ Notes: - 'current' and 'master' use docker-compose.yml (dev, with local postgres). - 'enterprise-clean' uses docker-compose.enterprise-clean.yml (build from source, external postgres, corporate certs, optional nginx SSL). + - Individual build commands (build:backend|frontend|agent) produce local docker + images without touching dist/docker/. + - Single-image bundle commands (bundle:backend|frontend|agent) build + export + only one .tar.xz, useful for incremental deployments. HELP } @@ -772,7 +774,9 @@ main() { shift 2>/dev/null || true case "$CMD" in - up|down|restart|logs|bundle|bundle:embeddings|bundle:light|status|help|-h|--help) + up|down|restart|logs|status|help|-h|--help|bundle|bundle:embeddings|bundle:light|\ + build:backend|build:frontend|build:agent|\ + bundle:backend|bundle:frontend|bundle:agent) # Valid commands — proceed ;; *) @@ -784,15 +788,21 @@ main() { esac case "$CMD" in - up) compose_up "${1:-current}" ;; - down) compose_down "${1:-current}" ;; - restart) compose_restart "${1:-current}" ;; - logs) compose_logs "${1:-current}" ;; - bundle) bundle_release "$@" ;; - bundle:embeddings) bundle_embeddings "$@" ;; - bundle:light) bundle_light "$@" ;; - status) compose_status ;; - help|-h|--help) show_help ;; + up) compose_up "${1:-current}" ;; + down) compose_down "${1:-current}" ;; + restart) compose_restart "${1:-current}" ;; + logs) compose_logs "${1:-current}" ;; + bundle) bundle_release "$@" ;; + bundle:embeddings) bundle_embeddings "$@" ;; + bundle:light) bundle_light "$@" ;; + build:backend) build_backend "$@" ;; + build:frontend) build_frontend "$@" ;; + build:agent) build_agent "$@" ;; + bundle:backend) bundle_single_backend "$@" ;; + bundle:frontend) bundle_single_frontend "$@" ;; + bundle:agent) bundle_single_agent "$@" ;; + status) compose_status ;; + help|-h|--help) show_help ;; *) echo "Error: unknown command '$CMD'. See ./build.sh help" exit 1 diff --git a/docker/frontend.entrypoint.sh b/docker/frontend.entrypoint.sh index 00be62ef..7b5a4ce8 100644 --- a/docker/frontend.entrypoint.sh +++ b/docker/frontend.entrypoint.sh @@ -146,7 +146,7 @@ resolve_ssl_passphrase() { # ====================================================================== # decrypt_key_if_needed — расшифровка passphrase-защищённого ключа # ====================================================================== -# #region docker.frontend.entrypoint.decrypt_key [C:3] [TYPE Function] [SEMANTICS decryption,rsa,ssl,passphrase] +# #region docker.frontend.entrypoint.decrypt_key [C:3] [TYPE Function] [SEMANTICS decryption,pkey,ssl,passphrase] # @BRIEF Проверяет, зашифрован ли server.key, и если да — расшифровывает его. # @PRE ssl_dir — директория, куда скопирован server.key. ssl_passphrase — пароль (может быть пуст). # @POST Если ключ зашифрован и пароль верен — ключ расшифрован в ssl_dir. @@ -169,7 +169,8 @@ decrypt_key_if_needed() { [ -f "$ssl_key" ] || return 0 # Проверка: ключ зашифрован? - if openssl rsa -in "$ssl_key" -check -noout 2>/dev/null; then + # Используем openssl pkey (алгоритм-агностик — RSA, EC, Ed25519 и т.д.) + if openssl pkey -in "$ssl_key" -noout 2>/dev/null; then # Ключ незашифрован — ничего не делаем return 0 fi @@ -184,8 +185,8 @@ decrypt_key_if_needed() { echo "[entrypoint] 🔐 Приватный ключ зашифрован — расшифровываем..." - # Расшифровать через openssl rsa - if openssl rsa -in "$ssl_key" \ + # Расшифровать через openssl pkey (поддерживает RSA, EC, Ed25519 и т.д.) + if openssl pkey -in "$ssl_key" \ -passin "env:SSL_KEY_PASSPHRASE" \ -out "${ssl_key}.decrypted" 2>/dev/null; then mv "${ssl_key}.decrypted" "$ssl_key"