busya
f22b0cf41c
- Add @RELATION/@RATIONALE/@REJECTED headers to agent modules - Add GRACE contracts to backend services, models, schemas - Update build.sh with single-image build commands (build:backend|frontend|agent) - Update docker entrypoint: openssl rsa → openssl pkey (alg-agnostic) - Add @RATIONALE to backend core modules (cot_logger, config_manager, ssl)
814 lines
29 KiB
Bash
Executable File
814 lines
29 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# #region build [C:2] [TYPE Module]
|
|
# @PURPOSE: Unified build script — local docker compose + individual image builds + release bundles
|
|
# @COMPLEXITY: 2
|
|
#
|
|
# Usage: ./build.sh <command> [options]
|
|
#
|
|
# Commands (local docker):
|
|
# up [profile] Build and start local docker compose (default: current)
|
|
# down [profile] Stop services
|
|
# restart [profile] Rebuild and restart services
|
|
# logs [profile] Tail logs
|
|
# status Show running containers for all profiles
|
|
#
|
|
# Commands (individual image build — no export):
|
|
# build:backend <tag> Build backend image only (superset-tools-backend:<tag>)
|
|
# build:frontend <tag> Build frontend image only (superset-tools-frontend:<tag>)
|
|
# build:agent <tag> Build agent image only (superset-tools-agent:<tag>)
|
|
#
|
|
# Commands (single-image bundle — build + .tar.xz export):
|
|
# bundle:backend <tag> Build + export backend .tar.xz only
|
|
# bundle:frontend <tag> Build + export frontend .tar.xz only
|
|
# bundle:agent <tag> Build + export agent .tar.xz only
|
|
#
|
|
# Commands (full bundles — all three images):
|
|
# bundle <tag> Slim enterprise bundle (no embeddings agent). Default.
|
|
# bundle:embeddings <tag> Enterprise bundle WITH semantic embedding routing (larger agent)
|
|
# bundle:light <tag> Lightweight all-in-one bundle (<200 MB, .tar.xz, no Playwright)
|
|
#
|
|
# help Show this help
|
|
#
|
|
# Profiles: current (default), master, enterprise-clean
|
|
# #endregion build
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
cd "$SCRIPT_DIR"
|
|
|
|
BACKEND_ENV_FILE="$SCRIPT_DIR/backend/.env"
|
|
DIST_ROOT="$SCRIPT_DIR/dist/docker"
|
|
|
|
# ---- Docker Compose detection ----
|
|
if docker compose version >/dev/null 2>&1; then
|
|
COMPOSE_CMD=(docker compose)
|
|
elif command -v docker-compose >/dev/null 2>&1; then
|
|
COMPOSE_CMD=(docker-compose)
|
|
else
|
|
echo "Error: docker compose is not available."
|
|
exit 1
|
|
fi
|
|
|
|
# ======================================================================
|
|
# HELPERS
|
|
# ======================================================================
|
|
|
|
# shellcheck disable=SC2206
|
|
setup_compose_profile() {
|
|
local profile="$1"
|
|
case "$profile" in
|
|
master)
|
|
PROFILE_ENV_FILE="$SCRIPT_DIR/.env.master"
|
|
PROJECT_NAME="superset-tools-master"
|
|
COMPOSE_FILE="docker-compose.yml"
|
|
;;
|
|
current|"")
|
|
PROFILE_ENV_FILE="$SCRIPT_DIR/.env.current"
|
|
PROJECT_NAME="superset-tools-current"
|
|
COMPOSE_FILE="docker-compose.yml"
|
|
;;
|
|
enterprise-clean)
|
|
PROFILE_ENV_FILE="$SCRIPT_DIR/.env.enterprise-clean"
|
|
PROJECT_NAME="superset-tools-enterprise"
|
|
COMPOSE_FILE="docker-compose.enterprise-clean.yml"
|
|
;;
|
|
*)
|
|
echo "Error: unknown profile '$profile'. Use: current, master, enterprise-clean."
|
|
exit 1
|
|
;;
|
|
esac
|
|
COMPOSE_ARGS=(-p "$PROJECT_NAME" -f "$COMPOSE_FILE")
|
|
if [[ -f "$PROFILE_ENV_FILE" ]]; then
|
|
COMPOSE_ARGS+=(--env-file "$PROFILE_ENV_FILE")
|
|
fi
|
|
}
|
|
|
|
ensure_backend_encryption_key() {
|
|
if command -v python3 >/dev/null 2>&1; then
|
|
python3 - "$BACKEND_ENV_FILE" <<'PY'
|
|
import base64
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import List, Optional
|
|
|
|
|
|
def is_valid_fernet_key(raw_value: str) -> bool:
|
|
value = raw_value.strip()
|
|
if not value:
|
|
return False
|
|
try:
|
|
decoded = base64.urlsafe_b64decode(value.encode())
|
|
except Exception:
|
|
return False
|
|
return len(decoded) == 32
|
|
|
|
|
|
def generate_fernet_key() -> str:
|
|
return base64.urlsafe_b64encode(os.urandom(32)).decode()
|
|
|
|
|
|
env_path = Path(sys.argv[1])
|
|
env_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
existing_lines: List[str] = []
|
|
existing_key: Optional[str] = None
|
|
|
|
if env_path.exists():
|
|
existing_lines = env_path.read_text(encoding="utf-8").splitlines()
|
|
for line in existing_lines:
|
|
if line.startswith("ENCRYPTION_KEY="):
|
|
candidate = line.partition("=")[2].strip()
|
|
if is_valid_fernet_key(candidate):
|
|
existing_key = candidate
|
|
break
|
|
|
|
if existing_key is None:
|
|
generated_key = generate_fernet_key()
|
|
filtered_lines = [line for line in existing_lines if not line.startswith("ENCRYPTION_KEY=")]
|
|
filtered_lines.append(f"ENCRYPTION_KEY={generated_key}")
|
|
env_path.write_text("\n".join(filtered_lines) + "\n", encoding="utf-8")
|
|
print(f"[build] ENCRYPTION_KEY ensured in {env_path}")
|
|
else:
|
|
print(f"[build] Existing ENCRYPTION_KEY reused from {env_path}")
|
|
PY
|
|
return
|
|
fi
|
|
|
|
echo "Error: python3 is required to generate backend/.env with ENCRYPTION_KEY."
|
|
exit 1
|
|
}
|
|
|
|
get_image_id() {
|
|
docker image inspect --format '{{.Id}}' "$1"
|
|
}
|
|
|
|
get_repo_digest() {
|
|
local digest
|
|
digest="$(docker image inspect --format '{{join .RepoDigests ","}}' "$1" 2>/dev/null || true)"
|
|
if [[ -n "${digest}" && "${digest}" != "<no value>" ]]; then
|
|
printf '%s' "${digest}"
|
|
else
|
|
printf '%s' "unavailable"
|
|
fi
|
|
}
|
|
|
|
resolve_app_version() {
|
|
local tag="$1"
|
|
local git_commit
|
|
git_commit="$(git rev-parse --short HEAD 2>/dev/null || echo 'unknown')"
|
|
echo "${tag}+${git_commit}"
|
|
}
|
|
|
|
# ======================================================================
|
|
# SINGLE-IMAGE BUILD FUNCTIONS
|
|
# ======================================================================
|
|
|
|
build_backend() {
|
|
local tag="$1"
|
|
local image="superset-tools-backend:${tag}"
|
|
echo "[build] Building backend image ${image}..."
|
|
docker build -f docker/backend.Dockerfile -t "${image}" .
|
|
echo "[build] ✅ Backend image ${image} built"
|
|
}
|
|
|
|
build_frontend() {
|
|
local tag="$1"
|
|
local image="superset-tools-frontend:${tag}"
|
|
local app_version
|
|
app_version="$(resolve_app_version "${tag}")"
|
|
echo "[build] Building frontend image ${image} (APP_VERSION=${app_version})..."
|
|
docker build --build-arg "APP_VERSION=${app_version}" -f docker/frontend.Dockerfile -t "${image}" .
|
|
echo "[build] ✅ Frontend image ${image} built"
|
|
}
|
|
|
|
build_agent() {
|
|
local tag="$1"
|
|
local image="superset-tools-agent:${tag}"
|
|
echo "[build] Building agent image ${image}..."
|
|
docker build -f docker/Dockerfile.agent -t "${image}" .
|
|
echo "[build] ✅ Agent image ${image} built"
|
|
}
|
|
|
|
build_agent_embeddings() {
|
|
local tag="$1"
|
|
local image="superset-tools-agent:${tag}"
|
|
echo "[build] Building agent image ${image} (WITH_EMBEDDINGS=true)..."
|
|
docker build \
|
|
--build-arg WITH_EMBEDDINGS=true \
|
|
-f docker/Dockerfile.agent \
|
|
-t "${image}" .
|
|
echo "[build] ✅ Agent image ${image} built (embeddings)"
|
|
}
|
|
|
|
# ======================================================================
|
|
# SINGLE-IMAGE EXPORT
|
|
# ======================================================================
|
|
|
|
export_image() {
|
|
local image="$1"
|
|
local archive="$2"
|
|
echo "[bundle] Compressing ${image} -> ${archive}..."
|
|
mkdir -p "$(dirname "${archive}")"
|
|
docker save "${image}" | xz -T0 -3 > "${archive}"
|
|
echo "[bundle] ✅ ${archive} saved"
|
|
}
|
|
|
|
# ======================================================================
|
|
# COMPOSE COMMANDS
|
|
# ======================================================================
|
|
|
|
compose_up() {
|
|
local profile="${1:-current}"
|
|
setup_compose_profile "$profile"
|
|
ensure_backend_encryption_key
|
|
|
|
echo "[build] Profile: $profile (project: $PROJECT_NAME)"
|
|
if [[ -f "$PROFILE_ENV_FILE" ]]; then
|
|
echo "[build] Env file: $PROFILE_ENV_FILE"
|
|
fi
|
|
|
|
echo "[1/2] Building project images..."
|
|
"${COMPOSE_CMD[@]}" "${COMPOSE_ARGS[@]}" build
|
|
|
|
echo "[2/2] Starting Docker services..."
|
|
"${COMPOSE_CMD[@]}" "${COMPOSE_ARGS[@]}" up -d
|
|
|
|
echo "Done. Services are running."
|
|
echo " ${COMPOSE_CMD[*]} ${COMPOSE_ARGS[*]} ps"
|
|
echo " ${COMPOSE_CMD[*]} ${COMPOSE_ARGS[*]} logs -f"
|
|
}
|
|
|
|
compose_down() {
|
|
local profile="${1:-current}"
|
|
setup_compose_profile "$profile"
|
|
|
|
echo "[build] Stopping $profile (project: $PROJECT_NAME)..."
|
|
"${COMPOSE_CMD[@]}" "${COMPOSE_ARGS[@]}" down
|
|
echo "[build] Done."
|
|
}
|
|
|
|
compose_restart() {
|
|
local profile="${1:-current}"
|
|
setup_compose_profile "$profile"
|
|
ensure_backend_encryption_key
|
|
|
|
echo "[build] Restarting $profile (project: $PROJECT_NAME)..."
|
|
"${COMPOSE_CMD[@]}" "${COMPOSE_ARGS[@]}" build
|
|
"${COMPOSE_CMD[@]}" "${COMPOSE_ARGS[@]}" up -d --force-recreate
|
|
echo "[build] Done."
|
|
}
|
|
|
|
compose_logs() {
|
|
local profile="${1:-current}"
|
|
setup_compose_profile "$profile"
|
|
|
|
"${COMPOSE_CMD[@]}" "${COMPOSE_ARGS[@]}" logs -f
|
|
}
|
|
|
|
compose_status() {
|
|
echo "=== superset-tools docker status ==="
|
|
for profile in current master enterprise-clean; do
|
|
setup_compose_profile "$profile"
|
|
echo "--- $profile (project: $PROJECT_NAME) ---"
|
|
"${COMPOSE_CMD[@]}" "${COMPOSE_ARGS[@]}" ps 2>/dev/null || echo "(not running)"
|
|
done
|
|
}
|
|
|
|
# ======================================================================
|
|
# BUNDLE: full release (backend + frontend + agent .tar.xz's)
|
|
# ======================================================================
|
|
|
|
# ======================================================================
|
|
# SINGLE-IMAGE BUNDLE FUNCTIONS
|
|
# ======================================================================
|
|
|
|
bundle_single_backend() {
|
|
if [[ $# -lt 1 ]]; then
|
|
echo "Error: tag is required."
|
|
echo "Usage: ./build.sh bundle:backend <tag>"
|
|
exit 1
|
|
fi
|
|
local tag="$1"
|
|
local image="superset-tools-backend:${tag}"
|
|
mkdir -p "$DIST_ROOT"
|
|
build_backend "${tag}"
|
|
export_image "${image}" "${DIST_ROOT}/superset-tools-backend.${tag}.tar.xz"
|
|
(cd "${DIST_ROOT}" && sha256sum "superset-tools-backend.${tag}.tar.xz" > "sha256sums-backend.${tag}.txt")
|
|
echo "[bundle:backend] ✅ Backend bundle ${tag} created"
|
|
echo " xz -dc ${DIST_ROOT}/superset-tools-backend.${tag}.tar.xz | docker load"
|
|
}
|
|
|
|
bundle_single_frontend() {
|
|
if [[ $# -lt 1 ]]; then
|
|
echo "Error: tag is required."
|
|
echo "Usage: ./build.sh bundle:frontend <tag>"
|
|
exit 1
|
|
fi
|
|
local tag="$1"
|
|
local image="superset-tools-frontend:${tag}"
|
|
mkdir -p "$DIST_ROOT"
|
|
build_frontend "${tag}"
|
|
export_image "${image}" "${DIST_ROOT}/superset-tools-frontend.${tag}.tar.xz"
|
|
(cd "${DIST_ROOT}" && sha256sum "superset-tools-frontend.${tag}.tar.xz" > "sha256sums-frontend.${tag}.txt")
|
|
echo "[bundle:frontend] ✅ Frontend bundle ${tag} created"
|
|
echo " xz -dc ${DIST_ROOT}/superset-tools-frontend.${tag}.tar.xz | docker load"
|
|
}
|
|
|
|
bundle_single_agent() {
|
|
if [[ $# -lt 1 ]]; then
|
|
echo "Error: tag is required."
|
|
echo "Usage: ./build.sh bundle:agent <tag>"
|
|
exit 1
|
|
fi
|
|
local tag="$1"
|
|
local image="superset-tools-agent:${tag}"
|
|
mkdir -p "$DIST_ROOT"
|
|
build_agent "${tag}"
|
|
export_image "${image}" "${DIST_ROOT}/superset-tools-agent.${tag}.tar.xz"
|
|
(cd "${DIST_ROOT}" && sha256sum "superset-tools-agent.${tag}.tar.xz" > "sha256sums-agent.${tag}.txt")
|
|
echo "[bundle:agent] ✅ Agent bundle ${tag} created"
|
|
echo " xz -dc ${DIST_ROOT}/superset-tools-agent.${tag}.tar.xz | docker load"
|
|
}
|
|
|
|
# ======================================================================
|
|
# FULL BUNDLE: backend + frontend + agent
|
|
# ======================================================================
|
|
|
|
generate_deploy_compose() {
|
|
local tag="$1" backend_tag="$2" frontend_tag="$3" agent_tag="$4"
|
|
cat > "${DIST_ROOT}/docker-compose.enterprise-clean.yml" <<DEPLOY
|
|
services:
|
|
backend:
|
|
image: ${backend_tag}
|
|
pull_policy: never
|
|
restart: unless-stopped
|
|
environment:
|
|
DATABASE_URL: postgresql+psycopg2://\${POSTGRES_USER:-postgres}:\${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}@\${POSTGRES_HOST:?Set POSTGRES_HOST}:\${POSTGRES_PORT:-5432}/\${POSTGRES_DB:-ss_tools}
|
|
TASKS_DATABASE_URL: postgresql+psycopg2://\${POSTGRES_USER:-postgres}:\${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}@\${POSTGRES_HOST:?Set POSTGRES_HOST}:\${POSTGRES_PORT:-5432}/\${POSTGRES_DB:-ss_tools}
|
|
AUTH_DATABASE_URL: postgresql+psycopg2://\${POSTGRES_USER:-postgres}:\${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}@\${POSTGRES_HOST:?Set POSTGRES_HOST}:\${POSTGRES_PORT:-5432}/\${POSTGRES_DB:-ss_tools}
|
|
BACKEND_PORT: 8000
|
|
AUTH_SECRET_KEY: \${AUTH_SECRET_KEY:?Set AUTH_SECRET_KEY in .env}
|
|
ENCRYPTION_KEY: \${ENCRYPTION_KEY:?Set ENCRYPTION_KEY in .env}
|
|
ENABLE_BELIEF_STATE_LOGGING: \${ENABLE_BELIEF_STATE_LOGGING:-true}
|
|
TASK_LOG_LEVEL: \${TASK_LOG_LEVEL:-INFO}
|
|
INITIAL_ADMIN_CREATE: \${INITIAL_ADMIN_CREATE:-false}
|
|
INITIAL_ADMIN_USERNAME: \${INITIAL_ADMIN_USERNAME:-admin}
|
|
INITIAL_ADMIN_PASSWORD: \${INITIAL_ADMIN_PASSWORD:-}
|
|
INITIAL_ADMIN_EMAIL: \${INITIAL_ADMIN_EMAIL:-}
|
|
OPENAI_API_KEY: \${OPENAI_API_KEY:-}
|
|
ANTHROPIC_API_KEY: \${ANTHROPIC_API_KEY:-}
|
|
FEATURES__DATASET_REVIEW: \${FEATURES__DATASET_REVIEW:-true}
|
|
FEATURES__HEALTH_MONITOR: \${FEATURES__HEALTH_MONITOR:-true}
|
|
CERTS_PATH: /opt/certs
|
|
ports:
|
|
- "\${BACKEND_HOST_PORT:-8001}:8000"
|
|
volumes:
|
|
- ./storage:/app/storage
|
|
- \${CERTS_PATH:-./certs}:/opt/certs:ro
|
|
|
|
frontend:
|
|
image: ${frontend_tag}
|
|
pull_policy: never
|
|
restart: unless-stopped
|
|
depends_on:
|
|
- backend
|
|
- agent
|
|
environment:
|
|
SSL_KEY_PASSPHRASE: \${SSL_KEY_PASSPHRASE:-}
|
|
ports:
|
|
- "\${FRONTEND_HOST_PORT:-8000}:80"
|
|
- "\${FRONTEND_SSL_PORT:-443}:443"
|
|
volumes:
|
|
- \${CERTS_PATH:-./certs}:/opt/certs:ro
|
|
|
|
agent:
|
|
image: ${agent_tag}
|
|
pull_policy: never
|
|
restart: unless-stopped
|
|
depends_on:
|
|
- backend
|
|
environment:
|
|
LLM_API_KEY: \${LLM_API_KEY:-}
|
|
LLM_BASE_URL: \${LLM_BASE_URL:-https://api.openai.com/v1}
|
|
LLM_MODEL: \${LLM_MODEL:-gpt-4o}
|
|
FASTAPI_URL: http://backend:8000
|
|
AUTH_SECRET_KEY: \${AUTH_SECRET_KEY:?Set AUTH_SECRET_KEY in .env}
|
|
SERVICE_JWT: \${SERVICE_JWT:?Set SERVICE_JWT in .env}
|
|
DATABASE_URL: postgresql+psycopg2://\${POSTGRES_USER:-postgres}:\${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}@\${POSTGRES_HOST:?Set POSTGRES_HOST}:\${POSTGRES_PORT:-5432}/\${POSTGRES_DB:-ss_tools}
|
|
GRADIO_SERVER_PORT: 7860
|
|
GRADIO_ROOT_PATH: /api/agent/gradio
|
|
ports:
|
|
- "\${AGENT_HOST_PORT:-7860}:7860"
|
|
DEPLOY
|
|
}
|
|
|
|
generate_manifest() {
|
|
local tag="$1" backend_tag="$2" frontend_tag="$3" agent_tag="$4"
|
|
local suffix="${5:-}"
|
|
|
|
local backend_id frontend_id agent_id
|
|
backend_id="$(get_image_id "${backend_tag}")"
|
|
frontend_id="$(get_image_id "${frontend_tag}")"
|
|
agent_id="$(get_image_id "${agent_tag}")"
|
|
|
|
cat > "${DIST_ROOT}/manifest.${tag}${suffix}.txt" <<EOF
|
|
release_tag=${tag}
|
|
backend_image=${backend_tag}
|
|
backend_image_id=${backend_id}
|
|
backend_repo_digest=$(get_repo_digest "${backend_tag}")
|
|
backend_archive=superset-tools-backend.${tag}${suffix}.tar.xz
|
|
frontend_image=${frontend_tag}
|
|
frontend_image_id=${frontend_id}
|
|
frontend_repo_digest=$(get_repo_digest "${frontend_tag}")
|
|
frontend_archive=superset-tools-frontend.${tag}${suffix}.tar.xz
|
|
agent_image=${agent_tag}
|
|
agent_image_id=${agent_id}
|
|
agent_repo_digest=$(get_repo_digest "${agent_tag}")
|
|
agent_archive=superset-tools-agent.${tag}${suffix}.tar.xz
|
|
compose_file=docker-compose.enterprise-clean.yml
|
|
env_template=.env.enterprise-clean.example
|
|
env_bootstrap_fields=INITIAL_ADMIN_CREATE,INITIAL_ADMIN_USERNAME,INITIAL_ADMIN_PASSWORD,INITIAL_ADMIN_EMAIL
|
|
checksums_file=sha256sums.${tag}${suffix}.txt
|
|
generated_at_utc=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
EOF
|
|
|
|
cat > "${DIST_ROOT}/manifest.${tag}${suffix}.json" <<EOF
|
|
{
|
|
"release_tag": "${tag}",
|
|
"generated_at_utc": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")",
|
|
"images": [
|
|
{
|
|
"role": "backend",
|
|
"image": "${backend_tag}",
|
|
"image_id": "${backend_id}",
|
|
"repo_digest": "$(get_repo_digest "${backend_tag}")",
|
|
"archive": "superset-tools-backend.${tag}${suffix}.tar.xz"
|
|
},
|
|
{
|
|
"role": "frontend",
|
|
"image": "${frontend_tag}",
|
|
"image_id": "${frontend_id}",
|
|
"repo_digest": "$(get_repo_digest "${frontend_tag}")",
|
|
"archive": "superset-tools-frontend.${tag}${suffix}.tar.xz"
|
|
},
|
|
{
|
|
"role": "agent",
|
|
"image": "${agent_tag}",
|
|
"image_id": "${agent_id}",
|
|
"repo_digest": "$(get_repo_digest "${agent_tag}")",
|
|
"archive": "superset-tools-agent.${tag}${suffix}.tar.xz"
|
|
}
|
|
],
|
|
"compose_file": "docker-compose.enterprise-clean.yml",
|
|
"env_template": ".env.enterprise-clean.example",
|
|
"env_bootstrap_fields": [
|
|
"INITIAL_ADMIN_CREATE",
|
|
"INITIAL_ADMIN_USERNAME",
|
|
"INITIAL_ADMIN_PASSWORD",
|
|
"INITIAL_ADMIN_EMAIL"
|
|
],
|
|
"checksums_file": "sha256sums.${tag}${suffix}.txt"
|
|
}
|
|
EOF
|
|
}
|
|
|
|
bundle_release() {
|
|
if [[ $# -lt 1 ]]; then
|
|
echo "Error: tag is required."
|
|
echo "Usage: ./build.sh bundle <tag> (e.g. v1.0.0)"
|
|
exit 1
|
|
fi
|
|
local tag="$1"
|
|
local backend_tag="superset-tools-backend:${tag}"
|
|
local frontend_tag="superset-tools-frontend:${tag}"
|
|
local agent_tag="superset-tools-agent:${tag}"
|
|
|
|
mkdir -p "$DIST_ROOT"
|
|
|
|
# Build all three images
|
|
build_backend "${tag}"
|
|
build_frontend "${tag}"
|
|
build_agent "${tag}"
|
|
|
|
# Export .tar.xz archives
|
|
echo "[bundle] Exporting .tar.xz archives..."
|
|
export_image "${backend_tag}" "${DIST_ROOT}/superset-tools-backend.${tag}.tar.xz"
|
|
export_image "${frontend_tag}" "${DIST_ROOT}/superset-tools-frontend.${tag}.tar.xz"
|
|
export_image "${agent_tag}" "${DIST_ROOT}/superset-tools-agent.${tag}.tar.xz"
|
|
|
|
# Checksums
|
|
echo "[bundle] Calculating checksums..."
|
|
(cd "${DIST_ROOT}" && sha256sum \
|
|
"superset-tools-backend.${tag}.tar.xz" \
|
|
"superset-tools-frontend.${tag}.tar.xz" \
|
|
"superset-tools-agent.${tag}.tar.xz" > "sha256sums.${tag}.txt")
|
|
|
|
# Generate deploy compose + manifest
|
|
generate_deploy_compose "${tag}" "${backend_tag}" "${frontend_tag}" "${agent_tag}"
|
|
generate_manifest "${tag}" "${backend_tag}" "${frontend_tag}" "${agent_tag}"
|
|
|
|
cp "${SCRIPT_DIR}/.env.enterprise-clean.example" "${DIST_ROOT}/.env.enterprise-clean.example"
|
|
|
|
echo "[bundle] ✅ Bundle created in ${DIST_ROOT}"
|
|
echo " # Load images (decompress stream):"
|
|
echo " xz -dc ${DIST_ROOT}/superset-tools-backend.${tag}.tar.xz | docker load"
|
|
echo " xz -dc ${DIST_ROOT}/superset-tools-frontend.${tag}.tar.xz | docker load"
|
|
echo " xz -dc ${DIST_ROOT}/superset-tools-agent.${tag}.tar.xz | docker load"
|
|
echo " # Edit .env.enterprise-clean.example -> .env.enterprise-clean, then:"
|
|
echo " docker compose -f ${DIST_ROOT}/docker-compose.enterprise-clean.yml up"
|
|
echo ""
|
|
echo " NOTE: '${tag}' bundle uses external PostgreSQL (set POSTGRES_HOST)."
|
|
echo " Corporate certs: mount .crt files via CERTS_PATH=./certs (optional)."
|
|
}
|
|
|
|
# ======================================================================
|
|
# BUNDLE: lightweight all-in-one (single image, no Playwright, <200 MB)
|
|
# ======================================================================
|
|
|
|
bundle_light() {
|
|
if [[ $# -lt 1 ]]; then
|
|
echo "Error: tag is required."
|
|
echo "Usage: ./build.sh bundle:light <tag> (e.g. v1.0.0)"
|
|
exit 1
|
|
fi
|
|
local tag="$1"
|
|
local image_tag="superset-tools:${tag}"
|
|
|
|
# Git commit hash for APP_VERSION (e.g. "0.5.2+58d06fb2")
|
|
local git_commit
|
|
git_commit="$(git rev-parse --short HEAD 2>/dev/null || echo 'unknown')"
|
|
local app_version="${tag}+${git_commit}"
|
|
|
|
mkdir -p "$DIST_ROOT"
|
|
|
|
echo "[bundle:light] Building all-in-one image ${image_tag}..."
|
|
docker build --build-arg "APP_VERSION=${app_version}" -f docker/all-in-one.Dockerfile -t "${image_tag}" .
|
|
|
|
echo "[bundle:light] Exporting .tar.xz..."
|
|
docker save "${image_tag}" | xz -T0 -3 > "${DIST_ROOT}/superset-tools.${tag}.tar.xz"
|
|
|
|
echo "[bundle:light] Calculating checksums..."
|
|
(
|
|
cd "${DIST_ROOT}"
|
|
sha256sum "superset-tools.${tag}.tar.xz" > "sha256sums-light.${tag}.txt"
|
|
)
|
|
|
|
local img_id repo_digest
|
|
img_id="$(get_image_id "${image_tag}")"
|
|
repo_digest="$(get_repo_digest "${image_tag}")"
|
|
|
|
cat > "${DIST_ROOT}/manifest-light.${tag}.txt" <<EOF
|
|
release_tag=${tag}
|
|
image=${image_tag}
|
|
image_id=${img_id}
|
|
repo_digest=${repo_digest}
|
|
archive=superset-tools.${tag}.tar.xz
|
|
compose_file=docker-compose.light.yml
|
|
checksums_file=sha256sums-light.${tag}.txt
|
|
generated_at_utc=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
EOF
|
|
|
|
cat > "${DIST_ROOT}/manifest-light.${tag}.json" <<EOF
|
|
{
|
|
"release_tag": "${tag}",
|
|
"generated_at_utc": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")",
|
|
"image": "${image_tag}",
|
|
"image_id": "${img_id}",
|
|
"repo_digest": "${repo_digest}",
|
|
"archive": "superset-tools.${tag}.tar.xz",
|
|
"compose_file": "docker-compose.light.yml",
|
|
"checksums_file": "sha256sums-light.${tag}.txt"
|
|
}
|
|
EOF
|
|
|
|
cat > "${DIST_ROOT}/docker-compose.light.yml" <<'COMPOSE'
|
|
services:
|
|
db:
|
|
image: postgres:16-alpine
|
|
pull_policy: never
|
|
restart: unless-stopped
|
|
environment:
|
|
POSTGRES_DB: ${POSTGRES_DB:-ss_tools}
|
|
POSTGRES_USER: ${POSTGRES_USER:-postgres}
|
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}
|
|
ports:
|
|
- "${POSTGRES_HOST_PORT:-5432}:5432"
|
|
volumes:
|
|
- postgres_data:/var/lib/postgresql/data
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres} -d ${POSTGRES_DB:-ss_tools}"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 10
|
|
|
|
app:
|
|
image: ${IMAGE:-superset-tools:latest}
|
|
pull_policy: never
|
|
restart: unless-stopped
|
|
depends_on:
|
|
db:
|
|
condition: service_healthy
|
|
environment:
|
|
DATABASE_URL: postgresql+psycopg2://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB:-ss_tools}
|
|
BACKEND_PORT: 8000
|
|
INITIAL_ADMIN_CREATE: ${INITIAL_ADMIN_CREATE:-false}
|
|
INITIAL_ADMIN_USERNAME: ${INITIAL_ADMIN_USERNAME:-admin}
|
|
INITIAL_ADMIN_PASSWORD: ${INITIAL_ADMIN_PASSWORD:-}
|
|
INITIAL_ADMIN_EMAIL: ${INITIAL_ADMIN_EMAIL:-}
|
|
OPENAI_API_KEY: ${OPENAI_API_KEY:-}
|
|
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
|
|
FEATURES__DATASET_REVIEW: ${FEATURES__DATASET_REVIEW:-true}
|
|
FEATURES__HEALTH_MONITOR: ${FEATURES__HEALTH_MONITOR:-true}
|
|
ports:
|
|
- "${APP_HOST_PORT:-8000}:8000"
|
|
volumes:
|
|
- ./storage:/app/storage
|
|
|
|
volumes:
|
|
postgres_data:
|
|
COMPOSE
|
|
|
|
cp "${SCRIPT_DIR}/.env.enterprise-clean.example" "${DIST_ROOT}/.env.light.example"
|
|
|
|
echo "[bundle:light] ✅ Lightweight bundle created in ${DIST_ROOT}"
|
|
echo " # Load image:"
|
|
echo " xz -dc ${DIST_ROOT}/superset-tools.${tag}.tar.xz | docker load"
|
|
echo " docker compose -f ${DIST_ROOT}/docker-compose.light.yml up"
|
|
echo ""
|
|
echo " Single container (without compose):"
|
|
echo " docker run -p 8000:8000 --env-file backend/.env ${image_tag}"
|
|
}
|
|
|
|
# ======================================================================
|
|
# BUNDLE: enterprise with embeddings (agent + sentence-transformers)
|
|
# ======================================================================
|
|
|
|
bundle_embeddings() {
|
|
if [[ $# -lt 1 ]]; then
|
|
echo "Error: tag is required."
|
|
echo "Usage: ./build.sh bundle:embeddings <tag> (e.g. v1.0.0)"
|
|
exit 1
|
|
fi
|
|
local tag="$1"
|
|
local suffix="-embeddings"
|
|
local backend_tag="superset-tools-backend:${tag}${suffix}"
|
|
local frontend_tag="superset-tools-frontend:${tag}${suffix}"
|
|
local agent_tag="superset-tools-agent:${tag}${suffix}"
|
|
|
|
mkdir -p "$DIST_ROOT"
|
|
|
|
# Build all three images
|
|
build_backend "${tag}${suffix}"
|
|
build_frontend "${tag}${suffix}"
|
|
build_agent_embeddings "${tag}${suffix}"
|
|
|
|
# Export .tar.xz archives
|
|
echo "[bundle:embeddings] Exporting .tar.xz archives..."
|
|
export_image "${backend_tag}" "${DIST_ROOT}/superset-tools-backend.${tag}${suffix}.tar.xz"
|
|
export_image "${frontend_tag}" "${DIST_ROOT}/superset-tools-frontend.${tag}${suffix}.tar.xz"
|
|
export_image "${agent_tag}" "${DIST_ROOT}/superset-tools-agent.${tag}${suffix}.tar.xz"
|
|
|
|
# Checksums
|
|
echo "[bundle:embeddings] Calculating checksums..."
|
|
(cd "${DIST_ROOT}" && sha256sum \
|
|
"superset-tools-backend.${tag}${suffix}.tar.xz" \
|
|
"superset-tools-frontend.${tag}${suffix}.tar.xz" \
|
|
"superset-tools-agent.${tag}${suffix}.tar.xz" > "sha256sums.${tag}${suffix}.txt")
|
|
|
|
# Generate deploy compose + manifest
|
|
generate_deploy_compose "${tag}${suffix}" "${backend_tag}" "${frontend_tag}" "${agent_tag}"
|
|
generate_manifest "${tag}" "${backend_tag}" "${frontend_tag}" "${agent_tag}" "${suffix}"
|
|
|
|
# Add variant field to manifest for embeddings
|
|
cat >> "${DIST_ROOT}/manifest.${tag}${suffix}.txt" <<EOF
|
|
agent_embeddings=true
|
|
EOF
|
|
|
|
cp "${SCRIPT_DIR}/.env.enterprise-clean.example" "${DIST_ROOT}/.env.enterprise-clean.example"
|
|
|
|
echo "[bundle:embeddings] ✅ Embeddings bundle created in ${DIST_ROOT}"
|
|
echo " # Load images:"
|
|
echo " xz -dc ${DIST_ROOT}/superset-tools-backend.${tag}${suffix}.tar.xz | docker load"
|
|
echo " xz -dc ${DIST_ROOT}/superset-tools-frontend.${tag}${suffix}.tar.xz | docker load"
|
|
echo " xz -dc ${DIST_ROOT}/superset-tools-agent.${tag}${suffix}.tar.xz | docker load"
|
|
echo " # Edit .env.enterprise-clean.example -> .env.enterprise-clean, then:"
|
|
echo " docker compose -f ${DIST_ROOT}/docker-compose.enterprise-clean.yml up"
|
|
echo ""
|
|
echo " NOTE: Agent image includes sentence-transformers+torch (~500 MB+)."
|
|
echo " Use 'bundle' for the default slim agent (no embedding model)."
|
|
}
|
|
|
|
# ======================================================================
|
|
# HELP
|
|
# ======================================================================
|
|
|
|
show_help() {
|
|
cat <<'HELP'
|
|
Usage: ./build.sh <command> [options]
|
|
|
|
Commands for local docker:
|
|
up [profile] Build and start services (default: current)
|
|
down [profile] Stop and remove containers
|
|
restart [profile] Rebuild and restart services
|
|
logs [profile] Tail logs from all services
|
|
status Show running containers for all profiles
|
|
|
|
Commands for individual image build (docker build only, no export):
|
|
build:backend <tag> Build backend image (superset-tools-backend:<tag>)
|
|
build:frontend <tag> Build frontend image (superset-tools-frontend:<tag>)
|
|
build:agent <tag> Build agent image (superset-tools-agent:<tag>)
|
|
|
|
Commands for single-image bundle (build + .tar.xz export):
|
|
bundle:backend <tag> Build + export backend .tar.xz only
|
|
bundle:frontend <tag> Build + export frontend .tar.xz only
|
|
bundle:agent <tag> Build + export agent .tar.xz only
|
|
|
|
Commands for full bundles (all three images):
|
|
bundle <tag> Default slim bundle (backend + frontend + agent .tar.xz's).
|
|
Agent: NO sentence-transformers/embeddings (~500 MB saved).
|
|
Backend: NO agent/ML/dev deps.
|
|
NOTE: external PostgreSQL only — no bundled postgres image.
|
|
REQUIRED: AUTH_SECRET_KEY, ENCRYPTION_KEY, POSTGRES_PASSWORD, SERVICE_JWT.
|
|
Example: ./build.sh bundle v1.0.0
|
|
|
|
bundle:embeddings <tag>
|
|
Enterprise bundle with semantic embedding routing.
|
|
Agent built WITH sentence-transformers+torch (larger image).
|
|
Same backend/frontend as default bundle.
|
|
REQUIRED: AUTH_SECRET_KEY, ENCRYPTION_KEY, POSTGRES_PASSWORD, SERVICE_JWT.
|
|
Example: ./build.sh bundle:embeddings v1.0.0
|
|
|
|
bundle:light <tag> Lightweight all-in-one image (.tar.xz, <200 MB, no Playwright)
|
|
Example: ./build.sh bundle:light v1.0.0
|
|
|
|
Profiles: current (default), master, enterprise-clean
|
|
|
|
Backward-compatible shortcuts:
|
|
./build.sh -> ./build.sh up current
|
|
./build.sh master -> ./build.sh up master
|
|
./build.sh enterprise-clean -> ./build.sh up enterprise-clean
|
|
|
|
Output:
|
|
Local docker: docker compose -p superset-tools-{profile}
|
|
Bundle artifacts: dist/docker/
|
|
|
|
Notes:
|
|
- 'current' and 'master' use docker-compose.yml (dev, with local postgres).
|
|
- 'enterprise-clean' uses docker-compose.enterprise-clean.yml (build from source,
|
|
external postgres, corporate certs, optional nginx SSL).
|
|
- Individual build commands (build:backend|frontend|agent) produce local docker
|
|
images without touching dist/docker/.
|
|
- Single-image bundle commands (bundle:backend|frontend|agent) build + export
|
|
only one .tar.xz, useful for incremental deployments.
|
|
HELP
|
|
}
|
|
|
|
# ======================================================================
|
|
# MAIN DISPATCH
|
|
# ======================================================================
|
|
|
|
main() {
|
|
local CMD="${1:-up}"
|
|
shift 2>/dev/null || true
|
|
|
|
case "$CMD" in
|
|
up|down|restart|logs|status|help|-h|--help|bundle|bundle:embeddings|bundle:light|\
|
|
build:backend|build:frontend|build:agent|\
|
|
bundle:backend|bundle:frontend|bundle:agent)
|
|
# Valid commands — proceed
|
|
;;
|
|
*)
|
|
# Backward compat: treat unknown arg as profile for "up"
|
|
local profile="$CMD"
|
|
CMD="up"
|
|
set -- "$profile" "$@"
|
|
;;
|
|
esac
|
|
|
|
case "$CMD" in
|
|
up) compose_up "${1:-current}" ;;
|
|
down) compose_down "${1:-current}" ;;
|
|
restart) compose_restart "${1:-current}" ;;
|
|
logs) compose_logs "${1:-current}" ;;
|
|
bundle) bundle_release "$@" ;;
|
|
bundle:embeddings) bundle_embeddings "$@" ;;
|
|
bundle:light) bundle_light "$@" ;;
|
|
build:backend) build_backend "$@" ;;
|
|
build:frontend) build_frontend "$@" ;;
|
|
build:agent) build_agent "$@" ;;
|
|
bundle:backend) bundle_single_backend "$@" ;;
|
|
bundle:frontend) bundle_single_frontend "$@" ;;
|
|
bundle:agent) bundle_single_agent "$@" ;;
|
|
status) compose_status ;;
|
|
help|-h|--help) show_help ;;
|
|
*)
|
|
echo "Error: unknown command '$CMD'. See ./build.sh help"
|
|
exit 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
main "$@"
|