From 2595d35682a5200b877acb71764b4eb195a39ea4 Mon Sep 17 00:00:00 2001 From: mptyl Date: Tue, 4 Aug 2026 14:47:16 +0200 Subject: [PATCH] deploy: unify local and server compose stack --- .env.example | 49 +++++-------------- compose.yaml | 80 +++++++++++++------------------ deploy/compose.local.yaml | 13 +++-- deploy/compose.server.yaml | 17 +++++++ deploy/env/local.env.example | 16 +++++++ deploy/env/server.env.example | 19 ++++++++ scripts/test-default-compose.sh | 53 +++++++-------------- scripts/test-unified-compose.sh | 84 +++++++++++++++++++++++++++++++++ 8 files changed, 208 insertions(+), 123 deletions(-) create mode 100644 deploy/compose.server.yaml create mode 100644 deploy/env/local.env.example create mode 100644 deploy/env/server.env.example create mode 100755 scripts/test-unified-compose.sh diff --git a/.env.example b/.env.example index 9b5d3af0..17e67a4a 100644 --- a/.env.example +++ b/.env.example @@ -1,42 +1,15 @@ -# ThothII Compose defaults. Copy this file to .env in the repository root. -# The root .env is loaded automatically by Docker Compose; do not put secrets here. +# Common non-secret Compose values. Select local.env or server.env with --env-file. +# Run Compose with both files explicitly, for example: +# docker compose -f compose.yaml -f deploy/compose.local.yaml up -d --build -COMPOSE_FILE=compose.yaml -COMPOSE_PROFILES= -THT_SECRETS_FILE=deploy/secrets/thothii.secrets - -THOTH_HTTP_PORT=8080 -AUTH_MODE=none -THOTH_PUBLIC_EXPOSURE=false MAX_PI_PROCESSES=4 -PI_PROVIDER= -PI_MODEL= -PI_THINKING= -PI_AUTH_FILE=${HOME}/.pi/agent/auth.json - -# Git-backed workspace registry. Set the remote only in the installation environment; -# credentials and SSH/CA files remain outside this repository and are bind-mounted read-only. -THT_WORKSPACE_REGISTRY_ROOT=/data/workspace-registry +THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git THT_WORKSPACE_GIT_BRANCH=main -THT_WORKSPACE_INSTALLATION_ID=local -# THT_WORKSPACE_GIT_REMOTE=ssh://git@your-git-host/your-org/thoth-workspaces.git -# THT_WORKSPACE_GIT_CREDENTIALS_FILE=/absolute/path/to/git-credentials -# THT_WORKSPACE_GIT_CA_FILE=/absolute/path/to/git-ca.pem -# THT_WORKSPACE_GIT_SSH_KEY_FILE=/absolute/path/to/git-ssh-key -# THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=/absolute/path/to/git-known-hosts +THT_WORKSPACE_GIT_AUTHOR_NAME="Thoth Workspace Registry" +THT_WORKSPACE_GIT_AUTHOR_EMAIL=thoth-workspace-registry@example.invalid -# Set these for the selected DWH/vector/embedding adapters. -THT_DB_NAME= -THT_DWH_REST_URL= -THT_VEC_REST_URL= -THT_VEC_WRITE_REST_URL= -THT_OLLAMA_URL= -THT_DOCS_ROOT=/data/workspaces/example/evidence-source -THT_PROFILE=server - -# Local-vector defaults (used by the optional local-vector overlay). -THT_VECTOR_DATABASE=thoth -THT_VECTOR_BOOTSTRAP_USER=postgres -THT_VECTOR_MIGRATOR_USER=thoth_vector_migrator -THT_VECTOR_READER_USER=thoth_vector_reader -THT_VECTOR_WRITER_USER=thoth_vector_writer +THT_DB_NAME=warehouse +THT_DWH_REST_URL=https://dwh.example.invalid +THT_VEC_REST_URL=https://vector.example.invalid +THT_VEC_WRITE_REST_URL=https://vector-write.example.invalid +THT_OLLAMA_URL=https://embeddings.example.invalid diff --git a/compose.yaml b/compose.yaml index e9c600bb..36caa3d1 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,14 +1,3 @@ -# ThothII — deploy embedded nel portale omics_portal (PRODUZIONE). -# core + frontend sulla rete esterna del portale (omics_portal_omics_network, -# creata da Compose col prefisso project). Alias thothii-core/thothii-frontend -# per il DNS usato dagli upstream nginx del portale. -# NESSUNA porta host esposta: il backend è invisibile dall'esterno. -# -# Prereq: devono esistere entrambe le reti esterne: il portale crea -# omics_portal_omics_network e lo stack vLLM crea localllm_default. -# Avvia il portale con: -# cd /home/chirone/omics_portal && docker compose up -d -# Poi: docker compose up -d --build name: thothii services: @@ -17,68 +6,65 @@ services: context: . dockerfile: docker/core.Dockerfile image: thothii-core:local - env_file: - - path: deploy/thothii.env - required: false environment: HOST: 0.0.0.0 PORT: "8787" THT_HARNESS_DIR: /app/harness THT_BIN: /opt/venv/bin/tht + THT_DATA_ROOT: /data SETTINGS_FILE: /data/settings/settings.json - THT_CONFIG: /app/harness/workspaces/local.yaml # configPath di default per i route tht senza workspace esplicito - THT_MODEL_API_KEY_FILE: /data/secrets/model_api_key # provider key per buildPiChildEnv (codex) THT_WORKSPACE_REGISTRY_ROOT: /data/workspace-registry THT_WORKSPACE_GIT_REMOTE: ${THT_WORKSPACE_GIT_REMOTE:?set THT_WORKSPACE_GIT_REMOTE} THT_WORKSPACE_GIT_BRANCH: ${THT_WORKSPACE_GIT_BRANCH:-main} - THT_WORKSPACE_INSTALLATION_ID: ${THT_WORKSPACE_INSTALLATION_ID:-server} + THT_WORKSPACE_INSTALLATION_ID: ${THT_WORKSPACE_INSTALLATION_ID:-local} THT_WORKSPACE_GIT_AUTHOR_NAME: ${THT_WORKSPACE_GIT_AUTHOR_NAME:-Thoth Workspace Registry} THT_WORKSPACE_GIT_AUTHOR_EMAIL: ${THT_WORKSPACE_GIT_AUTHOR_EMAIL:-thoth-workspace-registry@localhost} THT_WORKSPACE_SECRET_ROOTS: /run/secrets - GIT_CONFIG_COUNT: "2" - GIT_CONFIG_KEY_0: credential.helper - GIT_CONFIG_VALUE_0: store --file=/run/secrets/workspace-registry-git-credentials - GIT_CONFIG_KEY_1: http.sslCAInfo - GIT_CONFIG_VALUE_1: /run/secrets/workspace-registry-git-ca - GIT_SSH_COMMAND: ssh -i /run/secrets/workspace-registry-git-ssh-key -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=/run/secrets/workspace-registry-git-known-hosts - AUTH_MODE: ${AUTH_MODE:-none} + THT_DB_NAME: ${THT_DB_NAME:-} + THT_DWH_REST_URL: ${THT_DWH_REST_URL:-} + THT_VEC_REST_URL: ${THT_VEC_REST_URL:-} + THT_VEC_WRITE_REST_URL: ${THT_VEC_WRITE_REST_URL:-} + THT_OLLAMA_URL: ${THT_OLLAMA_URL:-} MAX_PI_PROCESSES: ${MAX_PI_PROCESSES:-4} - extra_hosts: - - "host.docker.internal:host-gateway" # Supabase :5438 + Ollama :11434 sull'host volumes: - - /home/chirone/thothii-data:/data + - settings:/data/settings + - pi-state:/home/thoth/.pi - workspace-registry:/data/workspace-registry - - /home/chirone/thothii-data/pi-config:/home/thoth/.pi - - ${PI_AUTH_FILE:?set PI_AUTH_FILE}:/home/thoth/.pi/agent/auth.json:ro - - /home/chirone/chirone/etl/docs/evidence:/data/evidence:ro - - ${THT_WORKSPACE_GIT_CREDENTIALS_FILE:-/dev/null}:/run/secrets/workspace-registry-git-credentials:ro - - ${THT_WORKSPACE_GIT_CA_FILE:-/etc/ssl/certs/ca-certificates.crt}:/run/secrets/workspace-registry-git-ca:ro - - ${THT_WORKSPACE_GIT_SSH_KEY_FILE:-/dev/null}:/run/secrets/workspace-registry-git-ssh-key:ro - - ${THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE:-/dev/null}:/run/secrets/workspace-registry-git-known-hosts:ro - restart: unless-stopped + - sessions:/data/sessions + healthcheck: + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8787/health"] + interval: 15s + timeout: 3s + retries: 5 + start_period: 30s networks: - omics_portal_omics_network: - aliases: ["thothii-core"] - localllm_default: {} + - thothii frontend: build: context: . dockerfile: docker/frontend.Dockerfile args: - VITE_BASE: /datamart-builder/assets/ - VITE_BACKEND_URL: /datamart-builder/api + VITE_BASE: / + VITE_BACKEND_URL: /api image: thothii-frontend:local - restart: unless-stopped + depends_on: + core: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/ || exit 1"] + interval: 15s + timeout: 3s + retries: 5 + start_period: 10s networks: - omics_portal_omics_network: - aliases: ["thothii-frontend"] + - thothii networks: - omics_portal_omics_network: - external: true - localllm_default: - external: true + thothii: volumes: + settings: + pi-state: workspace-registry: + sessions: diff --git a/deploy/compose.local.yaml b/deploy/compose.local.yaml index 9c4ebbf9..4b9bf1b2 100644 --- a/deploy/compose.local.yaml +++ b/deploy/compose.local.yaml @@ -1,6 +1,13 @@ services: core: environment: - # Non-secret settings come from the root .env interpolation file. - AUTH_MODE: "${AUTH_MODE:-none}" - THT_SECRETS_FILE: /run/secrets/thothii.secrets + AUTH_MODE: none + THT_WORKSPACE_INSTALLATION_ID: local + ports: + - "127.0.0.1:${THOTH_CORE_HTTP_PORT:-8787}:8787" + restart: "no" + + frontend: + ports: + - "127.0.0.1:${THOTH_HTTP_PORT:-8080}:8080" + restart: "no" diff --git a/deploy/compose.server.yaml b/deploy/compose.server.yaml new file mode 100644 index 00000000..65c213c7 --- /dev/null +++ b/deploy/compose.server.yaml @@ -0,0 +1,17 @@ +services: + core: + environment: + AUTH_MODE: upstream + THOTH_PUBLIC_EXPOSURE: "true" + THT_DATA_ROOT: /data + THT_WORKSPACE_INSTALLATION_ID: server + volumes: !override + - ${THT_DATA_ROOT:?set THT_DATA_ROOT}:/data + - ${THT_PI_STATE_ROOT:?set THT_PI_STATE_ROOT}:/home/thoth/.pi + - ${THT_WORKSPACE_REGISTRY_ROOT:?set THT_WORKSPACE_REGISTRY_ROOT}:/data/workspace-registry + restart: unless-stopped + + frontend: + ports: + - "${THOTH_SERVER_BIND:-127.0.0.1}:${THOTH_HTTP_PORT:-8080}:8080" + restart: unless-stopped diff --git a/deploy/env/local.env.example b/deploy/env/local.env.example new file mode 100644 index 00000000..040e7d36 --- /dev/null +++ b/deploy/env/local.env.example @@ -0,0 +1,16 @@ +# Local profile defaults. Copy this file to an untracked local.env and pass it with --env-file. +# Values are non-secret documentation values only. +THOTH_HTTP_PORT=8080 +THOTH_CORE_HTTP_PORT=8787 +MAX_PI_PROCESSES=4 + +THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git +THT_WORKSPACE_GIT_BRANCH=main +THT_WORKSPACE_GIT_AUTHOR_NAME="Thoth Workspace Registry" +THT_WORKSPACE_GIT_AUTHOR_EMAIL=thoth-workspace-registry@example.invalid + +THT_DB_NAME=warehouse +THT_DWH_REST_URL=https://dwh.example.invalid +THT_VEC_REST_URL=https://vector.example.invalid +THT_VEC_WRITE_REST_URL=https://vector-write.example.invalid +THT_OLLAMA_URL=https://embeddings.example.invalid diff --git a/deploy/env/server.env.example b/deploy/env/server.env.example new file mode 100644 index 00000000..9888b7f2 --- /dev/null +++ b/deploy/env/server.env.example @@ -0,0 +1,19 @@ +# Server profile defaults. Copy this file to a reviewed, untracked server.env and pass it with --env-file. +# Values are non-secret documentation values only. +THOTH_SERVER_BIND=127.0.0.1 +THOTH_HTTP_PORT=8080 +MAX_PI_PROCESSES=4 + +THT_DATA_ROOT=/srv/thothii/data +THT_PI_STATE_ROOT=/srv/thothii/pi-state +THT_WORKSPACE_REGISTRY_ROOT=/srv/thothii/workspace-registry +THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git +THT_WORKSPACE_GIT_BRANCH=main +THT_WORKSPACE_GIT_AUTHOR_NAME="Thoth Workspace Registry" +THT_WORKSPACE_GIT_AUTHOR_EMAIL=thoth-workspace-registry@example.invalid + +THT_DB_NAME=warehouse +THT_DWH_REST_URL=https://dwh.example.invalid +THT_VEC_REST_URL=https://vector.example.invalid +THT_VEC_WRITE_REST_URL=https://vector-write.example.invalid +THT_OLLAMA_URL=https://embeddings.example.invalid diff --git a/scripts/test-default-compose.sh b/scripts/test-default-compose.sh index b9fcd803..aea682f8 100755 --- a/scripts/test-default-compose.sh +++ b/scripts/test-default-compose.sh @@ -1,43 +1,26 @@ -#!/bin/sh -set -eu +#!/usr/bin/env bash +set -euo pipefail cd "$(dirname "$0")/.." test -f .env.example -test -f deploy/secrets/thothii.secrets.example -grep -q '^docker compose up --build -d$' docs/installazione-docker-4-contesti.md -if grep -q 'cp deploy/env.example deploy/.env\|THT_[A-Z0-9_]*_SECRET_FILE=' docs/installazione-docker-4-contesti.md; then - echo "installation guide still presents the legacy per-file secret setup" >&2 +test -f deploy/env/local.env.example +test -f deploy/env/server.env.example + +rendered=$(mktemp) +trap 'rm -f "$rendered"' EXIT HUP INT TERM + +docker compose --env-file deploy/env/local.env.example \ + -f compose.yaml -f deploy/compose.local.yaml config >"$rendered" + +grep -q '^ core:' "$rendered" +grep -q '^ frontend:' "$rendered" +grep -q 'host_ip: 127.0.0.1' "$rendered" +grep -q 'AUTH_MODE: none' "$rendered" +grep -q 'THT_WORKSPACE_INSTALLATION_ID: local' "$rendered" +if grep -Eqi 'omics_portal|chirone|localllm_default|/home/chirone' "$rendered"; then + echo "default Compose contains application-specific coupling" >&2 exit 1 fi -tmp=$(mktemp -d) -trap 'rm -rf "$tmp"' EXIT HUP INT TERM - -mkdir -p "$tmp/deploy/secrets" "$tmp/deploy/workspaces" -cp compose.yaml "$tmp/compose.yaml" -cp .env.example "$tmp/.env" -cp deploy/secrets/thothii.secrets.example "$tmp/deploy/secrets/thothii.secrets" -printf '%s\n' 'THT_MODEL_API_KEY=example-secret' >>"$tmp/deploy/secrets/thothii.secrets" -chmod 0600 "$tmp/deploy/secrets/thothii.secrets" - -services=$(docker compose --project-directory "$tmp" config --services) -[ "$services" = "core -frontend" ] || { - echo "default Compose services must be core and frontend (got: $services)" >&2 - exit 1 -} - -rendered=$(docker compose --project-directory "$tmp" config) -printf '%s\n' "$rendered" | grep -q 'target: thothii.secrets' -if printf '%s\n' "$rendered" | grep -Eq 'dwh_api_key|vector_reader_api_key|vector_writer_api_key|model_api_key|thoth_ca'; then - echo "default Compose must not declare legacy per-secret mounts" >&2 - exit 1 -fi -if printf '%s\n' "$rendered" | grep -Eq 'THT_[A-Z0-9_]+_SECRET_FILE:'; then - echo "default Compose must not require legacy secret-file variables" >&2 - exit 1 -fi -printf '%s\n' "$rendered" | grep -q 'THT_SECRETS_FILE: /run/secrets/thothii.secrets' - echo "default Compose contract passed." diff --git a/scripts/test-unified-compose.sh b/scripts/test-unified-compose.sh new file mode 100755 index 00000000..40e096a7 --- /dev/null +++ b/scripts/test-unified-compose.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT HUP INT TERM + +render_profile() { + local profile=$1 + local env_file=$2 + local compose_file=$3 + local rendered="$tmp/$profile.json" + + docker compose --env-file "$env_file" -f compose.yaml -f "$compose_file" \ + config --format json >"$rendered" + + node - "$rendered" "$profile" <<'NODE' +const fs = require("fs"); + +const [configPath, profile] = process.argv.slice(2); +const config = JSON.parse(fs.readFileSync(configPath, "utf8")); +const services = Object.keys(config.services).sort(); +if (services.join(",") !== "core,frontend") throw new Error("mandatory stack must be core,frontend"); +if (/omics_portal|chirone|localllm_default|\/home\/chirone/i.test(JSON.stringify(config))) { + throw new Error("forbidden application coupling"); +} +if (!config.networks || !config.networks.thothii) throw new Error("base stack must define the thothii network"); + +const ports = Object.fromEntries( + Object.entries(config.services).map(([name, service]) => [name, service.ports || []]), +); +if (profile === "local") { + if (!ports.frontend.some((port) => port.host_ip === "127.0.0.1")) { + throw new Error("local frontend must publish a loopback port"); + } + if (ports.core.length !== 0 && !ports.core.every((port) => port.host_ip === "127.0.0.1")) { + throw new Error("local core may publish only loopback ports"); + } +} else { + if (ports.core.length !== 0) throw new Error("server core must not publish a host port"); + if (ports.frontend.length === 0) throw new Error("server frontend must publish a host port"); +} +NODE +} + +assert_remote_required() { + local env_file=$1 + local compose_file=$2 + local without_remote="$tmp/without-remote.env" + grep -v '^THT_WORKSPACE_GIT_REMOTE=' "$env_file" >"$without_remote" + + if env -u THT_WORKSPACE_GIT_REMOTE docker compose --env-file "$without_remote" -f compose.yaml -f "$compose_file" \ + config --format json >"$tmp/missing-remote.out" 2>"$tmp/missing-remote.err"; then + echo "Compose must require THT_WORKSPACE_GIT_REMOTE" >&2 + exit 1 + fi + grep -q 'THT_WORKSPACE_GIT_REMOTE' "$tmp/missing-remote.err" +} + +THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git \ +PI_AUTH_FILE=/dev/null \ + docker compose -f compose.yaml config --format json >"$tmp/base.json" +node - "$tmp/base.json" <<'NODE' +const fs = require("fs"); + +const config = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); +const services = Object.keys(config.services).sort(); +if (services.join(",") !== "core,frontend") throw new Error("mandatory stack must be core,frontend"); +if (/omics_portal|chirone|localllm_default|\/home\/chirone/i.test(JSON.stringify(config))) { + throw new Error("forbidden application coupling"); +} +if (!config.networks || !config.networks.thothii) throw new Error("base stack must define the thothii network"); +for (const volume of ["settings", "pi-state", "workspace-registry", "sessions"]) { + if (!config.volumes || !config.volumes[volume]) throw new Error(`missing required volume: ${volume}`); +} +NODE + +render_profile local deploy/env/local.env.example deploy/compose.local.yaml +render_profile server deploy/env/server.env.example deploy/compose.server.yaml +assert_remote_required deploy/env/local.env.example deploy/compose.local.yaml +assert_remote_required deploy/env/server.env.example deploy/compose.server.yaml + +echo "unified Compose contract passed."