From 32a2b71687305475254ec632de1592805456f5cd Mon Sep 17 00:00:00 2001 From: mptyl Date: Sun, 12 Jul 2026 11:14:36 +0200 Subject: [PATCH] build(compose): make root startup the default --- .env.example | 28 ++++++++++++++++++ .gitignore | 1 + .superpowers/sdd/task-2-report.md | 30 +++++++++++++++++++ compose.yaml | 19 ++++++++++-- deploy/compose.local.yaml | 7 +++-- deploy/compose.production.yaml | 29 +----------------- deploy/env.example | 41 +++++++------------------- deploy/secrets/thothii.secrets.example | 20 +++++++++++++ scripts/test-container-deployment.sh | 31 +++++++++---------- scripts/test-default-compose.sh | 34 +++++++++++++++++++++ 10 files changed, 161 insertions(+), 79 deletions(-) create mode 100644 .env.example create mode 100644 .superpowers/sdd/task-2-report.md create mode 100644 deploy/secrets/thothii.secrets.example create mode 100755 scripts/test-default-compose.sh diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..bac64f66 --- /dev/null +++ b/.env.example @@ -0,0 +1,28 @@ +# ThothII Compose defaults. Copy this file to .env in the repository root. +# The root .env is loaded automatically by Docker Compose; do not put secrets here. + +COMPOSE_FILE=compose.yaml +COMPOSE_PROFILES= +THT_SECRETS_FILE=deploy/secrets/thothii.secrets + +THOTH_HTTP_PORT=8080 +AUTH_MODE=none +THOTH_PUBLIC_EXPOSURE=false +MAX_PI_PROCESSES=4 +PI_PROVIDER= +PI_MODEL= +PI_THINKING= + +# Set these for the selected DWH/vector/embedding adapters. +THT_DB_NAME= +THT_DWH_REST_URL= +THT_VEC_REST_URL= +THT_OLLAMA_URL= +THT_DOCS_ROOT=/data/workspaces/example/evidence-source + +# Local-vector defaults (used by the optional local-vector overlay). +THT_VECTOR_DATABASE=thoth +THT_VECTOR_BOOTSTRAP_USER=postgres +THT_VECTOR_MIGRATOR_USER=thoth_vector_migrator +THT_VECTOR_READER_USER=thoth_vector_reader +THT_VECTOR_WRITER_USER=thoth_vector_writer diff --git a/.gitignore b/.gitignore index bb12ea7c..7b0178fc 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ config/ca-chain.pem deploy/.env deploy/secrets/* !deploy/secrets/README.md +!deploy/secrets/*.example # === Runtime data (sessions contain PII; indexes are derived) === harness/sessions/ diff --git a/.superpowers/sdd/task-2-report.md b/.superpowers/sdd/task-2-report.md new file mode 100644 index 00000000..f9061d8c --- /dev/null +++ b/.superpowers/sdd/task-2-report.md @@ -0,0 +1,30 @@ +# Task 2 report — root Compose startup + +Status: DONE + +Implemented the root Compose defaults and the single bundle declaration: + +- added `.env.example` with automatic Compose defaults (`COMPOSE_FILE=compose.yaml`, an empty + profile, and the relative `THT_SECRETS_FILE` path); +- removed the mandatory `external` profile from `core` and `frontend`; +- mounted `deploy/secrets/thothii.secrets` at `/run/secrets/thothii.secrets` and passed only the + mounted path into the core container; +- changed the production overlay to inherit that bundle instead of declaring per-secret mounts; +- removed the local overlay's legacy `env_file` dependency; +- added the versioned bundle template and `.gitignore` exception; +- updated deployment security checks and added `scripts/test-default-compose.sh`. + +Focused verification: + +```text +./scripts/test-default-compose.sh # default Compose contract passed. +./scripts/test-container-deployment.sh # container deployment security contract passed. +./scripts/test-preprocess-compose-config.sh # preprocess compose config: ok +docker compose --env-file .env.example config --quiet + (with a temporary mode-0600 bundle via THT_SECRETS_FILE) +git diff --check +``` + +The local-vector and preprocess service secret declarations remain for Task 3, which converts +those services to the same bundle helper. Documentation and smoke command migration is reserved +for Task 4. diff --git a/compose.yaml b/compose.yaml index 2855be81..836179b9 100644 --- a/compose.yaml +++ b/compose.yaml @@ -6,7 +6,6 @@ x-smoke-labels: &smoke-labels services: core: image: thothii-core:local - profiles: [external] build: context: . dockerfile: docker/core.Dockerfile @@ -14,8 +13,21 @@ services: environment: AUTH_MODE: "${AUTH_MODE:-none}" THOTH_PUBLIC_EXPOSURE: "${THOTH_PUBLIC_EXPOSURE:-false}" + PI_PROVIDER: "${PI_PROVIDER:-}" + PI_MODEL: "${PI_MODEL:-}" + PI_THINKING: "${PI_THINKING:-}" + MAX_PI_PROCESSES: "${MAX_PI_PROCESSES:-4}" + THT_DB_NAME: "${THT_DB_NAME:-}" + THT_DWH_REST_URL: "${THT_DWH_REST_URL:-}" + THT_VEC_REST_URL: "${THT_VEC_REST_URL:-}" + THT_OLLAMA_URL: "${THT_OLLAMA_URL:-}" + THT_DOCS_ROOT: "${THT_DOCS_ROOT:-/data/workspaces/example/evidence-source}" + THT_SECRETS_FILE: /run/secrets/thothii.secrets THT_DATA_ROOT: /data SETTINGS_FILE: /data/settings/settings.json + secrets: + - source: thothii_secrets + target: thothii.secrets volumes: - thoth_data:/data - ./deploy/workspaces:/app/harness/workspaces:ro @@ -28,7 +40,6 @@ services: restart: unless-stopped frontend: - profiles: [external] build: context: . dockerfile: docker/frontend.Dockerfile @@ -55,3 +66,7 @@ volumes: networks: default: labels: *smoke-labels + +secrets: + thothii_secrets: + file: "${THT_SECRETS_FILE:-deploy/secrets/thothii.secrets}" diff --git a/deploy/compose.local.yaml b/deploy/compose.local.yaml index 846935cc..9c4ebbf9 100644 --- a/deploy/compose.local.yaml +++ b/deploy/compose.local.yaml @@ -1,5 +1,6 @@ services: core: - env_file: - - path: ./deploy/.env - required: false + environment: + # Non-secret settings come from the root .env interpolation file. + AUTH_MODE: "${AUTH_MODE:-none}" + THT_SECRETS_FILE: /run/secrets/thothii.secrets diff --git a/deploy/compose.production.yaml b/deploy/compose.production.yaml index 8931aa91..ab419743 100644 --- a/deploy/compose.production.yaml +++ b/deploy/compose.production.yaml @@ -8,31 +8,4 @@ services: THT_VEC_REST_URL: ${THT_VEC_REST_URL:?set THT_VEC_REST_URL} THT_OLLAMA_URL: ${THT_OLLAMA_URL:?set THT_OLLAMA_URL} THT_DOCS_ROOT: ${THT_DOCS_ROOT:-/data/workspaces/example/evidence-source} - THT_DWH_API_KEY_FILE: /run/secrets/dwh_api_key - THT_VEC_API_KEY_FILE: /run/secrets/vector_reader_api_key - THT_VEC_WRITE_API_KEY_FILE: /run/secrets/vector_writer_api_key - THT_MODEL_API_KEY_FILE: /run/secrets/model_api_key - THT_SSL_CA: /run/secrets/thoth_ca.pem - secrets: - - source: dwh_api_key - target: dwh_api_key - - source: vector_reader_api_key - target: vector_reader_api_key - - source: vector_writer_api_key - target: vector_writer_api_key - - source: thoth_ca - target: thoth_ca.pem - - source: model_api_key - target: model_api_key - -secrets: - dwh_api_key: - file: ${THT_DWH_API_KEY_SECRET_FILE:?set THT_DWH_API_KEY_SECRET_FILE} - vector_reader_api_key: - file: ${THT_VEC_API_KEY_SECRET_FILE:?set THT_VEC_API_KEY_SECRET_FILE} - vector_writer_api_key: - file: ${THT_VEC_WRITE_API_KEY_SECRET_FILE:?set THT_VEC_WRITE_API_KEY_SECRET_FILE} - thoth_ca: - file: ${THT_CA_SECRET_FILE:?set THT_CA_SECRET_FILE} - model_api_key: - file: ${THT_MODEL_API_KEY_SECRET_FILE:?set THT_MODEL_API_KEY_SECRET_FILE} + THT_SECRETS_FILE: /run/secrets/thothii.secrets diff --git a/deploy/env.example b/deploy/env.example index a9fe87ba..05e4d1f3 100644 --- a/deploy/env.example +++ b/deploy/env.example @@ -1,47 +1,26 @@ -# LOCAL/PRODUCTION CONFIGURATION TEMPLATE. Copy to deploy/.env. -# Never commit deploy/.env or the files under deploy/secrets/. +# Deprecated compatibility template. +# New installations should copy ../.env.example to ../.env and run +# `docker compose up --build -d` from the repository root. +# Never put secret values in this file. + +COMPOSE_FILE=compose.yaml:deploy/compose.local.yaml +COMPOSE_PROFILES= +THT_SECRETS_FILE=deploy/secrets/thothii.secrets -# Optional application defaults PI_PROVIDER= PI_MODEL= PI_THINKING= -# Container-only path is assigned by deploy/compose.production.yaml. -THT_MODEL_API_KEY_SECRET_FILE=deploy/secrets/model-api-key MAX_PI_PROCESSES=4 AUTH_MODE=none -# External DWH (example workspace uses the HTTP adapters) THT_DB_NAME= THT_DWH_REST_URL= -THT_DWH_API_KEY= -THT_DWH_API_KEY_SECRET_FILE=deploy/secrets/dwh-api-key - -# External vector service. Use a distinct write key where the service supports one. THT_VEC_REST_URL= -THT_VEC_API_KEY= -THT_VEC_WRITE_API_KEY= -THT_VEC_API_KEY_SECRET_FILE=deploy/secrets/vector-reader-api-key -THT_VEC_WRITE_API_KEY_SECRET_FILE=deploy/secrets/vector-writer-api-key -THT_CA_SECRET_FILE=deploy/secrets/ca-chain.pem +THT_OLLAMA_URL= +THT_DOCS_ROOT=/data/workspaces/example/evidence-source -# Optional local-vector profile. Keep these secret files outside Git and readable by Docker. THT_VECTOR_DATABASE=thoth THT_VECTOR_BOOTSTRAP_USER=postgres THT_VECTOR_MIGRATOR_USER=thoth_vector_migrator THT_VECTOR_READER_USER=thoth_vector_reader THT_VECTOR_WRITER_USER=thoth_vector_writer -THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE=deploy/secrets/vector_bootstrap_password -# Changing the file alone does not rotate an initialized DB; use -# scripts/vector-rotate-bootstrap-password.sh OLD_SECRET_FILE NEW_SECRET_FILE. -THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE=deploy/secrets/vector_migrator_password -THT_VECTOR_READER_PASSWORD_SECRET_FILE=deploy/secrets/vector_reader_password -THT_VECTOR_WRITER_PASSWORD_SECRET_FILE=deploy/secrets/vector_writer_password - -# External embeddings service -THT_OLLAMA_URL= - -# Evidence source visible inside the persistent data volume -THT_DOCS_ROOT=/data/workspaces/example/evidence-source - -# Optional CA file mounted separately by an operator, for example via a Compose override. -THT_SSL_CA= diff --git a/deploy/secrets/thothii.secrets.example b/deploy/secrets/thothii.secrets.example new file mode 100644 index 00000000..c6598060 --- /dev/null +++ b/deploy/secrets/thothii.secrets.example @@ -0,0 +1,20 @@ +# Copy to deploy/secrets/thothii.secrets and chmod 600. +# Values are read as literal strings (no shell expansion or command substitution). +# Leave unused keys out of the file. + +# Hosted model provider (single-key providers only). +# THT_MODEL_API_KEY=replace-me + +# External DWH and vector adapters. +# THT_DWH_API_KEY=replace-me +# THT_VEC_API_KEY=replace-me +# THT_VEC_WRITE_API_KEY=replace-me + +# Optional local-vector roles. +# THT_VECTOR_BOOTSTRAP_PASSWORD=replace-me +# THT_VECTOR_MIGRATOR_PASSWORD=replace-me +# THT_VECTOR_READER_PASSWORD=replace-me +# THT_VECTOR_WRITER_PASSWORD=replace-me + +# Optional CA material/path understood by the configured adapter. +# THT_CA=/run/secrets/ca-chain.pem diff --git a/scripts/test-container-deployment.sh b/scripts/test-container-deployment.sh index 6b0e5eaf..a15a3fbe 100755 --- a/scripts/test-container-deployment.sh +++ b/scripts/test-container-deployment.sh @@ -6,12 +6,15 @@ cd "$(dirname "$0")/.." tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT HUP INT TERM -docker compose --profile external config >"$tmp/base.yaml" +docker compose config >"$tmp/base.yaml" +grep -q '^ core:' "$tmp/base.yaml" +grep -q '^ frontend:' "$tmp/base.yaml" grep -q 'host_ip: 127.0.0.1' "$tmp/base.yaml" grep -q 'AUTH_MODE: none' "$tmp/base.yaml" grep -q 'THOTH_PUBLIC_EXPOSURE: "false"' "$tmp/base.yaml" -if grep -q 'env_file:' "$tmp/base.yaml"; then - echo "base/production-neutral Compose must not load the local env file" >&2 +grep -q 'THT_SECRETS_FILE: /run/secrets/thothii.secrets' "$tmp/base.yaml" +if grep -q 'THT_[A-Z0-9_]*_SECRET_FILE:' "$tmp/base.yaml"; then + echo "base Compose must not require legacy secret-file variables" >&2 exit 1 fi @@ -34,24 +37,22 @@ fi docker compose -f compose.yaml -f deploy/compose.local.yaml \ --profile external config >"$tmp/local.yaml" -grep -q 'env_file:' deploy/compose.local.yaml +if grep -q 'env_file:' "$tmp/local.yaml"; then + echo "local Compose must use the root .env interpolation file" >&2 + exit 1 +fi -for secret in dwh reader writer ca model; do printf '%s\n' "test-$secret" >"$tmp/$secret"; done -THT_DWH_API_KEY_SECRET_FILE="$tmp/dwh" \ -THT_VEC_API_KEY_SECRET_FILE="$tmp/reader" \ -THT_VEC_WRITE_API_KEY_SECRET_FILE="$tmp/writer" \ -THT_CA_SECRET_FILE="$tmp/ca" \ -THT_MODEL_API_KEY_SECRET_FILE="$tmp/model" \ +printf '%s\n' 'THT_MODEL_API_KEY=test-model' >"$tmp/thothii.secrets" +chmod 0600 "$tmp/thothii.secrets" +THT_SECRETS_FILE="$tmp/thothii.secrets" \ THT_DB_NAME=test THT_DWH_REST_URL=https://dwh.example.test \ THT_VEC_REST_URL=https://vector.example.test THT_OLLAMA_URL=https://embed.example.test \ docker compose -f compose.yaml -f deploy/compose.production.yaml \ --profile external config >"$tmp/production.yaml" grep -q 'AUTH_MODE: upstream' "$tmp/production.yaml" grep -q 'THOTH_PUBLIC_EXPOSURE: "true"' "$tmp/production.yaml" -grep -q 'target: thoth_ca.pem' "$tmp/production.yaml" -grep -q 'THT_DWH_API_KEY_FILE: /run/secrets/dwh_api_key' "$tmp/production.yaml" -grep -q 'THT_MODEL_API_KEY_FILE: /run/secrets/model_api_key' "$tmp/production.yaml" -grep -q 'target: model_api_key' "$tmp/production.yaml" +grep -q 'THT_SECRETS_FILE: /run/secrets/thothii.secrets' "$tmp/production.yaml" +grep -q 'target: thothii.secrets' "$tmp/production.yaml" if grep -q 'test-model' "$tmp/production.yaml"; then echo "rendered production config leaked the model API key" >&2 exit 1 @@ -66,7 +67,7 @@ if grep -q 'must-not-leak' "$tmp/legacy-model.err"; then echo "legacy model credential leaked through entrypoint diagnostics" >&2 exit 1 fi -if grep -q 'THT_VECTOR_READER_PASSWORD_FILE\|THT_VECTOR_WRITER_PASSWORD_FILE\|target: vector_reader_password\|target: vector_writer_password' "$tmp/production.yaml"; then +if grep -q 'THT_VECTOR_READER_PASSWORD_FILE\|THT_VECTOR_WRITER_PASSWORD_FILE\|target: vector_reader_password\|target: vector_writer_password\|dwh_api_key\|model_api_key' "$tmp/production.yaml"; then echo "production external config contains local direct vector secrets" >&2 exit 1 fi diff --git a/scripts/test-default-compose.sh b/scripts/test-default-compose.sh new file mode 100755 index 00000000..97540ba3 --- /dev/null +++ b/scripts/test-default-compose.sh @@ -0,0 +1,34 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT HUP INT TERM + +mkdir -p "$tmp/deploy/secrets" "$tmp/deploy/workspaces" +cp compose.yaml "$tmp/compose.yaml" +cp .env.example "$tmp/.env" +printf '%s\n' 'THT_MODEL_API_KEY=example-secret' >"$tmp/deploy/secrets/thothii.secrets" +chmod 0600 "$tmp/deploy/secrets/thothii.secrets" + +services=$(docker compose --project-directory "$tmp" config --services) +[ "$services" = "core +frontend" ] || { + echo "default Compose services must be core and frontend (got: $services)" >&2 + exit 1 +} + +rendered=$(docker compose --project-directory "$tmp" config) +printf '%s\n' "$rendered" | grep -q 'target: thothii.secrets' +if printf '%s\n' "$rendered" | grep -Eq 'dwh_api_key|vector_reader_api_key|vector_writer_api_key|model_api_key|thoth_ca'; then + echo "default Compose must not declare legacy per-secret mounts" >&2 + exit 1 +fi +if printf '%s\n' "$rendered" | grep -Eq 'THT_[A-Z0-9_]+_SECRET_FILE:'; then + echo "default Compose must not require legacy secret-file variables" >&2 + exit 1 +fi +printf '%s\n' "$rendered" | grep -q 'THT_SECRETS_FILE: /run/secrets/thothii.secrets' + +echo "default Compose contract passed."