mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(compose): use one secret bundle for local services

Browse Source
This commit is contained in:
marcopan
2026-07-12 11:30:43 +02:00
parent 32a2b71687
commit 70a19f290d
15 changed files with 411 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/compose.local-vector.yaml
+19 -21
View File
@@ -6,9 +6,8 @@ services:
THT_VECTOR_BOOTSTRAP_USER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}"
THT_VECTOR_READER_USER: "${THT_VECTOR_READER_USER:-thoth_vector_reader}"
THT_VECTOR_WRITER_USER: "${THT_VECTOR_WRITER_USER:-thoth_vector_writer}"
THT_VECTOR_READER_PASSWORD_FILE: /run/secrets/vector_reader_password
THT_VECTOR_WRITER_PASSWORD_FILE: /run/secrets/vector_writer_password
secrets: [vector_reader_password, vector_writer_password]
THT_SECRETS_FILE: /run/secrets/thothii.secrets
secrets: [{source: thothii_secrets, target: thothii.secrets}]
depends_on:
vector-migrate:
condition: service_completed_successfully
@@ -23,12 +22,16 @@ services:
environment:
POSTGRES_DB: "${THT_VECTOR_DATABASE:-thoth}"
POSTGRES_USER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}"
POSTGRES_PASSWORD_FILE: /run/secrets/vector_bootstrap_password
THT_VECTOR_MIGRATOR_USER: "${THT_VECTOR_MIGRATOR_USER:-thoth_vector_migrator}"
THT_VECTOR_READER_USER: "${THT_VECTOR_READER_USER:-thoth_vector_reader}"
THT_VECTOR_WRITER_USER: "${THT_VECTOR_WRITER_USER:-thoth_vector_writer}"
secrets: [vector_bootstrap_password, vector_migrator_password, vector_reader_password, vector_writer_password]
volumes: [vector_data:/var/lib/postgresql/data]
THT_SECRETS_FILE: /run/secrets/thothii.secrets
secrets: [{source: thothii_secrets, target: thothii.secrets}]
entrypoint: [/opt/thoth/vector-db-entrypoint.sh]
volumes:
- vector_data:/var/lib/postgresql/data
- ./deploy/vector/vector-db-entrypoint.sh:/opt/thoth/vector-db-entrypoint.sh:ro
- ./deploy/vector/secret-policy.sh:/opt/thoth/secret-policy.sh:ro
healthcheck:
test: [CMD-SHELL, "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
interval: 5s
@@ -50,8 +53,9 @@ services:
THT_VECTOR_MIGRATOR_USER: "${THT_VECTOR_MIGRATOR_USER:-thoth_vector_migrator}"
THT_VECTOR_READER_USER: "${THT_VECTOR_READER_USER:-thoth_vector_reader}"
THT_VECTOR_WRITER_USER: "${THT_VECTOR_WRITER_USER:-thoth_vector_writer}"
THT_SECRETS_FILE: /run/secrets/thothii.secrets
entrypoint: [/opt/thoth/reconcile-roles.sh]
secrets: [vector_bootstrap_password, vector_migrator_password, vector_reader_password, vector_writer_password]
secrets: [{source: thothii_secrets, target: thothii.secrets}]
volumes:
- ./deploy/vector/reconcile-roles.sh:/opt/thoth/reconcile-roles.sh:ro
- ./deploy/vector/secret-policy.sh:/opt/thoth/secret-policy.sh:ro
@@ -69,10 +73,14 @@ services:
entrypoint: [sh, -ec]
command:
- |
password=$$(cat /run/secrets/vector_migrator_password)
encoded=$$(python -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$$password")
exec /opt/venv/bin/tht vector migrate --database-url "postgresql+psycopg2://${THT_VECTOR_MIGRATOR_USER:-thoth_vector_migrator}:$$encoded@vector-db:5432/${THT_VECTOR_DATABASE:-thoth}" --json
secrets: [vector_migrator_password]
. /opt/thoth/secret-policy.sh
export PGPASSWORD=$$(read_bundle_secret /run/secrets/thothii.secrets THT_VECTOR_MIGRATOR_PASSWORD)
exec /opt/venv/bin/tht vector migrate --database-url "postgresql+psycopg2://${THT_VECTOR_MIGRATOR_USER:-thoth_vector_migrator}@vector-db:5432/${THT_VECTOR_DATABASE:-thoth}" --json
secrets: [{source: thothii_secrets, target: thothii.secrets}]
environment:
THT_SECRETS_FILE: /run/secrets/thothii.secrets
volumes:
- ./deploy/vector/secret-policy.sh:/opt/thoth/secret-policy.sh:ro
depends_on:
vector-reconcile: {condition: service_completed_successfully}
restart: "no"
@@ -80,13 +88,3 @@ services:
volumes:
vector_data:
labels: {io.thothii.smoke-owner: "${THOTH_SMOKE_OWNER:-operator}"}
secrets:
vector_bootstrap_password:
file: ${THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE:-deploy/secrets/vector_bootstrap_password}
vector_migrator_password:
file: ${THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE:-deploy/secrets/vector_migrator_password}
vector_reader_password:
file: ${THT_VECTOR_READER_PASSWORD_SECRET_FILE:-deploy/secrets/vector_reader_password}
vector_writer_password:
file: ${THT_VECTOR_WRITER_PASSWORD_SECRET_FILE:-deploy/secrets/vector_writer_password}
deploy/compose.preprocess-local-vector.yaml
+4 -11
View File
@@ -1,21 +1,14 @@
services:
preprocess-evidence:
environment:
THT_VECTOR_READER_PASSWORD_FILE: /run/secrets/vector_reader_password
THT_VECTOR_WRITER_PASSWORD_FILE: /run/secrets/vector_writer_password
secrets: [vector_reader_password, vector_writer_password]
THT_SECRETS_FILE: /run/secrets/thothii.secrets
secrets: [{source: thothii_secrets, target: thothii.secrets}]
depends_on:
vector-migrate: {condition: service_completed_successfully}
preprocess-dwh:
environment:
THT_VECTOR_READER_PASSWORD_FILE: /run/secrets/vector_reader_password
secrets: [vector_reader_password]
THT_SECRETS_FILE: /run/secrets/thothii.secrets
secrets: [{source: thothii_secrets, target: thothii.secrets}]
depends_on:
vector-migrate: {condition: service_completed_successfully}
secrets:
vector_reader_password:
file: ${THT_VECTOR_READER_PASSWORD_SECRET_FILE:?set THT_VECTOR_READER_PASSWORD_SECRET_FILE}
vector_writer_password:
file: ${THT_VECTOR_WRITER_PASSWORD_SECRET_FILE:?set THT_VECTOR_WRITER_PASSWORD_SECRET_FILE}
deploy/compose.preprocess.yaml
+4
View File
@@ -10,6 +10,8 @@ services:
environment:
THT_DATA_ROOT: /data
THT_OLLAMA_URL: "${THT_OLLAMA_URL:-http://host.docker.internal:11434}"
THT_SECRETS_FILE: /run/secrets/thothii.secrets
secrets: [{source: thothii_secrets, target: thothii.secrets}]
volumes:
- thoth_data:/data
- ./deploy/workspaces:/app/harness/workspaces:ro
@@ -25,6 +27,8 @@ services:
command: ["mkdir -p /data/workspaces/preprocess-dwh && exec /app/docker/core-entrypoint.sh preprocess dwh --steps introspect --json -c /app/harness/workspaces/preprocess-dwh.yaml"]
environment:
THT_DATA_ROOT: /data
THT_SECRETS_FILE: /run/secrets/thothii.secrets
secrets: [{source: thothii_secrets, target: thothii.secrets}]
volumes:
- thoth_data:/data
- ./deploy/workspaces:/app/harness/workspaces:ro
deploy/vector/reconcile-roles.sh
+5 -4
View File
@@ -3,10 +3,11 @@ set -eu
. /opt/thoth/secret-policy.sh
export PGPASSWORD=$(read_secret_file /run/secrets/vector_bootstrap_password vector_bootstrap_password)
migrator_password=$(read_secret_file /run/secrets/vector_migrator_password vector_migrator_password)
reader_password=$(read_secret_file /run/secrets/vector_reader_password vector_reader_password)
writer_password=$(read_secret_file /run/secrets/vector_writer_password vector_writer_password)
bundle=${THT_SECRETS_FILE:-/run/secrets/thothii.secrets}
export PGPASSWORD=$(read_bundle_secret "$bundle" THT_VECTOR_BOOTSTRAP_PASSWORD)
migrator_password=$(read_bundle_secret "$bundle" THT_VECTOR_MIGRATOR_PASSWORD)
reader_password=$(read_bundle_secret "$bundle" THT_VECTOR_READER_PASSWORD)
writer_password=$(read_bundle_secret "$bundle" THT_VECTOR_WRITER_PASSWORD)
psql --set=ON_ERROR_STOP=1 \
--set=migrator_user="$THT_VECTOR_MIGRATOR_USER" \
deploy/vector/secret-policy.sh
+47 -1
View File
@@ -3,7 +3,7 @@
validate_secret_file() {
secret_path=$1
secret_name=$2
if [ ! -f "$secret_path" ] || [ ! -r "$secret_path" ] || [ ! -s "$secret_path" ]; then
if [ -L "$secret_path" ] || [ ! -f "$secret_path" ] || [ ! -r "$secret_path" ] || [ ! -s "$secret_path" ]; then
echo "$secret_name must be a readable, non-empty regular file" >&2
return 2
fi
@@ -22,3 +22,49 @@ read_secret_file() {
validate_secret_file "$1" "$2" || return
cat "$1"
}
# Read one value from the deployment bundle without putting the bundle itself in
# a service environment. The parser is deliberately strict: one KEY=VALUE per
# line, no duplicate keys, no unknown syntax, and no whitespace in credentials.
read_bundle_secret() {
bundle_path=$1
bundle_key=$2
if [ -L "$bundle_path" ] || [ ! -f "$bundle_path" ] || [ ! -r "$bundle_path" ] || [ ! -s "$bundle_path" ]; then
echo "secret bundle must be a readable, non-empty regular file" >&2
return 2
fi
mode=$(stat -c '%a' "$bundle_path" 2>/dev/null || stat -f '%Lp' "$bundle_path" 2>/dev/null) || return 2
case "$bundle_path:$mode" in
/run/secrets/*:444|/run/secrets/*:400|/run/secrets/*:600|*:600|*:400) ;;
*) echo "secret bundle must have mode 0600 or stricter (Docker secrets may be 0444)" >&2; return 2 ;;
esac
case "$bundle_key" in
THT_[A-Z0-9_]*|PI_PROVIDER_API_KEY) ;;
*) echo "invalid secret bundle key" >&2; return 2 ;;
esac
value=$(awk -v wanted="$bundle_key" '
/^[[:space:]]*$/ || /^[[:space:]]*#/ { next }
/^[A-Z][A-Z0-9_]*=/ {
key=$0; sub(/=.*/, "", key)
val=$0; sub(/^[^=]*=/, "", val)
if (key !~ /^(THT_MODEL_API_KEY|THT_DWH_API_KEY|THT_VEC_API_KEY|THT_VEC_WRITE_API_KEY|THT_CA|THT_SSL_CA|THT_VECTOR_BOOTSTRAP_PASSWORD|THT_VECTOR_MIGRATOR_PASSWORD|THT_VECTOR_READER_PASSWORD|THT_VECTOR_WRITER_PASSWORD|PI_PROVIDER_API_KEY)$/) exit 6
if (val == "") exit 7
if (++seen[key] > 1) exit 8
if (key == wanted) {
if (found) exit 3
found=1; print val
}
next
}
{ exit 4 }
END { if (!found) exit 5 }
' "$bundle_path") || {
echo "$bundle_key is unavailable in secret bundle" >&2
return 2
}
if [ -z "$value" ] || printf '%s' "$value" | LC_ALL=C grep -q '[[:space:]]'; then
echo "$bundle_key must contain no whitespace" >&2
return 2
fi
printf '%s' "$value"
}
deploy/vector/vector-db-entrypoint.sh
Executable
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
. /opt/thoth/secret-policy.sh
bundle=${THT_SECRETS_FILE:-/run/secrets/thothii.secrets}
export POSTGRES_PASSWORD=$(read_bundle_secret "$bundle" THT_VECTOR_BOOTSTRAP_PASSWORD)
unset THT_SECRETS_FILE
exec /usr/local/bin/docker-entrypoint.sh postgres