feat(compose): use one secret bundle for local services

scripts/local-vector-smoke.sh

@@ -22,16 +22,27 @@ marker="local-vector-$smoke_project"
 restore_container="${smoke_project}-restore"
 restore_volume="${smoke_project}-restore-data"
 
-for secret in bootstrap migrator reader writer; do
-  password="smoke-${secret}-${smoke_project}"
-  printf '%s' "$password" >"$secret_dir/$secret"
-  chmod 0600 "$secret_dir/$secret"
-done
-
-export THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE="$secret_dir/bootstrap"
-export THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE="$secret_dir/migrator"
-export THT_VECTOR_READER_PASSWORD_SECRET_FILE="$secret_dir/reader"
-export THT_VECTOR_WRITER_PASSWORD_SECRET_FILE="$secret_dir/writer"
+bootstrap_password="smoke-bootstrap-$smoke_project"
+migrator_password="smoke-migrator-$smoke_project"
+reader_password="smoke-reader-$smoke_project"
+writer_password="smoke-writer-$smoke_project"
+bundle="$secret_dir/thothii.secrets"
+write_bundle() {
+  umask 077
+  {
+    printf 'THT_VECTOR_BOOTSTRAP_PASSWORD=%s\n' "$bootstrap_password"
+    printf 'THT_VECTOR_MIGRATOR_PASSWORD=%s\n' "$migrator_password"
+    printf 'THT_VECTOR_READER_PASSWORD=%s\n' "$reader_password"
+    printf 'THT_VECTOR_WRITER_PASSWORD=%s\n' "$writer_password"
+  } >"$bundle"
+  chmod 0600 "$bundle"
+}
+write_bundle
+export THT_SECRETS_FILE="$bundle"
+# The rotation helper has an old/new file interface; these are test-only
+# scratch files and are never mounted into a Compose service.
+printf '%s' "$bootstrap_password" >"$secret_dir/bootstrap"
+chmod 0600 "$secret_dir/bootstrap"
 export THT_VECTOR_BOOTSTRAP_USER=thoth_bootstrap_smoke
 export THOTH_SMOKE_OWNER="$smoke_owner"
 
@@ -109,7 +120,16 @@ if [ "$mode" = "--live-collision-test" ]; then
 fi
 
 probe_vector() {
-  compose exec -T core /opt/venv/bin/python - "$marker" "$1" <<'PY'
+  compose exec -T core sh -ec '
+    . /app/docker/secret-policy.sh
+    tmp=$(mktemp -d); trap "rm -rf \"$tmp\"" EXIT
+    for role in READER WRITER; do
+      file="$tmp/$role"
+      read_bundle_secret /run/secrets/thothii.secrets "THT_VECTOR_${role}_PASSWORD" >"$file"
+      export "THT_VECTOR_${role}_PASSWORD_FILE=$file"
+    done
+    exec /opt/venv/bin/python - "$1" "$2"
+  ' sh "$marker" "$1" <<'PY'
 import hashlib
 import os
 import sys
@@ -173,23 +193,23 @@ if printf '%s' "$inspect_env" | grep -q "smoke-\(reader\|writer\)-${smoke_projec
   echo "docker inspect exposed a direct vector password" >&2
   exit 1
 fi
-printf '%s' "$inspect_env" | grep -q 'THT_VECTOR_READER_PASSWORD_FILE=/run/secrets/vector_reader_password'
-printf '%s' "$inspect_env" | grep -q 'THT_VECTOR_WRITER_PASSWORD_FILE=/run/secrets/vector_writer_password'
+printf '%s' "$inspect_env" | grep -q 'THT_SECRETS_FILE=/run/secrets/thothii.secrets'
 migration_status=$(compose run --rm --no-deps vector-migrate)
 printf '%s\n' "$migration_status" | grep -q '"pending": \[\]'
 migrator_flags=$(compose run --rm --no-deps --entrypoint sh vector-reconcile -ec '
-  export PGPASSWORD=$(cat /run/secrets/vector_bootstrap_password)
+  . /opt/thoth/secret-policy.sh
+  export PGPASSWORD=$(read_bundle_secret /run/secrets/thothii.secrets THT_VECTOR_BOOTSTRAP_PASSWORD)
   psql -At --host vector-db --username "$THT_VECTOR_BOOTSTRAP_USER" --dbname thoth \
     --command "SELECT (NOT rolcreaterole) AND (NOT rolcreatedb) AND (NOT rolsuper) FROM pg_roles WHERE rolname = '\''thoth_vector_migrator'\''"
 ')
 test "$migrator_flags" = t
 probe_vector write
 
-old_reader_password=$(cat "$secret_dir/reader")
-for secret in migrator reader writer; do
-  password="rotated-${secret}-${smoke_project}"
-  printf '%s' "$password" >"$secret_dir/$secret"
-done
+old_reader_password="$reader_password"
+migrator_password="rotated-migrator-$smoke_project"
+reader_password="rotated-reader-$smoke_project"
+writer_password="rotated-writer-$smoke_project"
+write_bundle
 
 compose run --rm vector-reconcile
 rotation_status=$(compose run --rm --no-deps vector-migrate)
@@ -205,7 +225,7 @@ fi
 compose up --force-recreate --no-deps --wait core
 probe_vector read
 
-old_bootstrap_password=$(cat "$secret_dir/bootstrap")
+old_bootstrap_password="$bootstrap_password"
 printf '%s' "wrong-bootstrap-${smoke_project}" >"$secret_dir/bootstrap-wrong"
 printf '%s' "next-bootstrap-'quoted-${smoke_project}" >"$secret_dir/bootstrap-next"
 cp "$secret_dir/bootstrap" "$secret_dir/bootstrap-before-negative"
@@ -238,6 +258,8 @@ COMPOSE_PROJECT_NAME="$smoke_project" \
   ./scripts/vector-rotate-bootstrap-password.sh \
   "$secret_dir/bootstrap" "$secret_dir/bootstrap-next"
 new_bootstrap_password=$(cat "$secret_dir/bootstrap")
+bootstrap_password="$new_bootstrap_password"
+write_bundle
 test "$new_bootstrap_password" != "$old_bootstrap_password"
 if compose run --rm --no-deps --entrypoint psql \
     -e PGPASSWORD="$old_bootstrap_password" vector-reconcile \
@@ -271,9 +293,12 @@ if [ "$mode" = "--backup-restore" ]; then
     --label "io.thothii.smoke-owner=$smoke_owner" \
     --network "$network" --network-alias vector-db-restore \
     --mount "type=volume,source=$restore_volume,target=/var/lib/postgresql/data" \
-    --mount "type=bind,source=$secret_dir/bootstrap,target=/run/secrets/bootstrap,readonly" \
+    --mount "type=bind,source=$bundle,target=/run/secrets/thothii.secrets,readonly" \
+    --mount "type=bind,source=$(pwd)/deploy/vector/vector-db-entrypoint.sh,target=/opt/thoth/vector-db-entrypoint.sh,readonly" \
+    --mount "type=bind,source=$(pwd)/deploy/vector/secret-policy.sh,target=/opt/thoth/secret-policy.sh,readonly" \
     -e POSTGRES_DB=thoth -e POSTGRES_USER="$THT_VECTOR_BOOTSTRAP_USER" \
-    -e POSTGRES_PASSWORD_FILE=/run/secrets/bootstrap "$image" >/dev/null
+    -e THT_SECRETS_FILE=/run/secrets/thothii.secrets \
+    --entrypoint /opt/thoth/vector-db-entrypoint.sh "$image" >/dev/null
   attempts=0
   until docker exec "$restore_container" pg_isready \
       -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth >/dev/null 2>&1; do
@@ -300,7 +325,8 @@ if [ "$mode" = "--backup-restore" ]; then
     --output /scratch/vector.dump
 
   compose exec -T vector-db sh -ec '
-    export PGPASSWORD=$(cat /run/secrets/vector_bootstrap_password)
+    . /opt/thoth/secret-policy.sh
+    export PGPASSWORD=$(read_bundle_secret /run/secrets/thothii.secrets THT_VECTOR_BOOTSTRAP_PASSWORD)
     psql -X -U "$POSTGRES_USER" -d thoth -v ON_ERROR_STOP=1 --command \
       "UPDATE vectors.memory SET content_hash = '\''mutated-after-backup'\'' WHERE record_key = '\''$1'\''"' \
     sh "$marker" >/dev/null
@@ -339,23 +365,25 @@ if [ "$mode" = "--backup-restore" ]; then
   docker run --rm --network "$network" \
     --mount "type=bind,source=$(pwd)/deploy/vector/reconcile-roles.sh,target=/opt/thoth/reconcile-roles.sh,readonly" \
     --mount "type=bind,source=$(pwd)/deploy/vector/secret-policy.sh,target=/opt/thoth/secret-policy.sh,readonly" \
-    --mount "type=bind,source=$secret_dir/bootstrap,target=/run/secrets/vector_bootstrap_password,readonly" \
-    --mount "type=bind,source=$secret_dir/migrator,target=/run/secrets/vector_migrator_password,readonly" \
-    --mount "type=bind,source=$secret_dir/reader,target=/run/secrets/vector_reader_password,readonly" \
-    --mount "type=bind,source=$secret_dir/writer,target=/run/secrets/vector_writer_password,readonly" \
+    --mount "type=bind,source=$bundle,target=/run/secrets/thothii.secrets,readonly" \
     -e PGHOST=vector-db-restore -e PGDATABASE=thoth \
     -e PGUSER="$THT_VECTOR_BOOTSTRAP_USER" \
+    -e THT_SECRETS_FILE=/run/secrets/thothii.secrets \
     -e THT_VECTOR_MIGRATOR_USER=thoth_vector_migrator \
     -e THT_VECTOR_READER_USER=thoth_vector_reader \
     -e THT_VECTOR_WRITER_USER=thoth_vector_writer \
     --entrypoint /opt/thoth/reconcile-roles.sh "$image" >/dev/null
 
-  docker run --rm -i --network "$network" \
-    --mount "type=bind,source=$secret_dir/reader,target=/run/secrets/vector_reader_password,readonly" \
-    --mount "type=bind,source=$secret_dir/writer,target=/run/secrets/vector_writer_password,readonly" \
-    -e THT_VECTOR_READER_PASSWORD_FILE=/run/secrets/vector_reader_password \
-    -e THT_VECTOR_WRITER_PASSWORD_FILE=/run/secrets/vector_writer_password \
-    --entrypoint /opt/venv/bin/python thothii-core:local - "$marker" <<'PY'
+  compose exec -T core sh -ec '
+    . /app/docker/secret-policy.sh
+    tmp=$(mktemp -d); trap "rm -rf \"$tmp\"" EXIT
+    for role in READER WRITER; do
+      file="$tmp/$role"
+      read_bundle_secret /run/secrets/thothii.secrets "THT_VECTOR_${role}_PASSWORD" >"$file"
+      export "THT_VECTOR_${role}_PASSWORD_FILE=$file"
+    done
+    exec /opt/venv/bin/python - "$1"
+  ' sh "$marker" <<'PY'
 import hashlib
 import os
 import sys

scripts/preprocess-smoke.sh

@@ -48,14 +48,15 @@ trap 'exit 130' INT
 trap 'exit 143' TERM
 mkdir -p "$tmp/source/evidence"
 printf '%s\n' '# Evidence' 'generation one' >"$tmp/source/evidence/a.md"
-for name in bootstrap migrator reader writer; do
-  printf '%s' "smoke-$name-$project" >"$tmp/$name"
-  chmod 0600 "$tmp/$name"
-done
-export THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE="$tmp/bootstrap"
-export THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE="$tmp/migrator"
-export THT_VECTOR_READER_PASSWORD_SECRET_FILE="$tmp/reader"
-export THT_VECTOR_WRITER_PASSWORD_SECRET_FILE="$tmp/writer"
+bundle="$tmp/thothii.secrets"
+{
+  printf 'THT_VECTOR_BOOTSTRAP_PASSWORD=smoke-bootstrap-%s\n' "$project"
+  printf 'THT_VECTOR_MIGRATOR_PASSWORD=smoke-migrator-%s\n' "$project"
+  printf 'THT_VECTOR_READER_PASSWORD=smoke-reader-%s\n' "$project"
+  printf 'THT_VECTOR_WRITER_PASSWORD=smoke-writer-%s\n' "$project"
+} >"$bundle"
+chmod 0600 "$bundle"
+export THT_SECRETS_FILE="$bundle"
 export THT_OLLAMA_URL=http://mock-embeddings:8081
 
 cat >"$tmp/smoke.yaml" <<YAML

scripts/test-preprocess-compose-config.sh

@@ -3,10 +3,16 @@ set -eu
 
 cd "$(dirname "$0")/.."
 
-export THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE=${THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE:-/tmp/vector-bootstrap}
-export THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE=${THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE:-/tmp/vector-migrator}
-export THT_VECTOR_READER_PASSWORD_SECRET_FILE=${THT_VECTOR_READER_PASSWORD_SECRET_FILE:-/tmp/vector-reader}
-export THT_VECTOR_WRITER_PASSWORD_SECRET_FILE=${THT_VECTOR_WRITER_PASSWORD_SECRET_FILE:-/tmp/vector-writer}
+tmp_bundle=$(mktemp)
+trap 'rm -f "$tmp_bundle"' EXIT HUP INT TERM
+cat >"$tmp_bundle" <<'EOF'
+THT_VECTOR_BOOTSTRAP_PASSWORD=test-bootstrap
+THT_VECTOR_MIGRATOR_PASSWORD=test-migrator
+THT_VECTOR_READER_PASSWORD=test-reader
+THT_VECTOR_WRITER_PASSWORD=test-writer
+EOF
+chmod 0600 "$tmp_bundle"
+export THT_SECRETS_FILE="$tmp_bundle"
 
 local_files="-f compose.yaml -f deploy/compose.local-vector.yaml -f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml"
 local_json=$(docker compose $local_files --profile local-vector --profile preprocess config --format json)
@@ -16,6 +22,16 @@ import json, sys
 
 config = json.load(sys.stdin)
 services = config["services"]
+assert "thothii_secrets" in config.get("secrets", {}), config.get("secrets")
+assert "vector_bootstrap_password" not in config.get("secrets", {})
+assert "vector_migrator_password" not in config.get("secrets", {})
+assert "vector_reader_password" not in config.get("secrets", {})
+assert "vector_writer_password" not in config.get("secrets", {})
+for name, service in services.items():
+    if name.startswith("vector-") or name.startswith("preprocess-") or name == "core":
+        assert any(item.get("target") == "thothii.secrets" for item in service.get("secrets", []) if isinstance(item, dict)), (name, service.get("secrets"))
+        assert "vector_reader_password" not in str(service)
+        assert "vector_writer_password" not in str(service)
 for name in ("preprocess-evidence", "preprocess-dwh"):
     dependency = services[name].get("depends_on", {}).get("vector-migrate")
     assert dependency is not None, f"{name} does not depend on vector-migrate"
@@ -37,7 +53,32 @@ assert "vector-reconcile" not in services
 for name in ("preprocess-evidence", "preprocess-dwh"):
     service = services[name]
     assert "depends_on" not in service
-    assert not service.get("secrets"), service.get("secrets")
+    assert all(item.get("target") == "thothii.secrets" for item in service.get("secrets", []) if isinstance(item, dict)), service.get("secrets")
+    assert "vector_reader_password" not in str(service)
+    assert "vector_writer_password" not in str(service)
 '
 
+python3 - <<'PY'
+import os
+from pathlib import Path
+
+os.environ.update({
+    "THT_DB_NAME": "thoth",
+    "THT_DWH_REST_URL": "http://dwh.invalid",
+    "THT_DWH_API_KEY": "dwh",
+    "THT_VECTOR_DATABASE": "thoth",
+    "THT_VECTOR_READER_USER": "reader",
+    "THT_VECTOR_WRITER_USER": "writer",
+    "THT_VECTOR_READER_PASSWORD_FILE": "/tmp/generated-reader",
+    "THT_VECTOR_WRITER_PASSWORD_FILE": "/tmp/generated-writer",
+    "THT_DOCS_ROOT": "/data/source",
+    "THT_OLLAMA_URL": "http://ollama.invalid",
+})
+text = Path("deploy/workspaces/local-vector.yaml").read_text()
+assert "password_file: ${THT_VECTOR_READER_PASSWORD_FILE}" in text
+assert "password_file: ${THT_VECTOR_WRITER_PASSWORD_FILE}" in text
+assert "${THT_SECRETS_FILE}" not in text
+print("local-vector workspace resolution contract: ok")
+PY
+
 echo "preprocess compose config: ok"

scripts/test-vector-backup-restore-safety.sh

@@ -88,6 +88,13 @@ grep -q -- '--exit-on-error' "$tmp/restore.log"
 
 # The live restore smoke must follow the packaged migration set instead of a stale
 # hard-coded count when a new migration is added.
+if grep -Eq 'vector_(bootstrap|migrator|reader|writer)_password' \
+    deploy/compose.local-vector.yaml deploy/compose.preprocess-local-vector.yaml; then
+  echo "local-vector Compose still declares legacy per-password secrets" >&2
+  exit 1
+fi
+grep -Fq 'thothii_secrets' deploy/compose.local-vector.yaml
+grep -Fq 'thothii_secrets' deploy/compose.preprocess-local-vector.yaml
 if grep -Fq 'SELECT count(*) = 3 FROM public.tht_vector_migrations' \
     scripts/local-vector-smoke.sh; then
   echo "local vector smoke hard-codes the pre-004 migration count" >&2

scripts/test-vector-secret-policy.sh

@@ -14,10 +14,13 @@ printf 'safe-quoted-\047-dollar-$' >"$tmp/valid"
 printf 'docker-secret' >"$tmp/docker"
 printf 'owner-readonly' >"$tmp/readonly"
 printf 'too-open' >"$tmp/open"
+printf '# comment\n\nTHT_VECTOR_READER_PASSWORD=reader\nTHT_VECTOR_WRITER_PASSWORD=writer\n' >"$tmp/bundle"
+printf 'THT_VECTOR_READER_PASSWORD=reader\nTHT_VECTOR_WRITER_PASSWORD=writer\nTHT_DWH_API_KEY=one\nTHT_DWH_API_KEY=two\n' >"$tmp/duplicate-bundle"
 chmod 0600 "$tmp/valid"
 chmod 0444 "$tmp/docker"
 chmod 0400 "$tmp/readonly"
 chmod 0640 "$tmp/open"
+chmod 0600 "$tmp/bundle" "$tmp/duplicate-bundle"
 
 for invalid in empty newline space; do
   if validate_secret_file "$tmp/$invalid" "$invalid" >/dev/null 2>&1; then
@@ -36,5 +39,10 @@ if validate_secret_file "$tmp/open" open >/dev/null 2>&1; then
   exit 1
 fi
 test "$(read_secret_file "$tmp/valid" valid)" = "safe-quoted-'-dollar-$"
+test "$(read_bundle_secret "$tmp/bundle" THT_VECTOR_READER_PASSWORD)" = reader
+if read_bundle_secret "$tmp/duplicate-bundle" THT_VECTOR_READER_PASSWORD >/dev/null 2>&1; then
+  echo "secret policy accepted a duplicate unrelated bundle key" >&2
+  exit 1
+fi
 
 echo "shared vector secret policy contracts passed."