fix(deploy): reconcile local vector credentials safely

2026-07-12 01:50:42 +02:00
parent 0ca9783f61
commit 62e0ff1f12
7 changed files with 323 additions and 64 deletions
@@ -53,3 +53,36 @@ the extension there. A clean-volume rerun passed.
  file-backed secrets and are never exposed to core.
 - The smoke intentionally refuses the operator project name `thothii` and removes only its unique
  project namespace and volumes.
+
+## Follow-up hardening — credential reconciliation and cleanup ownership
+
+Review findings were resolved in a separate follow-up:
+
+- Replaced fresh-volume-only initialization with `vector-reconcile`, an idempotent one-shot that
+  runs after database health and before `vector-migrate`. It authenticates with only the bootstrap
+  admin secret, safely creates missing identities, reconciles role attributes and passwords on
+  existing volumes, restores memberships/ownership, and leaves vector data untouched.
+- The migrator is explicitly `NOSUPERUSER NOCREATEDB NOCREATEROLE`. Schema/database ownership is
+  sufficient for all packaged migrations because reconciliation creates the two group roles first.
+- The live smoke rotates migrator, reader, and writer secrets on the same populated volume, rejects
+  the old reader credential, reruns migrations, recreates core with the new runtime credentials,
+  and retrieves the record written before rotation and again after database/core restart.
+- Smoke project names are no longer caller-controlled. Each run creates a unique namespace and
+  ownership token. Containers, networks, and volumes carry the ownership label; preflight refuses
+  any collision and cleanup verifies every discovered resource before `down --volumes`.
+- Added a dynamic fake-Docker contract suite for caller override, collision, and mismatched cleanup
+  labels, plus a real-Docker collision probe using a unique labeled volume.
+
+Follow-up verification:
+
+- `./scripts/local-vector-smoke.sh`: PASS, including live secret rotation and persisted retrieval
+- `./scripts/test-local-vector-smoke-safety.sh`: PASS
+- `./scripts/test-local-vector-smoke-live-collision.sh`: PASS
+- harness: 477 passed, 5 deselected
+- backend: 84 passed; TypeScript typecheck PASS
+- frontend: 226 passed; TypeScript typecheck PASS
+- Compose security, backend URL, config, shell syntax, and diff checks: PASS
+
+Remaining operational constraint: the bootstrap admin secret must continue to match the PostgreSQL
+bootstrap account stored in the volume. Runtime migrator/reader/writer rotation is supported without
+data deletion; bootstrap-account password rotation is a distinct database-administration operation.
@@ -1,5 +1,8 @@
 name: thothii

+x-smoke-labels: &smoke-labels
+  io.thothii.smoke-owner: "${THOTH_SMOKE_OWNER:-operator}"
+
 services:
  core:
    image: thothii-core:local
@@ -7,6 +10,7 @@ services:
    build:
      context: .
      dockerfile: docker/core.Dockerfile
+    labels: *smoke-labels
    environment:
      AUTH_MODE: "${AUTH_MODE:-none}"
      THOTH_PUBLIC_EXPOSURE: "${THOTH_PUBLIC_EXPOSURE:-false}"
@@ -37,6 +41,7 @@ services:
    build:
      context: .
      dockerfile: docker/frontend.Dockerfile
+    labels: *smoke-labels
    environment:
      BACKEND_BASE_URL: /api
    ports:
@@ -55,6 +60,7 @@ services:
  vector-db:
    image: pgvector/pgvector:0.8.5-pg16@sha256:1d533553fefe4f12e5d80c7b80622ba0c382abb5758856f52983d8789179f0fb
    profiles: [local-vector]
+    labels: *smoke-labels
    environment:
      POSTGRES_DB: "${THT_VECTOR_DATABASE:-thoth}"
      POSTGRES_USER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}"
@@ -69,7 +75,6 @@ services:
      - vector_writer_password
    volumes:
      - vector_data:/var/lib/postgresql/data
-      - ./deploy/vector/init/00-bootstrap.sh:/docker-entrypoint-initdb.d/00-bootstrap.sh:ro
    healthcheck:
      test: [CMD-SHELL, "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
      interval: 5s
@@ -78,12 +83,38 @@ services:
      start_period: 10s
    restart: unless-stopped

+  vector-reconcile:
+    image: pgvector/pgvector:0.8.5-pg16@sha256:1d533553fefe4f12e5d80c7b80622ba0c382abb5758856f52983d8789179f0fb
+    profiles: [local-vector]
+    labels: *smoke-labels
+    environment:
+      PGHOST: vector-db
+      PGPORT: 5432
+      PGDATABASE: "${THT_VECTOR_DATABASE:-thoth}"
+      PGUSER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}"
+      THT_VECTOR_MIGRATOR_USER: "${THT_VECTOR_MIGRATOR_USER:-thoth_vector_migrator}"
+      THT_VECTOR_READER_USER: "${THT_VECTOR_READER_USER:-thoth_vector_reader}"
+      THT_VECTOR_WRITER_USER: "${THT_VECTOR_WRITER_USER:-thoth_vector_writer}"
+    entrypoint: [/opt/thoth/reconcile-roles.sh]
+    secrets:
+      - vector_bootstrap_password
+      - vector_migrator_password
+      - vector_reader_password
+      - vector_writer_password
+    volumes:
+      - ./deploy/vector/reconcile-roles.sh:/opt/thoth/reconcile-roles.sh:ro
+    depends_on:
+      vector-db:
+        condition: service_healthy
+    restart: "no"
+
  vector-migrate:
    image: thothii-core:local
    profiles: [local-vector]
    build:
      context: .
      dockerfile: docker/core.Dockerfile
+    labels: *smoke-labels
    entrypoint: [sh, -ec]
    command:
      - |
@@ -95,13 +126,19 @@ services:
    secrets:
      - vector_migrator_password
    depends_on:
-      vector-db:
-        condition: service_healthy
+      vector-reconcile:
+        condition: service_completed_successfully
    restart: "no"

 volumes:
  thoth_data:
+    labels: *smoke-labels
  vector_data:
+    labels: *smoke-labels
+
+networks:
+  default:
+    labels: *smoke-labels

 secrets:
  vector_bootstrap_password:
@@ -1,36 +0,0 @@
-#!/bin/sh
-set -eu
-
-read_secret() {
-  value=$(cat "/run/secrets/$1")
-  if [ -z "$value" ] || printf '%s' "$value" | grep -q '[[:space:]]'; then
-    echo "$1 must be non-empty and contain no whitespace" >&2
-    exit 2
-  fi
-  printf '%s' "$value"
-}
-
-migrator_password=$(read_secret vector_migrator_password)
-reader_password=$(read_secret vector_reader_password)
-writer_password=$(read_secret vector_writer_password)
-
-psql --set=ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" \
-  --set=migrator_user="$THT_VECTOR_MIGRATOR_USER" \
-  --set=migrator_password="$migrator_password" \
-  --set=reader_user="$THT_VECTOR_READER_USER" \
-  --set=reader_password="$reader_password" \
-  --set=writer_user="$THT_VECTOR_WRITER_USER" \
-  --set=writer_password="$writer_password" <<'SQL'
-CREATE ROLE vector_reader NOLOGIN;
-CREATE ROLE vector_writer NOLOGIN;
-
-SELECT format('CREATE ROLE %I LOGIN PASSWORD %L CREATEROLE', :'migrator_user', :'migrator_password') \gexec
-SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'reader_user', :'reader_password') \gexec
-SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'writer_user', :'writer_password') \gexec
-SELECT format('GRANT vector_reader TO %I', :'reader_user') \gexec
-SELECT format('GRANT vector_writer TO %I', :'writer_user') \gexec
-SELECT format('ALTER DATABASE %I OWNER TO %I', current_database(), :'migrator_user') \gexec
-SELECT format('CREATE SCHEMA vectors AUTHORIZATION %I', :'migrator_user') \gexec
-REVOKE ALL ON SCHEMA vectors FROM PUBLIC;
-CREATE EXTENSION vector WITH SCHEMA vectors;
-SQL
@@ -0,0 +1,59 @@
+#!/bin/sh
+set -eu
+
+read_secret() {
+  value=$(cat "/run/secrets/$1")
+  if [ -z "$value" ] || printf '%s' "$value" | grep -q '[[:space:]]'; then
+    echo "$1 must be non-empty and contain no whitespace" >&2
+    exit 2
+  fi
+  printf '%s' "$value"
+}
+
+export PGPASSWORD=$(read_secret vector_bootstrap_password)
+migrator_password=$(read_secret vector_migrator_password)
+reader_password=$(read_secret vector_reader_password)
+writer_password=$(read_secret vector_writer_password)
+
+psql --set=ON_ERROR_STOP=1 \
+  --set=migrator_user="$THT_VECTOR_MIGRATOR_USER" \
+  --set=migrator_password="$migrator_password" \
+  --set=reader_user="$THT_VECTOR_READER_USER" \
+  --set=reader_password="$reader_password" \
+  --set=writer_user="$THT_VECTOR_WRITER_USER" \
+  --set=writer_password="$writer_password" <<'SQL'
+SELECT 'CREATE ROLE vector_reader NOLOGIN'
+WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'vector_reader') \gexec
+SELECT 'CREATE ROLE vector_writer NOLOGIN'
+WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'vector_writer') \gexec
+ALTER ROLE vector_reader NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION;
+ALTER ROLE vector_writer NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION;
+
+SELECT format('CREATE ROLE %I LOGIN', :'migrator_user')
+WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'migrator_user') \gexec
+SELECT format('CREATE ROLE %I LOGIN', :'reader_user')
+WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'reader_user') \gexec
+SELECT format('CREATE ROLE %I LOGIN', :'writer_user')
+WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'writer_user') \gexec
+
+SELECT format(
+  'ALTER ROLE %I LOGIN PASSWORD %L NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION',
+  :'migrator_user', :'migrator_password'
+) \gexec
+SELECT format(
+  'ALTER ROLE %I LOGIN PASSWORD %L NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION',
+  :'reader_user', :'reader_password'
+) \gexec
+SELECT format(
+  'ALTER ROLE %I LOGIN PASSWORD %L NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION',
+  :'writer_user', :'writer_password'
+) \gexec
+
+SELECT format('GRANT vector_reader TO %I', :'reader_user') \gexec
+SELECT format('GRANT vector_writer TO %I', :'writer_user') \gexec
+SELECT format('ALTER DATABASE %I OWNER TO %I', current_database(), :'migrator_user') \gexec
+SELECT format('CREATE SCHEMA IF NOT EXISTS vectors AUTHORIZATION %I', :'migrator_user') \gexec
+SELECT format('ALTER SCHEMA vectors OWNER TO %I', :'migrator_user') \gexec
+REVOKE ALL ON SCHEMA vectors FROM PUBLIC;
+CREATE EXTENSION IF NOT EXISTS vector WITH SCHEMA vectors;
+SQL
@@ -3,22 +3,23 @@ set -eu

 cd "$(dirname "$0")/.."

-smoke_project=${SMOKE_PROJECT:-"thothii-vector-smoke-$(date +%s)-$$"}
-keep_resources=${KEEP_SMOKE_RESOURCES:-0}
-secret_dir=$(mktemp -d)
-marker="local-vector-$smoke_project"
-
-case "$smoke_project" in
-  thothii)
-    echo "SMOKE_PROJECT=thothii is reserved for the operator stack" >&2
-    exit 2
-    ;;
-  ""|*[!a-z0-9_-]*|[!a-z0-9]*)
-    echo "invalid SMOKE_PROJECT: use lowercase letters, digits, hyphens, or underscores" >&2
-    exit 2
-    ;;
+mode=${1:-run}
+case "$mode" in
+  run|--live-collision-test) ;;
+  *) echo "usage: $0 [--live-collision-test]" >&2; exit 2 ;;
 esac

+keep_resources=${KEEP_SMOKE_RESOURCES:-0}
+if [ "${SMOKE_PROJECT+x}" = x ]; then
+  echo "SMOKE_PROJECT is not accepted; the smoke always generates an owned namespace" >&2
+  exit 2
+fi
+secret_dir=$(mktemp -d "${TMPDIR:-/tmp}/thothii-vector-smoke.XXXXXX")
+suffix=$(basename "$secret_dir" | tr -cd 'a-z0-9')
+smoke_project="thothii-vector-smoke-$(date +%s)-$$-$suffix"
+smoke_owner="$smoke_project-owner"
+marker="local-vector-$smoke_project"
+
 for secret in bootstrap migrator reader writer; do
  password="smoke-${secret}-${smoke_project}"
  printf '%s' "$password" >"$secret_dir/$secret"
@@ -31,23 +32,80 @@ export THT_VECTOR_READER_PASSWORD_SECRET_FILE="$secret_dir/reader"
 export THT_VECTOR_WRITER_PASSWORD_SECRET_FILE="$secret_dir/writer"
 export THT_VECTOR_READER_PASSWORD="smoke-reader-${smoke_project}"
 export THT_VECTOR_WRITER_PASSWORD="smoke-writer-${smoke_project}"
+export THOTH_SMOKE_OWNER="$smoke_owner"

 compose() {
  docker compose --project-name "$smoke_project" --profile local-vector "$@"
 }

+resource_ids() {
+  case "$1" in
+    container) docker ps -aq --filter "label=com.docker.compose.project=$smoke_project" ;;
+    volume) docker volume ls -q --filter "label=com.docker.compose.project=$smoke_project" ;;
+    network) docker network ls -q --filter "label=com.docker.compose.project=$smoke_project" ;;
+  esac
+}
+
+resource_owner() {
+  case "$1" in
+    container) docker inspect --format '{{ index .Config.Labels "io.thothii.smoke-owner" }}' "$2" ;;
+    volume) docker volume inspect --format '{{ index .Labels "io.thothii.smoke-owner" }}' "$2" ;;
+    network) docker network inspect --format '{{ index .Labels "io.thothii.smoke-owner" }}' "$2" ;;
+  esac
+}
+
+assert_no_collision() {
+  for kind in container volume network; do
+    ids=$(resource_ids "$kind")
+    if [ -n "$ids" ]; then
+      echo "refusing existing Compose project resources for generated namespace $smoke_project" >&2
+      return 1
+    fi
+  done
+}
+
+verify_owned_resources() {
+  for kind in container volume network; do
+    for id in $(resource_ids "$kind"); do
+      owner=$(resource_owner "$kind" "$id" 2>/dev/null || true)
+      if [ "$owner" != "$smoke_owner" ]; then
+        echo "refusing cleanup of resource not owned by this smoke: $kind $id" >&2
+        return 1
+      fi
+    done
+  done
+}
+
 cleanup() {
  if [ "$keep_resources" = "1" ]; then
    echo "Keeping smoke resources for project $smoke_project (KEEP_SMOKE_RESOURCES=1)." >&2
  else
-    compose down --volumes >/dev/null 2>&1 || true
+    if verify_owned_resources; then
+      compose down --volumes >/dev/null 2>&1 || true
+    fi
  fi
  rm -rf "$secret_dir"
 }
 trap cleanup EXIT HUP INT TERM

+if [ "$mode" = "--live-collision-test" ]; then
+  collision_volume="${smoke_project}-collision"
+  docker volume create \
+    --label "com.docker.compose.project=$smoke_project" \
+    --label 'io.thothii.smoke-owner=foreign-owner' \
+    "$collision_volume" >/dev/null
+  if assert_no_collision 2>/dev/null; then
+    echo "live collision probe was not detected" >&2
+    docker volume rm "$collision_volume" >/dev/null
+    exit 1
+  fi
+  docker volume rm "$collision_volume" >/dev/null
+  echo "live local-vector project collision refusal passed."
+  exit 0
+fi
+
 probe_vector() {
-  compose exec -T core /opt/venv/bin/python - "$marker" <<'PY'
+  compose exec -T core /opt/venv/bin/python - "$marker" "$1" <<'PY'
 import hashlib
 import os
 import sys
@@ -58,6 +116,7 @@ from tht.ports.vector import VectorWriteRecord
 from tht.vectorstore.records import VectorRecord

 marker = sys.argv[1]
+mode = sys.argv[2]
 database = "thoth"
 host = "vector-db"

@@ -85,28 +144,59 @@ record = VectorRecord(
    metadata={"smoke": True},
 )
 embedding = [1.0] + [0.0] * 767
-store.upsert(
-    "memory",
-    [VectorWriteRecord(record, embedding, hashlib.sha256(marker.encode()).hexdigest())],
-)
+if mode == "write":
+    store.upsert(
+        "memory",
+        [VectorWriteRecord(record, embedding, hashlib.sha256(marker.encode()).hexdigest())],
+    )
 hits = store.search(["memory"], embedding, limit=1, kinds=["memory"])
 assert hits and hits[0].id == marker, hits
-print(f"role health, upsert, and search passed for {marker}")
+print(f"role health and persisted search passed for {marker} ({mode})")
 PY
 }

+assert_no_collision
 compose config --quiet
 services=$(compose config --services)
 printf '%s\n' "$services" | grep -qx vector-db
+printf '%s\n' "$services" | grep -qx vector-reconcile
 printf '%s\n' "$services" | grep -qx vector-migrate

-compose up --build --wait vector-migrate core
+compose up --build --wait vector-reconcile vector-migrate core
 migration_status=$(compose run --rm --no-deps vector-migrate)
 printf '%s\n' "$migration_status" | grep -q '"pending": \[\]'
-probe_vector
+migrator_flags=$(compose run --rm --no-deps --entrypoint sh vector-reconcile -ec '
+  export PGPASSWORD=$(cat /run/secrets/vector_bootstrap_password)
+  psql -At --host vector-db --username postgres --dbname thoth \
+    --command "SELECT (NOT rolcreaterole) AND (NOT rolcreatedb) AND (NOT rolsuper) FROM pg_roles WHERE rolname = '\''thoth_vector_migrator'\''"
+')
+test "$migrator_flags" = t
+probe_vector write
+
+old_reader_password=$THT_VECTOR_READER_PASSWORD
+for secret in migrator reader writer; do
+  password="rotated-${secret}-${smoke_project}"
+  printf '%s' "$password" >"$secret_dir/$secret"
+done
+export THT_VECTOR_READER_PASSWORD="rotated-reader-${smoke_project}"
+export THT_VECTOR_WRITER_PASSWORD="rotated-writer-${smoke_project}"
+
+compose run --rm vector-reconcile
+rotation_status=$(compose run --rm --no-deps vector-migrate)
+printf '%s\n' "$rotation_status" | grep -q '"pending": \[\]'
+if compose run --rm --no-deps --entrypoint psql \
+    -e PGPASSWORD="$old_reader_password" vector-reconcile \
+    --host vector-db --username thoth_vector_reader --dbname thoth --command 'SELECT 1' \
+    >/dev/null 2>&1; then
+  echo "old reader credential still works after rotation" >&2
+  exit 1
+fi
+
+compose up --force-recreate --no-deps --wait core
+probe_vector read

 compose restart vector-db core
 compose up --wait vector-db core
-probe_vector
+probe_vector read

-echo "Local pgvector migration, least-privilege roles, search, and restart persistence passed."
+echo "Local pgvector migration, credential rotation, least-privilege roles, and persistence passed."
@@ -0,0 +1,5 @@
+#!/bin/sh
+set -eu
+
+cd "$(dirname "$0")/.."
+./scripts/local-vector-smoke.sh --live-collision-test
@@ -0,0 +1,71 @@
+#!/bin/sh
+set -eu
+
+cd "$(dirname "$0")/.."
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT HUP INT TERM
+
+fake="$tmp/docker"
+log="$tmp/docker.log"
+state="$tmp/state"
+
+cat >"$fake" <<'SH'
+#!/bin/sh
+set -eu
+printf '%s\n' "$*" >>"$FAKE_DOCKER_LOG"
+
+if [ "${FAKE_COLLISION:-0}" = 1 ] && [ "$1 $2" = "ps -aq" ]; then
+  printf '%s\n' collision-container
+  exit 0
+fi
+
+if [ "$1 $2" = "ps -aq" ] || [ "$1 $2" = "volume ls" ] || [ "$1 $2" = "network ls" ]; then
+  if [ "${FAKE_MISMATCH_ON_CLEANUP:-0}" = 1 ] && [ -f "$FAKE_DOCKER_STATE" ]; then
+    printf '%s\n' foreign-resource
+  fi
+  : >"$FAKE_DOCKER_STATE"
+  exit 0
+fi
+
+if [ "$1" = inspect ] || [ "$1 $2" = "volume inspect" ] || [ "$1 $2" = "network inspect" ]; then
+  printf '%s\n' foreign-owner
+  exit 0
+fi
+
+case "$*" in
+  *"config --services"*) printf '%s\n' vector-db vector-reconcile vector-migrate core frontend ;;
+  *"run --rm --no-deps vector-migrate"*) printf '%s\n' '{"applied":["001","002","003"],"drifted":[],"pending":[]}' ;;
+esac
+exit 0
+SH
+chmod 0755 "$fake"
+
+if PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" FAKE_DOCKER_STATE="$state" \
+    SMOKE_PROJECT=operator-owned ./scripts/local-vector-smoke.sh >"$tmp/out" 2>"$tmp/err"; then
+  echo "smoke accepted caller-controlled SMOKE_PROJECT" >&2
+  exit 1
+fi
+grep -q 'SMOKE_PROJECT is not accepted' "$tmp/err"
+test ! -s "$log"
+
+: >"$log"
+rm -f "$state"
+PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" FAKE_DOCKER_STATE="$state" \
+  FAKE_COLLISION=1 ./scripts/local-vector-smoke.sh >"$tmp/out" 2>"$tmp/err" || true
+grep -q 'refusing existing Compose project resources' "$tmp/err"
+if grep -q 'compose.*up' "$log"; then
+  echo "smoke started after detecting a project collision" >&2
+  exit 1
+fi
+
+: >"$log"
+rm -f "$state"
+PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" FAKE_DOCKER_STATE="$state" \
+  FAKE_MISMATCH_ON_CLEANUP=1 ./scripts/local-vector-smoke.sh >"$tmp/out" 2>"$tmp/err" || true
+grep -q 'refusing cleanup of resource not owned by this smoke' "$tmp/err"
+if grep -q 'down --volumes' "$log"; then
+  echo "smoke removed resources after ownership mismatch" >&2
+  exit 1
+fi
+
+echo "local-vector smoke collision and cleanup ownership contracts passed."