diff --git a/.superpowers/sdd/pgvector-task-3-report.md b/.superpowers/sdd/pgvector-task-3-report.md index d92088c3..40381cb2 100644 --- a/.superpowers/sdd/pgvector-task-3-report.md +++ b/.superpowers/sdd/pgvector-task-3-report.md @@ -53,3 +53,36 @@ the extension there. A clean-volume rerun passed. file-backed secrets and are never exposed to core. - The smoke intentionally refuses the operator project name `thothii` and removes only its unique project namespace and volumes. + +## Follow-up hardening — credential reconciliation and cleanup ownership + +Review findings were resolved in a separate follow-up: + +- Replaced fresh-volume-only initialization with `vector-reconcile`, an idempotent one-shot that + runs after database health and before `vector-migrate`. It authenticates with only the bootstrap + admin secret, safely creates missing identities, reconciles role attributes and passwords on + existing volumes, restores memberships/ownership, and leaves vector data untouched. +- The migrator is explicitly `NOSUPERUSER NOCREATEDB NOCREATEROLE`. Schema/database ownership is + sufficient for all packaged migrations because reconciliation creates the two group roles first. +- The live smoke rotates migrator, reader, and writer secrets on the same populated volume, rejects + the old reader credential, reruns migrations, recreates core with the new runtime credentials, + and retrieves the record written before rotation and again after database/core restart. +- Smoke project names are no longer caller-controlled. Each run creates a unique namespace and + ownership token. Containers, networks, and volumes carry the ownership label; preflight refuses + any collision and cleanup verifies every discovered resource before `down --volumes`. +- Added a dynamic fake-Docker contract suite for caller override, collision, and mismatched cleanup + labels, plus a real-Docker collision probe using a unique labeled volume. + +Follow-up verification: + +- `./scripts/local-vector-smoke.sh`: PASS, including live secret rotation and persisted retrieval +- `./scripts/test-local-vector-smoke-safety.sh`: PASS +- `./scripts/test-local-vector-smoke-live-collision.sh`: PASS +- harness: 477 passed, 5 deselected +- backend: 84 passed; TypeScript typecheck PASS +- frontend: 226 passed; TypeScript typecheck PASS +- Compose security, backend URL, config, shell syntax, and diff checks: PASS + +Remaining operational constraint: the bootstrap admin secret must continue to match the PostgreSQL +bootstrap account stored in the volume. Runtime migrator/reader/writer rotation is supported without +data deletion; bootstrap-account password rotation is a distinct database-administration operation. diff --git a/compose.yaml b/compose.yaml index 2f9e424b..e0b15bf9 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,5 +1,8 @@ name: thothii +x-smoke-labels: &smoke-labels + io.thothii.smoke-owner: "${THOTH_SMOKE_OWNER:-operator}" + services: core: image: thothii-core:local @@ -7,6 +10,7 @@ services: build: context: . dockerfile: docker/core.Dockerfile + labels: *smoke-labels environment: AUTH_MODE: "${AUTH_MODE:-none}" THOTH_PUBLIC_EXPOSURE: "${THOTH_PUBLIC_EXPOSURE:-false}" @@ -37,6 +41,7 @@ services: build: context: . dockerfile: docker/frontend.Dockerfile + labels: *smoke-labels environment: BACKEND_BASE_URL: /api ports: @@ -55,6 +60,7 @@ services: vector-db: image: pgvector/pgvector:0.8.5-pg16@sha256:1d533553fefe4f12e5d80c7b80622ba0c382abb5758856f52983d8789179f0fb profiles: [local-vector] + labels: *smoke-labels environment: POSTGRES_DB: "${THT_VECTOR_DATABASE:-thoth}" POSTGRES_USER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}" @@ -69,7 +75,6 @@ services: - vector_writer_password volumes: - vector_data:/var/lib/postgresql/data - - ./deploy/vector/init/00-bootstrap.sh:/docker-entrypoint-initdb.d/00-bootstrap.sh:ro healthcheck: test: [CMD-SHELL, "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] interval: 5s @@ -78,12 +83,38 @@ services: start_period: 10s restart: unless-stopped + vector-reconcile: + image: pgvector/pgvector:0.8.5-pg16@sha256:1d533553fefe4f12e5d80c7b80622ba0c382abb5758856f52983d8789179f0fb + profiles: [local-vector] + labels: *smoke-labels + environment: + PGHOST: vector-db + PGPORT: 5432 + PGDATABASE: "${THT_VECTOR_DATABASE:-thoth}" + PGUSER: "${THT_VECTOR_BOOTSTRAP_USER:-postgres}" + THT_VECTOR_MIGRATOR_USER: "${THT_VECTOR_MIGRATOR_USER:-thoth_vector_migrator}" + THT_VECTOR_READER_USER: "${THT_VECTOR_READER_USER:-thoth_vector_reader}" + THT_VECTOR_WRITER_USER: "${THT_VECTOR_WRITER_USER:-thoth_vector_writer}" + entrypoint: [/opt/thoth/reconcile-roles.sh] + secrets: + - vector_bootstrap_password + - vector_migrator_password + - vector_reader_password + - vector_writer_password + volumes: + - ./deploy/vector/reconcile-roles.sh:/opt/thoth/reconcile-roles.sh:ro + depends_on: + vector-db: + condition: service_healthy + restart: "no" + vector-migrate: image: thothii-core:local profiles: [local-vector] build: context: . dockerfile: docker/core.Dockerfile + labels: *smoke-labels entrypoint: [sh, -ec] command: - | @@ -95,13 +126,19 @@ services: secrets: - vector_migrator_password depends_on: - vector-db: - condition: service_healthy + vector-reconcile: + condition: service_completed_successfully restart: "no" volumes: thoth_data: + labels: *smoke-labels vector_data: + labels: *smoke-labels + +networks: + default: + labels: *smoke-labels secrets: vector_bootstrap_password: diff --git a/deploy/vector/init/00-bootstrap.sh b/deploy/vector/init/00-bootstrap.sh deleted file mode 100755 index 44f1fb33..00000000 --- a/deploy/vector/init/00-bootstrap.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh -set -eu - -read_secret() { - value=$(cat "/run/secrets/$1") - if [ -z "$value" ] || printf '%s' "$value" | grep -q '[[:space:]]'; then - echo "$1 must be non-empty and contain no whitespace" >&2 - exit 2 - fi - printf '%s' "$value" -} - -migrator_password=$(read_secret vector_migrator_password) -reader_password=$(read_secret vector_reader_password) -writer_password=$(read_secret vector_writer_password) - -psql --set=ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" \ - --set=migrator_user="$THT_VECTOR_MIGRATOR_USER" \ - --set=migrator_password="$migrator_password" \ - --set=reader_user="$THT_VECTOR_READER_USER" \ - --set=reader_password="$reader_password" \ - --set=writer_user="$THT_VECTOR_WRITER_USER" \ - --set=writer_password="$writer_password" <<'SQL' -CREATE ROLE vector_reader NOLOGIN; -CREATE ROLE vector_writer NOLOGIN; - -SELECT format('CREATE ROLE %I LOGIN PASSWORD %L CREATEROLE', :'migrator_user', :'migrator_password') \gexec -SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'reader_user', :'reader_password') \gexec -SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'writer_user', :'writer_password') \gexec -SELECT format('GRANT vector_reader TO %I', :'reader_user') \gexec -SELECT format('GRANT vector_writer TO %I', :'writer_user') \gexec -SELECT format('ALTER DATABASE %I OWNER TO %I', current_database(), :'migrator_user') \gexec -SELECT format('CREATE SCHEMA vectors AUTHORIZATION %I', :'migrator_user') \gexec -REVOKE ALL ON SCHEMA vectors FROM PUBLIC; -CREATE EXTENSION vector WITH SCHEMA vectors; -SQL diff --git a/deploy/vector/reconcile-roles.sh b/deploy/vector/reconcile-roles.sh new file mode 100755 index 00000000..ed8564a1 --- /dev/null +++ b/deploy/vector/reconcile-roles.sh @@ -0,0 +1,59 @@ +#!/bin/sh +set -eu + +read_secret() { + value=$(cat "/run/secrets/$1") + if [ -z "$value" ] || printf '%s' "$value" | grep -q '[[:space:]]'; then + echo "$1 must be non-empty and contain no whitespace" >&2 + exit 2 + fi + printf '%s' "$value" +} + +export PGPASSWORD=$(read_secret vector_bootstrap_password) +migrator_password=$(read_secret vector_migrator_password) +reader_password=$(read_secret vector_reader_password) +writer_password=$(read_secret vector_writer_password) + +psql --set=ON_ERROR_STOP=1 \ + --set=migrator_user="$THT_VECTOR_MIGRATOR_USER" \ + --set=migrator_password="$migrator_password" \ + --set=reader_user="$THT_VECTOR_READER_USER" \ + --set=reader_password="$reader_password" \ + --set=writer_user="$THT_VECTOR_WRITER_USER" \ + --set=writer_password="$writer_password" <<'SQL' +SELECT 'CREATE ROLE vector_reader NOLOGIN' +WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'vector_reader') \gexec +SELECT 'CREATE ROLE vector_writer NOLOGIN' +WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'vector_writer') \gexec +ALTER ROLE vector_reader NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; +ALTER ROLE vector_writer NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; + +SELECT format('CREATE ROLE %I LOGIN', :'migrator_user') +WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'migrator_user') \gexec +SELECT format('CREATE ROLE %I LOGIN', :'reader_user') +WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'reader_user') \gexec +SELECT format('CREATE ROLE %I LOGIN', :'writer_user') +WHERE NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'writer_user') \gexec + +SELECT format( + 'ALTER ROLE %I LOGIN PASSWORD %L NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION', + :'migrator_user', :'migrator_password' +) \gexec +SELECT format( + 'ALTER ROLE %I LOGIN PASSWORD %L NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION', + :'reader_user', :'reader_password' +) \gexec +SELECT format( + 'ALTER ROLE %I LOGIN PASSWORD %L NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION', + :'writer_user', :'writer_password' +) \gexec + +SELECT format('GRANT vector_reader TO %I', :'reader_user') \gexec +SELECT format('GRANT vector_writer TO %I', :'writer_user') \gexec +SELECT format('ALTER DATABASE %I OWNER TO %I', current_database(), :'migrator_user') \gexec +SELECT format('CREATE SCHEMA IF NOT EXISTS vectors AUTHORIZATION %I', :'migrator_user') \gexec +SELECT format('ALTER SCHEMA vectors OWNER TO %I', :'migrator_user') \gexec +REVOKE ALL ON SCHEMA vectors FROM PUBLIC; +CREATE EXTENSION IF NOT EXISTS vector WITH SCHEMA vectors; +SQL diff --git a/scripts/local-vector-smoke.sh b/scripts/local-vector-smoke.sh index ca334542..92767eb5 100755 --- a/scripts/local-vector-smoke.sh +++ b/scripts/local-vector-smoke.sh @@ -3,22 +3,23 @@ set -eu cd "$(dirname "$0")/.." -smoke_project=${SMOKE_PROJECT:-"thothii-vector-smoke-$(date +%s)-$$"} -keep_resources=${KEEP_SMOKE_RESOURCES:-0} -secret_dir=$(mktemp -d) -marker="local-vector-$smoke_project" - -case "$smoke_project" in - thothii) - echo "SMOKE_PROJECT=thothii is reserved for the operator stack" >&2 - exit 2 - ;; - ""|*[!a-z0-9_-]*|[!a-z0-9]*) - echo "invalid SMOKE_PROJECT: use lowercase letters, digits, hyphens, or underscores" >&2 - exit 2 - ;; +mode=${1:-run} +case "$mode" in + run|--live-collision-test) ;; + *) echo "usage: $0 [--live-collision-test]" >&2; exit 2 ;; esac +keep_resources=${KEEP_SMOKE_RESOURCES:-0} +if [ "${SMOKE_PROJECT+x}" = x ]; then + echo "SMOKE_PROJECT is not accepted; the smoke always generates an owned namespace" >&2 + exit 2 +fi +secret_dir=$(mktemp -d "${TMPDIR:-/tmp}/thothii-vector-smoke.XXXXXX") +suffix=$(basename "$secret_dir" | tr -cd 'a-z0-9') +smoke_project="thothii-vector-smoke-$(date +%s)-$$-$suffix" +smoke_owner="$smoke_project-owner" +marker="local-vector-$smoke_project" + for secret in bootstrap migrator reader writer; do password="smoke-${secret}-${smoke_project}" printf '%s' "$password" >"$secret_dir/$secret" @@ -31,23 +32,80 @@ export THT_VECTOR_READER_PASSWORD_SECRET_FILE="$secret_dir/reader" export THT_VECTOR_WRITER_PASSWORD_SECRET_FILE="$secret_dir/writer" export THT_VECTOR_READER_PASSWORD="smoke-reader-${smoke_project}" export THT_VECTOR_WRITER_PASSWORD="smoke-writer-${smoke_project}" +export THOTH_SMOKE_OWNER="$smoke_owner" compose() { docker compose --project-name "$smoke_project" --profile local-vector "$@" } +resource_ids() { + case "$1" in + container) docker ps -aq --filter "label=com.docker.compose.project=$smoke_project" ;; + volume) docker volume ls -q --filter "label=com.docker.compose.project=$smoke_project" ;; + network) docker network ls -q --filter "label=com.docker.compose.project=$smoke_project" ;; + esac +} + +resource_owner() { + case "$1" in + container) docker inspect --format '{{ index .Config.Labels "io.thothii.smoke-owner" }}' "$2" ;; + volume) docker volume inspect --format '{{ index .Labels "io.thothii.smoke-owner" }}' "$2" ;; + network) docker network inspect --format '{{ index .Labels "io.thothii.smoke-owner" }}' "$2" ;; + esac +} + +assert_no_collision() { + for kind in container volume network; do + ids=$(resource_ids "$kind") + if [ -n "$ids" ]; then + echo "refusing existing Compose project resources for generated namespace $smoke_project" >&2 + return 1 + fi + done +} + +verify_owned_resources() { + for kind in container volume network; do + for id in $(resource_ids "$kind"); do + owner=$(resource_owner "$kind" "$id" 2>/dev/null || true) + if [ "$owner" != "$smoke_owner" ]; then + echo "refusing cleanup of resource not owned by this smoke: $kind $id" >&2 + return 1 + fi + done + done +} + cleanup() { if [ "$keep_resources" = "1" ]; then echo "Keeping smoke resources for project $smoke_project (KEEP_SMOKE_RESOURCES=1)." >&2 else - compose down --volumes >/dev/null 2>&1 || true + if verify_owned_resources; then + compose down --volumes >/dev/null 2>&1 || true + fi fi rm -rf "$secret_dir" } trap cleanup EXIT HUP INT TERM +if [ "$mode" = "--live-collision-test" ]; then + collision_volume="${smoke_project}-collision" + docker volume create \ + --label "com.docker.compose.project=$smoke_project" \ + --label 'io.thothii.smoke-owner=foreign-owner' \ + "$collision_volume" >/dev/null + if assert_no_collision 2>/dev/null; then + echo "live collision probe was not detected" >&2 + docker volume rm "$collision_volume" >/dev/null + exit 1 + fi + docker volume rm "$collision_volume" >/dev/null + echo "live local-vector project collision refusal passed." + exit 0 +fi + probe_vector() { - compose exec -T core /opt/venv/bin/python - "$marker" <<'PY' + compose exec -T core /opt/venv/bin/python - "$marker" "$1" <<'PY' import hashlib import os import sys @@ -58,6 +116,7 @@ from tht.ports.vector import VectorWriteRecord from tht.vectorstore.records import VectorRecord marker = sys.argv[1] +mode = sys.argv[2] database = "thoth" host = "vector-db" @@ -85,28 +144,59 @@ record = VectorRecord( metadata={"smoke": True}, ) embedding = [1.0] + [0.0] * 767 -store.upsert( - "memory", - [VectorWriteRecord(record, embedding, hashlib.sha256(marker.encode()).hexdigest())], -) +if mode == "write": + store.upsert( + "memory", + [VectorWriteRecord(record, embedding, hashlib.sha256(marker.encode()).hexdigest())], + ) hits = store.search(["memory"], embedding, limit=1, kinds=["memory"]) assert hits and hits[0].id == marker, hits -print(f"role health, upsert, and search passed for {marker}") +print(f"role health and persisted search passed for {marker} ({mode})") PY } +assert_no_collision compose config --quiet services=$(compose config --services) printf '%s\n' "$services" | grep -qx vector-db +printf '%s\n' "$services" | grep -qx vector-reconcile printf '%s\n' "$services" | grep -qx vector-migrate -compose up --build --wait vector-migrate core +compose up --build --wait vector-reconcile vector-migrate core migration_status=$(compose run --rm --no-deps vector-migrate) printf '%s\n' "$migration_status" | grep -q '"pending": \[\]' -probe_vector +migrator_flags=$(compose run --rm --no-deps --entrypoint sh vector-reconcile -ec ' + export PGPASSWORD=$(cat /run/secrets/vector_bootstrap_password) + psql -At --host vector-db --username postgres --dbname thoth \ + --command "SELECT (NOT rolcreaterole) AND (NOT rolcreatedb) AND (NOT rolsuper) FROM pg_roles WHERE rolname = '\''thoth_vector_migrator'\''" +') +test "$migrator_flags" = t +probe_vector write + +old_reader_password=$THT_VECTOR_READER_PASSWORD +for secret in migrator reader writer; do + password="rotated-${secret}-${smoke_project}" + printf '%s' "$password" >"$secret_dir/$secret" +done +export THT_VECTOR_READER_PASSWORD="rotated-reader-${smoke_project}" +export THT_VECTOR_WRITER_PASSWORD="rotated-writer-${smoke_project}" + +compose run --rm vector-reconcile +rotation_status=$(compose run --rm --no-deps vector-migrate) +printf '%s\n' "$rotation_status" | grep -q '"pending": \[\]' +if compose run --rm --no-deps --entrypoint psql \ + -e PGPASSWORD="$old_reader_password" vector-reconcile \ + --host vector-db --username thoth_vector_reader --dbname thoth --command 'SELECT 1' \ + >/dev/null 2>&1; then + echo "old reader credential still works after rotation" >&2 + exit 1 +fi + +compose up --force-recreate --no-deps --wait core +probe_vector read compose restart vector-db core compose up --wait vector-db core -probe_vector +probe_vector read -echo "Local pgvector migration, least-privilege roles, search, and restart persistence passed." +echo "Local pgvector migration, credential rotation, least-privilege roles, and persistence passed." diff --git a/scripts/test-local-vector-smoke-live-collision.sh b/scripts/test-local-vector-smoke-live-collision.sh new file mode 100755 index 00000000..c7e047ad --- /dev/null +++ b/scripts/test-local-vector-smoke-live-collision.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." +./scripts/local-vector-smoke.sh --live-collision-test diff --git a/scripts/test-local-vector-smoke-safety.sh b/scripts/test-local-vector-smoke-safety.sh new file mode 100755 index 00000000..f4bd4c1c --- /dev/null +++ b/scripts/test-local-vector-smoke-safety.sh @@ -0,0 +1,71 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT HUP INT TERM + +fake="$tmp/docker" +log="$tmp/docker.log" +state="$tmp/state" + +cat >"$fake" <<'SH' +#!/bin/sh +set -eu +printf '%s\n' "$*" >>"$FAKE_DOCKER_LOG" + +if [ "${FAKE_COLLISION:-0}" = 1 ] && [ "$1 $2" = "ps -aq" ]; then + printf '%s\n' collision-container + exit 0 +fi + +if [ "$1 $2" = "ps -aq" ] || [ "$1 $2" = "volume ls" ] || [ "$1 $2" = "network ls" ]; then + if [ "${FAKE_MISMATCH_ON_CLEANUP:-0}" = 1 ] && [ -f "$FAKE_DOCKER_STATE" ]; then + printf '%s\n' foreign-resource + fi + : >"$FAKE_DOCKER_STATE" + exit 0 +fi + +if [ "$1" = inspect ] || [ "$1 $2" = "volume inspect" ] || [ "$1 $2" = "network inspect" ]; then + printf '%s\n' foreign-owner + exit 0 +fi + +case "$*" in + *"config --services"*) printf '%s\n' vector-db vector-reconcile vector-migrate core frontend ;; + *"run --rm --no-deps vector-migrate"*) printf '%s\n' '{"applied":["001","002","003"],"drifted":[],"pending":[]}' ;; +esac +exit 0 +SH +chmod 0755 "$fake" + +if PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" FAKE_DOCKER_STATE="$state" \ + SMOKE_PROJECT=operator-owned ./scripts/local-vector-smoke.sh >"$tmp/out" 2>"$tmp/err"; then + echo "smoke accepted caller-controlled SMOKE_PROJECT" >&2 + exit 1 +fi +grep -q 'SMOKE_PROJECT is not accepted' "$tmp/err" +test ! -s "$log" + +: >"$log" +rm -f "$state" +PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" FAKE_DOCKER_STATE="$state" \ + FAKE_COLLISION=1 ./scripts/local-vector-smoke.sh >"$tmp/out" 2>"$tmp/err" || true +grep -q 'refusing existing Compose project resources' "$tmp/err" +if grep -q 'compose.*up' "$log"; then + echo "smoke started after detecting a project collision" >&2 + exit 1 +fi + +: >"$log" +rm -f "$state" +PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" FAKE_DOCKER_STATE="$state" \ + FAKE_MISMATCH_ON_CLEANUP=1 ./scripts/local-vector-smoke.sh >"$tmp/out" 2>"$tmp/err" || true +grep -q 'refusing cleanup of resource not owned by this smoke' "$tmp/err" +if grep -q 'down --volumes' "$log"; then + echo "smoke removed resources after ownership mismatch" >&2 + exit 1 +fi + +echo "local-vector smoke collision and cleanup ownership contracts passed."