#!/bin/sh set -eu cd "$(dirname "$0")/.." mode=${1:-run} case "$mode" in run|--live-collision-test|--backup-restore) ;; *) echo "usage: $0 [--live-collision-test|--backup-restore]" >&2; exit 2 ;; esac keep_resources=${KEEP_SMOKE_RESOURCES:-0} if [ "${SMOKE_PROJECT+x}" = x ]; then echo "SMOKE_PROJECT is not accepted; the smoke always generates an owned namespace" >&2 exit 2 fi secret_dir=$(mktemp -d "${TMPDIR:-/tmp}/thothii-vector-smoke.XXXXXX") suffix=$(basename "$secret_dir" | tr -cd 'a-z0-9') smoke_project="thothii-vector-smoke-$(date +%s)-$$-$suffix" smoke_owner="$smoke_project-owner" marker="local-vector-$smoke_project" restore_container="${smoke_project}-restore" restore_volume="${smoke_project}-restore-data" for secret in bootstrap migrator reader writer; do password="smoke-${secret}-${smoke_project}" printf '%s' "$password" >"$secret_dir/$secret" chmod 0600 "$secret_dir/$secret" done export THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE="$secret_dir/bootstrap" export THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE="$secret_dir/migrator" export THT_VECTOR_READER_PASSWORD_SECRET_FILE="$secret_dir/reader" export THT_VECTOR_WRITER_PASSWORD_SECRET_FILE="$secret_dir/writer" export THT_VECTOR_BOOTSTRAP_USER=thoth_bootstrap_smoke export THOTH_SMOKE_OWNER="$smoke_owner" compose() { docker compose -f compose.yaml -f deploy/compose.local-vector.yaml \ --project-name "$smoke_project" --profile local-vector "$@" } resource_ids() { case "$1" in container) docker ps -aq --filter "label=com.docker.compose.project=$smoke_project" ;; volume) docker volume ls -q --filter "label=com.docker.compose.project=$smoke_project" ;; network) docker network ls -q --filter "label=com.docker.compose.project=$smoke_project" ;; esac } resource_owner() { case "$1" in container) docker inspect --format '{{ index .Config.Labels "io.thothii.smoke-owner" }}' "$2" ;; volume) docker volume inspect --format '{{ index .Labels "io.thothii.smoke-owner" }}' "$2" ;; network) docker network inspect --format '{{ index .Labels "io.thothii.smoke-owner" }}' "$2" ;; esac } assert_no_collision() { for kind in container volume network; do ids=$(resource_ids "$kind") if [ -n "$ids" ]; then echo "refusing existing Compose project resources for generated namespace $smoke_project" >&2 return 1 fi done } verify_owned_resources() { for kind in container volume network; do for id in $(resource_ids "$kind"); do owner=$(resource_owner "$kind" "$id" 2>/dev/null || true) if [ "$owner" != "$smoke_owner" ]; then echo "refusing cleanup of resource not owned by this smoke: $kind $id" >&2 return 1 fi done done } cleanup() { if [ "$keep_resources" = "1" ]; then echo "Keeping smoke resources for project $smoke_project (KEEP_SMOKE_RESOURCES=1)." >&2 else if verify_owned_resources; then docker rm -f "$restore_container" >/dev/null 2>&1 || true docker volume rm "$restore_volume" >/dev/null 2>&1 || true compose down --volumes >/dev/null 2>&1 || true fi fi rm -rf "$secret_dir" } trap cleanup EXIT HUP INT TERM if [ "$mode" = "--live-collision-test" ]; then collision_volume="${smoke_project}-collision" docker volume create \ --label "com.docker.compose.project=$smoke_project" \ --label 'io.thothii.smoke-owner=foreign-owner' \ "$collision_volume" >/dev/null if assert_no_collision 2>/dev/null; then echo "live collision probe was not detected" >&2 docker volume rm "$collision_volume" >/dev/null exit 1 fi docker volume rm "$collision_volume" >/dev/null echo "live local-vector project collision refusal passed." exit 0 fi probe_vector() { compose exec -T core /opt/venv/bin/python - "$marker" "$1" <<'PY' import hashlib import os import sys from tht.adapters.vector.pgvector import PgVectorStore from tht.config import DatabaseConfig from tht.ports.vector import VectorWriteRecord from tht.vectorstore.records import VectorRecord marker = sys.argv[1] mode = sys.argv[2] database = "thoth" host = "vector-db" def credential(role: str) -> DatabaseConfig: return DatabaseConfig( host=host, port=5432, database=database, schema="vectors", user=f"thoth_vector_{role}", password=open(os.environ[f"THT_VECTOR_{role.upper()}_PASSWORD_FILE"]).read(), ) store = PgVectorStore(credential("reader"), credential("writer"), expected_dimension=768) health = store.health() assert health.ok, health assert health.read_reachable is True and health.write_reachable is True, health record = VectorRecord( id=marker, kind="memory", ref=marker, title="Local vector persistence smoke", content=marker, metadata={"smoke": True}, ) embedding = [1.0] + [0.0] * 767 if mode == "write": store.upsert( "memory", [VectorWriteRecord(record, embedding, hashlib.sha256(marker.encode()).hexdigest())], ) hits = store.search(["memory"], embedding, limit=1, kinds=["memory"]) assert hits and hits[0].id == marker, hits print(f"role health and persisted search passed for {marker} ({mode})") PY } assert_no_collision compose config --quiet services=$(compose config --services) printf '%s\n' "$services" | grep -qx vector-db printf '%s\n' "$services" | grep -qx vector-reconcile printf '%s\n' "$services" | grep -qx vector-migrate compose up --build --wait vector-reconcile vector-migrate core core_id=$(compose ps -q core) inspect_env=$(docker inspect --format '{{json .Config.Env}}' "$core_id") if printf '%s' "$inspect_env" | grep -q "smoke-\(reader\|writer\)-${smoke_project}"; then echo "docker inspect exposed a direct vector password" >&2 exit 1 fi printf '%s' "$inspect_env" | grep -q 'THT_VECTOR_READER_PASSWORD_FILE=/run/secrets/vector_reader_password' printf '%s' "$inspect_env" | grep -q 'THT_VECTOR_WRITER_PASSWORD_FILE=/run/secrets/vector_writer_password' migration_status=$(compose run --rm --no-deps vector-migrate) printf '%s\n' "$migration_status" | grep -q '"pending": \[\]' migrator_flags=$(compose run --rm --no-deps --entrypoint sh vector-reconcile -ec ' export PGPASSWORD=$(cat /run/secrets/vector_bootstrap_password) psql -At --host vector-db --username "$THT_VECTOR_BOOTSTRAP_USER" --dbname thoth \ --command "SELECT (NOT rolcreaterole) AND (NOT rolcreatedb) AND (NOT rolsuper) FROM pg_roles WHERE rolname = '\''thoth_vector_migrator'\''" ') test "$migrator_flags" = t probe_vector write old_reader_password=$(cat "$secret_dir/reader") for secret in migrator reader writer; do password="rotated-${secret}-${smoke_project}" printf '%s' "$password" >"$secret_dir/$secret" done compose run --rm vector-reconcile rotation_status=$(compose run --rm --no-deps vector-migrate) printf '%s\n' "$rotation_status" | grep -q '"pending": \[\]' if compose run --rm --no-deps --entrypoint psql \ -e PGPASSWORD="$old_reader_password" vector-reconcile \ --host vector-db --username thoth_vector_reader --dbname thoth --command 'SELECT 1' \ >/dev/null 2>&1; then echo "old reader credential still works after rotation" >&2 exit 1 fi compose up --force-recreate --no-deps --wait core probe_vector read old_bootstrap_password=$(cat "$secret_dir/bootstrap") printf '%s' "wrong-bootstrap-${smoke_project}" >"$secret_dir/bootstrap-wrong" printf '%s' "next-bootstrap-'quoted-${smoke_project}" >"$secret_dir/bootstrap-next" cp "$secret_dir/bootstrap" "$secret_dir/bootstrap-before-negative" printf 'invalid bootstrap password\n' >"$secret_dir/bootstrap-whitespace" chmod 0600 "$secret_dir/bootstrap-wrong" "$secret_dir/bootstrap-next" \ "$secret_dir/bootstrap-before-negative" "$secret_dir/bootstrap-whitespace" if COMPOSE_PROJECT_NAME="$smoke_project" \ ./scripts/vector-rotate-bootstrap-password.sh \ "$secret_dir/bootstrap" "$secret_dir/bootstrap-whitespace" \ >/dev/null 2>&1; then echo "bootstrap rotation accepted whitespace in a secret" >&2 exit 1 fi cmp "$secret_dir/bootstrap" "$secret_dir/bootstrap-before-negative" compose run --rm --no-deps --entrypoint psql \ -e PGPASSWORD="$old_bootstrap_password" vector-reconcile \ --host vector-db --username "$THT_VECTOR_BOOTSTRAP_USER" --dbname thoth \ --command 'SELECT 1' >/dev/null if COMPOSE_PROJECT_NAME="$smoke_project" \ ./scripts/vector-rotate-bootstrap-password.sh \ "$secret_dir/bootstrap-wrong" "$secret_dir/bootstrap-next" \ >/dev/null 2>&1; then echo "bootstrap rotation accepted the wrong old secret" >&2 exit 1 fi cmp "$secret_dir/bootstrap" "$secret_dir/bootstrap-before-negative" COMPOSE_PROJECT_NAME="$smoke_project" \ ./scripts/vector-rotate-bootstrap-password.sh \ "$secret_dir/bootstrap" "$secret_dir/bootstrap-next" new_bootstrap_password=$(cat "$secret_dir/bootstrap") test "$new_bootstrap_password" != "$old_bootstrap_password" if compose run --rm --no-deps --entrypoint psql \ -e PGPASSWORD="$old_bootstrap_password" vector-reconcile \ --host vector-db --username "$THT_VECTOR_BOOTSTRAP_USER" --dbname thoth --command 'SELECT 1' \ >/dev/null 2>&1; then echo "old bootstrap credential still works after rotation" >&2 exit 1 fi compose run --rm --no-deps --entrypoint psql \ -e PGPASSWORD="$new_bootstrap_password" vector-reconcile \ --host vector-db --username "$THT_VECTOR_BOOTSTRAP_USER" --dbname thoth --command 'SELECT 1' \ >/dev/null compose run --rm vector-reconcile bootstrap_rotation_status=$(compose run --rm --no-deps vector-migrate) printf '%s\n' "$bootstrap_rotation_status" | grep -q '"pending": \[\]' compose up --force-recreate --no-deps --wait core probe_vector read compose restart vector-db core compose up --wait vector-db core probe_vector read if [ "$mode" = "--backup-restore" ]; then image=$(compose images -q vector-db) network="${smoke_project}_default" docker volume create \ --label "com.docker.compose.project=$smoke_project" \ --label "io.thothii.smoke-owner=$smoke_owner" "$restore_volume" >/dev/null docker run -d --name "$restore_container" \ --label "com.docker.compose.project=$smoke_project" \ --label "io.thothii.smoke-owner=$smoke_owner" \ --network "$network" --network-alias vector-db-restore \ --mount "type=volume,source=$restore_volume,target=/var/lib/postgresql/data" \ --mount "type=bind,source=$secret_dir/bootstrap,target=/run/secrets/bootstrap,readonly" \ -e POSTGRES_DB=thoth -e POSTGRES_USER="$THT_VECTOR_BOOTSTRAP_USER" \ -e POSTGRES_PASSWORD_FILE=/run/secrets/bootstrap "$image" >/dev/null attempts=0 until docker exec "$restore_container" pg_isready \ -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth >/dev/null 2>&1; do attempts=$((attempts + 1)) [ "$attempts" -lt 30 ] || { echo "restore database did not become ready" >&2; exit 1; } sleep 1 done docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql -X \ -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth -v ON_ERROR_STOP=1 --command \ "CREATE SCHEMA vectors; CREATE EXTENSION vector WITH SCHEMA vectors; CREATE TABLE vectors.memory ( id bigserial PRIMARY KEY, record_key text UNIQUE NOT NULL, kind text NOT NULL, content_hash text NOT NULL, metadata jsonb NOT NULL, embedding vectors.vector(768) NOT NULL, indexed_at timestamptz NOT NULL DEFAULT now()); INSERT INTO vectors.memory (record_key, kind, content_hash, metadata, embedding) VALUES ('restore-sentinel', 'memory', 'sentinel-original', '{}', ('[' || '1,' || repeat('0,', 766) || '0]')::vectors.vector);" >/dev/null docker run --rm --network "$network" \ --mount "type=bind,source=$(pwd),target=/repo,readonly" \ --mount "type=bind,source=$secret_dir,target=/scratch" "$image" \ /repo/scripts/vector-backup.sh --host vector-db --database thoth \ --user "$THT_VECTOR_BOOTSTRAP_USER" --password-file /scratch/bootstrap \ --output /scratch/vector.dump compose exec -T vector-db sh -ec ' export PGPASSWORD=$(cat /run/secrets/vector_bootstrap_password) psql -X -U "$POSTGRES_USER" -d thoth -v ON_ERROR_STOP=1 --command \ "UPDATE vectors.memory SET content_hash = '\''mutated-after-backup'\'' WHERE record_key = '\''$1'\''"' \ sh "$marker" >/dev/null if docker run --rm --network "$network" \ --mount "type=bind,source=$(pwd),target=/repo,readonly" \ --mount "type=bind,source=$secret_dir,target=/scratch" "$image" \ /repo/scripts/vector-restore.sh \ --active-host vector-db --active-database thoth --active-user "$THT_VECTOR_BOOTSTRAP_USER" \ --active-password-file /scratch/bootstrap \ --target-host vector-db-restore --target-database thoth \ --target-user "$THT_VECTOR_BOOTSTRAP_USER" --target-password-file /scratch/bootstrap \ --input /scratch/vector.dump --force-nonempty >/dev/null 2>&1; then echo "forced restore unexpectedly succeeded without archived ACL roles" >&2 exit 1 fi sentinel=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \ -XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \ "SELECT content_hash FROM vectors.memory WHERE record_key='restore-sentinel'") test "$sentinel" = sentinel-original docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql -X \ -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth -v ON_ERROR_STOP=1 --command \ "DROP TABLE vectors.memory; CREATE ROLE vector_reader NOLOGIN; CREATE ROLE vector_writer NOLOGIN;" \ >/dev/null docker run --rm --network "$network" \ --mount "type=bind,source=$(pwd),target=/repo,readonly" \ --mount "type=bind,source=$secret_dir,target=/scratch" "$image" \ /repo/scripts/vector-restore.sh \ --active-host vector-db --active-database thoth --active-user "$THT_VECTOR_BOOTSTRAP_USER" \ --active-password-file /scratch/bootstrap \ --target-host vector-db-restore --target-database thoth \ --target-user "$THT_VECTOR_BOOTSTRAP_USER" --target-password-file /scratch/bootstrap \ --input /scratch/vector.dump docker run --rm --network "$network" \ --mount "type=bind,source=$(pwd)/deploy/vector/reconcile-roles.sh,target=/opt/thoth/reconcile-roles.sh,readonly" \ --mount "type=bind,source=$(pwd)/deploy/vector/secret-policy.sh,target=/opt/thoth/secret-policy.sh,readonly" \ --mount "type=bind,source=$secret_dir/bootstrap,target=/run/secrets/vector_bootstrap_password,readonly" \ --mount "type=bind,source=$secret_dir/migrator,target=/run/secrets/vector_migrator_password,readonly" \ --mount "type=bind,source=$secret_dir/reader,target=/run/secrets/vector_reader_password,readonly" \ --mount "type=bind,source=$secret_dir/writer,target=/run/secrets/vector_writer_password,readonly" \ -e PGHOST=vector-db-restore -e PGDATABASE=thoth \ -e PGUSER="$THT_VECTOR_BOOTSTRAP_USER" \ -e THT_VECTOR_MIGRATOR_USER=thoth_vector_migrator \ -e THT_VECTOR_READER_USER=thoth_vector_reader \ -e THT_VECTOR_WRITER_USER=thoth_vector_writer \ --entrypoint /opt/thoth/reconcile-roles.sh "$image" >/dev/null docker run --rm -i --network "$network" \ --mount "type=bind,source=$secret_dir/reader,target=/run/secrets/vector_reader_password,readonly" \ --mount "type=bind,source=$secret_dir/writer,target=/run/secrets/vector_writer_password,readonly" \ -e THT_VECTOR_READER_PASSWORD_FILE=/run/secrets/vector_reader_password \ -e THT_VECTOR_WRITER_PASSWORD_FILE=/run/secrets/vector_writer_password \ --entrypoint /opt/venv/bin/python thothii-core:local - "$marker" <<'PY' import hashlib import os import sys from tht.adapters.vector.pgvector import PgVectorStore from tht.config import DatabaseConfig from tht.ports.vector import VectorWriteRecord from tht.vectorstore.records import VectorRecord def config(role): return DatabaseConfig( host="vector-db-restore", port=5432, database="thoth", schema="vectors", user=f"thoth_vector_{role}", password=open(os.environ[f"THT_VECTOR_{role.upper()}_PASSWORD_FILE"]).read(), ) store = PgVectorStore(config("reader"), config("writer"), expected_dimension=768) assert store.health().ok, store.health() embedding = [1.0] + [0.0] * 767 marker = sys.argv[1] assert store.search(["memory"], embedding, limit=1, kinds=["memory"])[0].id == marker write_id = marker + "-restore-write" record = VectorRecord( id=write_id, kind="memory", ref=write_id, title="restore writer", content=write_id, metadata={}, ) store.upsert("memory", [VectorWriteRecord(record, embedding, hashlib.sha256(write_id.encode()).hexdigest())]) assert store.existing_hashes("memory", ["memory"])[write_id] PY restored=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \ -XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \ "SELECT content_hash <> 'mutated-after-backup' FROM vectors.memory WHERE record_key = '$marker'") test "$restored" = t pending=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \ -XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \ "SELECT count(*) = 3 FROM public.tht_vector_migrations") test "$pending" = t dimensions=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \ -XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \ "SELECT count(*) = 3 FROM pg_attribute a JOIN pg_class c ON c.oid=a.attrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='vectors' AND a.attname='embedding' AND format_type(a.atttypid,a.atttypmod)='vectors.vector(768)'") test "$dimensions" = t echo "Transactional rollback and disposable-volume restore adapter parity passed." fi echo "Local pgvector runtime/bootstrap rotation, least-privilege roles, and persistence passed."