fix(vector): harden backup restore parity gates

2026-07-12 02:29:53 +02:00
parent e4db2ea5e1
commit 015c496bda
9 changed files with 300 additions and 36 deletions
@@ -277,7 +277,13 @@ if [ "$mode" = "--backup-restore" ]; then
  docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql -X \
    -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth -v ON_ERROR_STOP=1 --command \
    "CREATE SCHEMA vectors; CREATE EXTENSION vector WITH SCHEMA vectors;
-     CREATE ROLE vector_reader NOLOGIN; CREATE ROLE vector_writer NOLOGIN;" >/dev/null
+     CREATE TABLE vectors.memory (
+       id bigserial PRIMARY KEY, record_key text UNIQUE NOT NULL, kind text NOT NULL,
+       content_hash text NOT NULL, metadata jsonb NOT NULL,
+       embedding vectors.vector(768) NOT NULL, indexed_at timestamptz NOT NULL DEFAULT now());
+     INSERT INTO vectors.memory (record_key, kind, content_hash, metadata, embedding)
+       VALUES ('restore-sentinel', 'memory', 'sentinel-original', '{}',
+               ('[' || '1,' || repeat('0,', 766) || '0]')::vectors.vector);" >/dev/null

  docker run --rm --network "$network" \
    --mount "type=bind,source=$(pwd),target=/repo,readonly" \
@@ -292,6 +298,27 @@ if [ "$mode" = "--backup-restore" ]; then
      "UPDATE vectors.memory SET content_hash = '\''mutated-after-backup'\'' WHERE record_key = '\''$1'\''"' \
    sh "$marker" >/dev/null

+  if docker run --rm --network "$network" \
+      --mount "type=bind,source=$(pwd),target=/repo,readonly" \
+      --mount "type=bind,source=$secret_dir,target=/scratch" "$image" \
+      /repo/scripts/vector-restore.sh \
+      --active-host vector-db --active-database thoth --active-user "$THT_VECTOR_BOOTSTRAP_USER" \
+      --active-password-file /scratch/bootstrap \
+      --target-host vector-db-restore --target-database thoth \
+      --target-user "$THT_VECTOR_BOOTSTRAP_USER" --target-password-file /scratch/bootstrap \
+      --input /scratch/vector.dump --force-nonempty >/dev/null 2>&1; then
+    echo "forced restore unexpectedly succeeded without archived ACL roles" >&2
+    exit 1
+  fi
+  sentinel=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \
+    -XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \
+    "SELECT content_hash FROM vectors.memory WHERE record_key='restore-sentinel'")
+  test "$sentinel" = sentinel-original
+  docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql -X \
+    -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth -v ON_ERROR_STOP=1 --command \
+    "DROP TABLE vectors.memory; CREATE ROLE vector_reader NOLOGIN; CREATE ROLE vector_writer NOLOGIN;" \
+    >/dev/null
+
  docker run --rm --network "$network" \
    --mount "type=bind,source=$(pwd),target=/repo,readonly" \
    --mount "type=bind,source=$secret_dir,target=/scratch" "$image" \
@@ -302,6 +329,53 @@ if [ "$mode" = "--backup-restore" ]; then
    --target-user "$THT_VECTOR_BOOTSTRAP_USER" --target-password-file /scratch/bootstrap \
    --input /scratch/vector.dump

+  docker run --rm --network "$network" \
+    --mount "type=bind,source=$(pwd)/deploy/vector/reconcile-roles.sh,target=/opt/thoth/reconcile-roles.sh,readonly" \
+    --mount "type=bind,source=$(pwd)/deploy/vector/secret-policy.sh,target=/opt/thoth/secret-policy.sh,readonly" \
+    --mount "type=bind,source=$secret_dir/bootstrap,target=/run/secrets/vector_bootstrap_password,readonly" \
+    --mount "type=bind,source=$secret_dir/migrator,target=/run/secrets/vector_migrator_password,readonly" \
+    --mount "type=bind,source=$secret_dir/reader,target=/run/secrets/vector_reader_password,readonly" \
+    --mount "type=bind,source=$secret_dir/writer,target=/run/secrets/vector_writer_password,readonly" \
+    -e PGHOST=vector-db-restore -e PGDATABASE=thoth \
+    -e PGUSER="$THT_VECTOR_BOOTSTRAP_USER" \
+    -e THT_VECTOR_MIGRATOR_USER=thoth_vector_migrator \
+    -e THT_VECTOR_READER_USER=thoth_vector_reader \
+    -e THT_VECTOR_WRITER_USER=thoth_vector_writer \
+    --entrypoint /opt/thoth/reconcile-roles.sh "$image" >/dev/null
+
+  docker run --rm -i --network "$network" \
+    -e THT_VECTOR_READER_PASSWORD="$THT_VECTOR_READER_PASSWORD" \
+    -e THT_VECTOR_WRITER_PASSWORD="$THT_VECTOR_WRITER_PASSWORD" \
+    --entrypoint /opt/venv/bin/python thothii-core:local - "$marker" <<'PY'
+import hashlib
+import os
+import sys
+
+from tht.adapters.vector.pgvector import PgVectorStore
+from tht.config import DatabaseConfig
+from tht.ports.vector import VectorWriteRecord
+from tht.vectorstore.records import VectorRecord
+
+def config(role):
+    return DatabaseConfig(
+        host="vector-db-restore", port=5432, database="thoth", schema="vectors",
+        user=f"thoth_vector_{role}", password=os.environ[f"THT_VECTOR_{role.upper()}_PASSWORD"],
+    )
+
+store = PgVectorStore(config("reader"), config("writer"), expected_dimension=768)
+assert store.health().ok, store.health()
+embedding = [1.0] + [0.0] * 767
+marker = sys.argv[1]
+assert store.search(["memory"], embedding, limit=1, kinds=["memory"])[0].id == marker
+write_id = marker + "-restore-write"
+record = VectorRecord(
+    id=write_id, kind="memory", ref=write_id, title="restore writer",
+    content=write_id, metadata={},
+)
+store.upsert("memory", [VectorWriteRecord(record, embedding, hashlib.sha256(write_id.encode()).hexdigest())])
+assert store.existing_hashes("memory", ["memory"])[write_id]
+PY
+
  restored=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \
    -XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \
    "SELECT content_hash <> 'mutated-after-backup' FROM vectors.memory WHERE record_key = '$marker'")
@@ -316,7 +390,7 @@ if [ "$mode" = "--backup-restore" ]; then
     JOIN pg_namespace n ON n.oid=c.relnamespace
     WHERE n.nspname='vectors' AND a.attname='embedding' AND format_type(a.atttypid,a.atttypmod)='vectors.vector(768)'")
  test "$dimensions" = t
-  echo "Disposable-volume backup, mutation, restore, ledger, health, and retrieval parity passed."
+  echo "Transactional rollback and disposable-volume restore adapter parity passed."
 fi

 echo "Local pgvector runtime/bootstrap rotation, least-privilege roles, and persistence passed."
@@ -0,0 +1,72 @@
+#!/bin/sh
+set -eu
+
+cd "$(dirname "$0")/.."
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT HUP INT TERM
+fakebin="$tmp/bin"
+mkdir "$fakebin"
+printf '%s' secret >"$tmp/password"
+
+cat >"$fakebin/pg_dump" <<'SH'
+#!/bin/sh
+set -eu
+for arg in "$@"; do case "$arg" in --file=*) output=${arg#--file=} ;; esac; done
+printf 'custom dump' >"$output"
+SH
+chmod 0755 "$fakebin/pg_dump"
+
+victim="$tmp/victim"
+output="$tmp/vector.dump"
+printf 'sentinel' >"$victim"
+ln -s "$victim" "$output.partial"
+PATH="$fakebin:$PATH" ./scripts/vector-backup.sh --host source --database thoth --user admin \
+  --password-file "$tmp/password" --output "$output" >/dev/null
+test "$(cat "$victim")" = sentinel
+test "$(cat "$output")" = 'custom dump'
+test -L "$output.partial"
+
+cat >"$fakebin/psql" <<'SH'
+#!/bin/sh
+set -eu
+case "$*" in
+  *pg_control_system*)
+    echo same-cluster ;;
+  *) echo 0 ;;
+esac
+SH
+cat >"$fakebin/pg_restore" <<'SH'
+#!/bin/sh
+printf '%s\n' "$*" >"$RESTORE_LOG"
+SH
+chmod 0755 "$fakebin/psql" "$fakebin/pg_restore"
+printf 'archive' >"$tmp/input"
+if PATH="$fakebin:$PATH" RESTORE_LOG="$tmp/restore.log" ./scripts/vector-restore.sh \
+    --active-host source --active-database active --active-user admin \
+    --active-password-file "$tmp/password" --target-host target --target-database restore \
+    --target-user admin --target-password-file "$tmp/password" --input "$tmp/input" \
+    >"$tmp/out" 2>"$tmp/err"; then
+  echo "restore accepted a target on the active PostgreSQL cluster" >&2
+  exit 1
+fi
+grep -q 'same PostgreSQL cluster' "$tmp/err"
+test ! -e "$tmp/restore.log"
+
+cat >"$fakebin/psql" <<'SH'
+#!/bin/sh
+set -eu
+case "$*" in
+  *pg_control_system*)
+    case "$*" in *--host=source*) echo same-cluster ;; *) echo other-cluster ;; esac ;;
+  *) echo 0 ;;
+esac
+SH
+chmod 0755 "$fakebin/psql"
+PATH="$fakebin:$PATH" RESTORE_LOG="$tmp/restore.log" ./scripts/vector-restore.sh \
+  --active-host source --active-database active --active-user admin \
+  --active-password-file "$tmp/password" --target-host target --target-database restore \
+  --target-user admin --target-password-file "$tmp/password" --input "$tmp/input" >/dev/null
+grep -q -- '--single-transaction' "$tmp/restore.log"
+grep -q -- '--exit-on-error' "$tmp/restore.log"
+
+echo "vector backup/restore filesystem, identity, and transaction contracts passed."
@@ -25,12 +25,16 @@ done
 [ -n "$password_file" ] && [ -n "$output" ] || usage
 validate_secret_file "$password_file" "backup password file"
 [ ! -e "$output" ] || { echo "refusing to overwrite existing backup: $output" >&2; exit 2; }
+output_dir=$(dirname "$output")
+output_name=$(basename "$output")
+[ -d "$output_dir" ] || { echo "backup destination directory does not exist" >&2; exit 2; }

 password=$(read_secret_file "$password_file" "backup password file")

 umask 077
 passfile=$(mktemp "${TMPDIR:-/tmp}/thoth-vector-pgpass.XXXXXX")
-cleanup() { rm -f "$passfile" "$output.partial"; }
+temporary_output=$(mktemp "$output_dir/.${output_name}.tmp.XXXXXX")
+cleanup() { rm -f "$passfile" "$temporary_output"; }
 trap cleanup EXIT HUP INT TERM
 escaped=$(printf '%s' "$password" | sed 's/\\/\\\\/g; s/:/\\:/g')
 printf '%s:%s:%s:%s:%s\n' "$host" "$port" "$database" "$user" "$escaped" >"$passfile"
@@ -40,6 +44,6 @@ PGPASSFILE=$passfile pg_dump \
  --host="$host" --port="$port" --username="$user" --dbname="$database" \
  --format=custom --compress=9 \
  --table=vectors.schema_records --table=vectors.evidence --table=vectors.memory \
-  --table=public.tht_vector_migrations --file="$output.partial"
-mv "$output.partial" "$output"
+  --table=public.tht_vector_migrations --file="$temporary_output"
+mv "$temporary_output" "$output"
 echo "Vector backup written: $output"
@@ -53,13 +53,14 @@ make_passfile "$active_host" "$active_port" "$active_database" "$active_user" \
 make_passfile "$target_host" "$target_port" "$target_database" "$target_user" \
  "$target_password_file" "$target_pass"

-identity_sql="SELECT system_identifier::text || ':' || d.oid::text FROM pg_control_system(), pg_database d WHERE d.datname = current_database()"
+identity_sql="SELECT system_identifier::text FROM pg_control_system()"
 active_identity=$(PGPASSFILE=$active_pass psql -XAt --host="$active_host" --port="$active_port" \
  --username="$active_user" --dbname="$active_database" --command="$identity_sql")
 target_identity=$(PGPASSFILE=$target_pass psql -XAt --host="$target_host" --port="$target_port" \
  --username="$target_user" --dbname="$target_database" --command="$identity_sql")
 [ "$active_identity" != "$target_identity" ] || {
-  echo "refusing restore: active source and target are the same database" >&2; exit 2;
+  echo "refusing restore: active source and target are on the same PostgreSQL cluster" >&2
+  exit 2
 }

 object_count=$(PGPASSFILE=$target_pass psql -XAt --host="$target_host" --port="$target_port" \
@@ -72,7 +73,8 @@ if [ "$object_count" != 0 ] && [ "$force" != 1 ]; then
  exit 2
 fi

-PGPASSFILE=$target_pass pg_restore --exit-on-error --clean --if-exists --no-owner \
+PGPASSFILE=$target_pass pg_restore --exit-on-error --single-transaction \
+  --clean --if-exists --no-owner \
  --host="$target_host" --port="$target_port" --username="$target_user" \
  --dbname="$target_database" "$input"
 echo "Vector restore completed into explicit target $target_host:$target_port/$target_database"