docs(vector): add local backup restore and parity gate

scripts/local-vector-smoke.sh

@@ -5,8 +5,8 @@ cd "$(dirname "$0")/.."
 
 mode=${1:-run}
 case "$mode" in
-  run|--live-collision-test) ;;
-  *) echo "usage: $0 [--live-collision-test]" >&2; exit 2 ;;
+  run|--live-collision-test|--backup-restore) ;;
+  *) echo "usage: $0 [--live-collision-test|--backup-restore]" >&2; exit 2 ;;
 esac
 
 keep_resources=${KEEP_SMOKE_RESOURCES:-0}
@@ -19,6 +19,8 @@ suffix=$(basename "$secret_dir" | tr -cd 'a-z0-9')
 smoke_project="thothii-vector-smoke-$(date +%s)-$$-$suffix"
 smoke_owner="$smoke_project-owner"
 marker="local-vector-$smoke_project"
+restore_container="${smoke_project}-restore"
+restore_volume="${smoke_project}-restore-data"
 
 for secret in bootstrap migrator reader writer; do
   password="smoke-${secret}-${smoke_project}"
@@ -82,6 +84,8 @@ cleanup() {
     echo "Keeping smoke resources for project $smoke_project (KEEP_SMOKE_RESOURCES=1)." >&2
   else
     if verify_owned_resources; then
+      docker rm -f "$restore_container" >/dev/null 2>&1 || true
+      docker volume rm "$restore_volume" >/dev/null 2>&1 || true
       compose down --volumes >/dev/null 2>&1 || true
     fi
   fi
@@ -249,4 +253,70 @@ compose restart vector-db core
 compose up --wait vector-db core
 probe_vector read
 
+if [ "$mode" = "--backup-restore" ]; then
+  image=$(compose images -q vector-db)
+  network="${smoke_project}_default"
+  docker volume create \
+    --label "com.docker.compose.project=$smoke_project" \
+    --label "io.thothii.smoke-owner=$smoke_owner" "$restore_volume" >/dev/null
+  docker run -d --name "$restore_container" \
+    --label "com.docker.compose.project=$smoke_project" \
+    --label "io.thothii.smoke-owner=$smoke_owner" \
+    --network "$network" --network-alias vector-db-restore \
+    --mount "type=volume,source=$restore_volume,target=/var/lib/postgresql/data" \
+    --mount "type=bind,source=$secret_dir/bootstrap,target=/run/secrets/bootstrap,readonly" \
+    -e POSTGRES_DB=thoth -e POSTGRES_USER="$THT_VECTOR_BOOTSTRAP_USER" \
+    -e POSTGRES_PASSWORD_FILE=/run/secrets/bootstrap "$image" >/dev/null
+  attempts=0
+  until docker exec "$restore_container" pg_isready \
+      -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth >/dev/null 2>&1; do
+    attempts=$((attempts + 1))
+    [ "$attempts" -lt 30 ] || { echo "restore database did not become ready" >&2; exit 1; }
+    sleep 1
+  done
+  docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql -X \
+    -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth -v ON_ERROR_STOP=1 --command \
+    "CREATE SCHEMA vectors; CREATE EXTENSION vector WITH SCHEMA vectors;
+     CREATE ROLE vector_reader NOLOGIN; CREATE ROLE vector_writer NOLOGIN;" >/dev/null
+
+  docker run --rm --network "$network" \
+    --mount "type=bind,source=$(pwd),target=/repo,readonly" \
+    --mount "type=bind,source=$secret_dir,target=/scratch" "$image" \
+    /repo/scripts/vector-backup.sh --host vector-db --database thoth \
+    --user "$THT_VECTOR_BOOTSTRAP_USER" --password-file /scratch/bootstrap \
+    --output /scratch/vector.dump
+
+  compose exec -T vector-db sh -ec '
+    export PGPASSWORD=$(cat /run/secrets/vector_bootstrap_password)
+    psql -X -U "$POSTGRES_USER" -d thoth -v ON_ERROR_STOP=1 --command \
+      "UPDATE vectors.memory SET content_hash = '\''mutated-after-backup'\'' WHERE record_key = '\''$1'\''"' \
+    sh "$marker" >/dev/null
+
+  docker run --rm --network "$network" \
+    --mount "type=bind,source=$(pwd),target=/repo,readonly" \
+    --mount "type=bind,source=$secret_dir,target=/scratch" "$image" \
+    /repo/scripts/vector-restore.sh \
+    --active-host vector-db --active-database thoth --active-user "$THT_VECTOR_BOOTSTRAP_USER" \
+    --active-password-file /scratch/bootstrap \
+    --target-host vector-db-restore --target-database thoth \
+    --target-user "$THT_VECTOR_BOOTSTRAP_USER" --target-password-file /scratch/bootstrap \
+    --input /scratch/vector.dump
+
+  restored=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \
+    -XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \
+    "SELECT content_hash <> 'mutated-after-backup' FROM vectors.memory WHERE record_key = '$marker'")
+  test "$restored" = t
+  pending=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \
+    -XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \
+    "SELECT count(*) = 3 FROM public.tht_vector_migrations")
+  test "$pending" = t
+  dimensions=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \
+    -XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \
+    "SELECT count(*) = 3 FROM pg_attribute a JOIN pg_class c ON c.oid=a.attrelid
+     JOIN pg_namespace n ON n.oid=c.relnamespace
+     WHERE n.nspname='vectors' AND a.attname='embedding' AND format_type(a.atttypid,a.atttypmod)='vectors.vector(768)'")
+  test "$dimensions" = t
+  echo "Disposable-volume backup, mutation, restore, ledger, health, and retrieval parity passed."
+fi
+
 echo "Local pgvector runtime/bootstrap rotation, least-privilege roles, and persistence passed."