fix(vector): harden backup restore parity gates

scripts/local-vector-smoke.sh

@@ -277,7 +277,13 @@ if [ "$mode" = "--backup-restore" ]; then
   docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql -X \
     -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth -v ON_ERROR_STOP=1 --command \
     "CREATE SCHEMA vectors; CREATE EXTENSION vector WITH SCHEMA vectors;
-     CREATE ROLE vector_reader NOLOGIN; CREATE ROLE vector_writer NOLOGIN;" >/dev/null
+     CREATE TABLE vectors.memory (
+       id bigserial PRIMARY KEY, record_key text UNIQUE NOT NULL, kind text NOT NULL,
+       content_hash text NOT NULL, metadata jsonb NOT NULL,
+       embedding vectors.vector(768) NOT NULL, indexed_at timestamptz NOT NULL DEFAULT now());
+     INSERT INTO vectors.memory (record_key, kind, content_hash, metadata, embedding)
+       VALUES ('restore-sentinel', 'memory', 'sentinel-original', '{}',
+               ('[' || '1,' || repeat('0,', 766) || '0]')::vectors.vector);" >/dev/null
 
   docker run --rm --network "$network" \
     --mount "type=bind,source=$(pwd),target=/repo,readonly" \
@@ -292,6 +298,27 @@ if [ "$mode" = "--backup-restore" ]; then
       "UPDATE vectors.memory SET content_hash = '\''mutated-after-backup'\'' WHERE record_key = '\''$1'\''"' \
     sh "$marker" >/dev/null
 
+  if docker run --rm --network "$network" \
+      --mount "type=bind,source=$(pwd),target=/repo,readonly" \
+      --mount "type=bind,source=$secret_dir,target=/scratch" "$image" \
+      /repo/scripts/vector-restore.sh \
+      --active-host vector-db --active-database thoth --active-user "$THT_VECTOR_BOOTSTRAP_USER" \
+      --active-password-file /scratch/bootstrap \
+      --target-host vector-db-restore --target-database thoth \
+      --target-user "$THT_VECTOR_BOOTSTRAP_USER" --target-password-file /scratch/bootstrap \
+      --input /scratch/vector.dump --force-nonempty >/dev/null 2>&1; then
+    echo "forced restore unexpectedly succeeded without archived ACL roles" >&2
+    exit 1
+  fi
+  sentinel=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \
+    -XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \
+    "SELECT content_hash FROM vectors.memory WHERE record_key='restore-sentinel'")
+  test "$sentinel" = sentinel-original
+  docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql -X \
+    -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth -v ON_ERROR_STOP=1 --command \
+    "DROP TABLE vectors.memory; CREATE ROLE vector_reader NOLOGIN; CREATE ROLE vector_writer NOLOGIN;" \
+    >/dev/null
+
   docker run --rm --network "$network" \
     --mount "type=bind,source=$(pwd),target=/repo,readonly" \
     --mount "type=bind,source=$secret_dir,target=/scratch" "$image" \
@@ -302,6 +329,53 @@ if [ "$mode" = "--backup-restore" ]; then
     --target-user "$THT_VECTOR_BOOTSTRAP_USER" --target-password-file /scratch/bootstrap \
     --input /scratch/vector.dump
 
+  docker run --rm --network "$network" \
+    --mount "type=bind,source=$(pwd)/deploy/vector/reconcile-roles.sh,target=/opt/thoth/reconcile-roles.sh,readonly" \
+    --mount "type=bind,source=$(pwd)/deploy/vector/secret-policy.sh,target=/opt/thoth/secret-policy.sh,readonly" \
+    --mount "type=bind,source=$secret_dir/bootstrap,target=/run/secrets/vector_bootstrap_password,readonly" \
+    --mount "type=bind,source=$secret_dir/migrator,target=/run/secrets/vector_migrator_password,readonly" \
+    --mount "type=bind,source=$secret_dir/reader,target=/run/secrets/vector_reader_password,readonly" \
+    --mount "type=bind,source=$secret_dir/writer,target=/run/secrets/vector_writer_password,readonly" \
+    -e PGHOST=vector-db-restore -e PGDATABASE=thoth \
+    -e PGUSER="$THT_VECTOR_BOOTSTRAP_USER" \
+    -e THT_VECTOR_MIGRATOR_USER=thoth_vector_migrator \
+    -e THT_VECTOR_READER_USER=thoth_vector_reader \
+    -e THT_VECTOR_WRITER_USER=thoth_vector_writer \
+    --entrypoint /opt/thoth/reconcile-roles.sh "$image" >/dev/null
+
+  docker run --rm -i --network "$network" \
+    -e THT_VECTOR_READER_PASSWORD="$THT_VECTOR_READER_PASSWORD" \
+    -e THT_VECTOR_WRITER_PASSWORD="$THT_VECTOR_WRITER_PASSWORD" \
+    --entrypoint /opt/venv/bin/python thothii-core:local - "$marker" <<'PY'
+import hashlib
+import os
+import sys
+
+from tht.adapters.vector.pgvector import PgVectorStore
+from tht.config import DatabaseConfig
+from tht.ports.vector import VectorWriteRecord
+from tht.vectorstore.records import VectorRecord
+
+def config(role):
+    return DatabaseConfig(
+        host="vector-db-restore", port=5432, database="thoth", schema="vectors",
+        user=f"thoth_vector_{role}", password=os.environ[f"THT_VECTOR_{role.upper()}_PASSWORD"],
+    )
+
+store = PgVectorStore(config("reader"), config("writer"), expected_dimension=768)
+assert store.health().ok, store.health()
+embedding = [1.0] + [0.0] * 767
+marker = sys.argv[1]
+assert store.search(["memory"], embedding, limit=1, kinds=["memory"])[0].id == marker
+write_id = marker + "-restore-write"
+record = VectorRecord(
+    id=write_id, kind="memory", ref=write_id, title="restore writer",
+    content=write_id, metadata={},
+)
+store.upsert("memory", [VectorWriteRecord(record, embedding, hashlib.sha256(write_id.encode()).hexdigest())])
+assert store.existing_hashes("memory", ["memory"])[write_id]
+PY
+
   restored=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \
     -XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \
     "SELECT content_hash <> 'mutated-after-backup' FROM vectors.memory WHERE record_key = '$marker'")
@@ -316,7 +390,7 @@ if [ "$mode" = "--backup-restore" ]; then
      JOIN pg_namespace n ON n.oid=c.relnamespace
      WHERE n.nspname='vectors' AND a.attname='embedding' AND format_type(a.atttypid,a.atttypmod)='vectors.vector(768)'")
   test "$dimensions" = t
-  echo "Disposable-volume backup, mutation, restore, ledger, health, and retrieval parity passed."
+  echo "Transactional rollback and disposable-volume restore adapter parity passed."
 fi
 
 echo "Local pgvector runtime/bootstrap rotation, least-privilege roles, and persistence passed."