docs(vector): add local backup restore and parity gate
This commit is contained in:
@@ -5,8 +5,8 @@ cd "$(dirname "$0")/.."
|
||||
|
||||
mode=${1:-run}
|
||||
case "$mode" in
|
||||
run|--live-collision-test) ;;
|
||||
*) echo "usage: $0 [--live-collision-test]" >&2; exit 2 ;;
|
||||
run|--live-collision-test|--backup-restore) ;;
|
||||
*) echo "usage: $0 [--live-collision-test|--backup-restore]" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
keep_resources=${KEEP_SMOKE_RESOURCES:-0}
|
||||
@@ -19,6 +19,8 @@ suffix=$(basename "$secret_dir" | tr -cd 'a-z0-9')
|
||||
smoke_project="thothii-vector-smoke-$(date +%s)-$$-$suffix"
|
||||
smoke_owner="$smoke_project-owner"
|
||||
marker="local-vector-$smoke_project"
|
||||
restore_container="${smoke_project}-restore"
|
||||
restore_volume="${smoke_project}-restore-data"
|
||||
|
||||
for secret in bootstrap migrator reader writer; do
|
||||
password="smoke-${secret}-${smoke_project}"
|
||||
@@ -82,6 +84,8 @@ cleanup() {
|
||||
echo "Keeping smoke resources for project $smoke_project (KEEP_SMOKE_RESOURCES=1)." >&2
|
||||
else
|
||||
if verify_owned_resources; then
|
||||
docker rm -f "$restore_container" >/dev/null 2>&1 || true
|
||||
docker volume rm "$restore_volume" >/dev/null 2>&1 || true
|
||||
compose down --volumes >/dev/null 2>&1 || true
|
||||
fi
|
||||
fi
|
||||
@@ -249,4 +253,70 @@ compose restart vector-db core
|
||||
compose up --wait vector-db core
|
||||
probe_vector read
|
||||
|
||||
if [ "$mode" = "--backup-restore" ]; then
|
||||
image=$(compose images -q vector-db)
|
||||
network="${smoke_project}_default"
|
||||
docker volume create \
|
||||
--label "com.docker.compose.project=$smoke_project" \
|
||||
--label "io.thothii.smoke-owner=$smoke_owner" "$restore_volume" >/dev/null
|
||||
docker run -d --name "$restore_container" \
|
||||
--label "com.docker.compose.project=$smoke_project" \
|
||||
--label "io.thothii.smoke-owner=$smoke_owner" \
|
||||
--network "$network" --network-alias vector-db-restore \
|
||||
--mount "type=volume,source=$restore_volume,target=/var/lib/postgresql/data" \
|
||||
--mount "type=bind,source=$secret_dir/bootstrap,target=/run/secrets/bootstrap,readonly" \
|
||||
-e POSTGRES_DB=thoth -e POSTGRES_USER="$THT_VECTOR_BOOTSTRAP_USER" \
|
||||
-e POSTGRES_PASSWORD_FILE=/run/secrets/bootstrap "$image" >/dev/null
|
||||
attempts=0
|
||||
until docker exec "$restore_container" pg_isready \
|
||||
-U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth >/dev/null 2>&1; do
|
||||
attempts=$((attempts + 1))
|
||||
[ "$attempts" -lt 30 ] || { echo "restore database did not become ready" >&2; exit 1; }
|
||||
sleep 1
|
||||
done
|
||||
docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql -X \
|
||||
-U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth -v ON_ERROR_STOP=1 --command \
|
||||
"CREATE SCHEMA vectors; CREATE EXTENSION vector WITH SCHEMA vectors;
|
||||
CREATE ROLE vector_reader NOLOGIN; CREATE ROLE vector_writer NOLOGIN;" >/dev/null
|
||||
|
||||
docker run --rm --network "$network" \
|
||||
--mount "type=bind,source=$(pwd),target=/repo,readonly" \
|
||||
--mount "type=bind,source=$secret_dir,target=/scratch" "$image" \
|
||||
/repo/scripts/vector-backup.sh --host vector-db --database thoth \
|
||||
--user "$THT_VECTOR_BOOTSTRAP_USER" --password-file /scratch/bootstrap \
|
||||
--output /scratch/vector.dump
|
||||
|
||||
compose exec -T vector-db sh -ec '
|
||||
export PGPASSWORD=$(cat /run/secrets/vector_bootstrap_password)
|
||||
psql -X -U "$POSTGRES_USER" -d thoth -v ON_ERROR_STOP=1 --command \
|
||||
"UPDATE vectors.memory SET content_hash = '\''mutated-after-backup'\'' WHERE record_key = '\''$1'\''"' \
|
||||
sh "$marker" >/dev/null
|
||||
|
||||
docker run --rm --network "$network" \
|
||||
--mount "type=bind,source=$(pwd),target=/repo,readonly" \
|
||||
--mount "type=bind,source=$secret_dir,target=/scratch" "$image" \
|
||||
/repo/scripts/vector-restore.sh \
|
||||
--active-host vector-db --active-database thoth --active-user "$THT_VECTOR_BOOTSTRAP_USER" \
|
||||
--active-password-file /scratch/bootstrap \
|
||||
--target-host vector-db-restore --target-database thoth \
|
||||
--target-user "$THT_VECTOR_BOOTSTRAP_USER" --target-password-file /scratch/bootstrap \
|
||||
--input /scratch/vector.dump
|
||||
|
||||
restored=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \
|
||||
-XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \
|
||||
"SELECT content_hash <> 'mutated-after-backup' FROM vectors.memory WHERE record_key = '$marker'")
|
||||
test "$restored" = t
|
||||
pending=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \
|
||||
-XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \
|
||||
"SELECT count(*) = 3 FROM public.tht_vector_migrations")
|
||||
test "$pending" = t
|
||||
dimensions=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \
|
||||
-XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \
|
||||
"SELECT count(*) = 3 FROM pg_attribute a JOIN pg_class c ON c.oid=a.attrelid
|
||||
JOIN pg_namespace n ON n.oid=c.relnamespace
|
||||
WHERE n.nspname='vectors' AND a.attname='embedding' AND format_type(a.atttypid,a.atttypmod)='vectors.vector(768)'")
|
||||
test "$dimensions" = t
|
||||
echo "Disposable-volume backup, mutation, restore, ledger, health, and retrieval parity passed."
|
||||
fi
|
||||
|
||||
echo "Local pgvector runtime/bootstrap rotation, least-privilege roles, and persistence passed."
|
||||
|
||||
Executable
+45
@@ -0,0 +1,45 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)
|
||||
. "$root/deploy/vector/secret-policy.sh"
|
||||
|
||||
usage() {
|
||||
echo "usage: $0 --host HOST --database DB --user USER --password-file FILE --output FILE [--port PORT]" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
host= database= user= password_file= output= port=5432
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--host) host=${2-}; shift 2 ;;
|
||||
--port) port=${2-}; shift 2 ;;
|
||||
--database) database=${2-}; shift 2 ;;
|
||||
--user) user=${2-}; shift 2 ;;
|
||||
--password-file) password_file=${2-}; shift 2 ;;
|
||||
--output) output=${2-}; shift 2 ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
done
|
||||
[ -n "$host" ] && [ -n "$database" ] && [ -n "$user" ] || usage
|
||||
[ -n "$password_file" ] && [ -n "$output" ] || usage
|
||||
validate_secret_file "$password_file" "backup password file"
|
||||
[ ! -e "$output" ] || { echo "refusing to overwrite existing backup: $output" >&2; exit 2; }
|
||||
|
||||
password=$(read_secret_file "$password_file" "backup password file")
|
||||
|
||||
umask 077
|
||||
passfile=$(mktemp "${TMPDIR:-/tmp}/thoth-vector-pgpass.XXXXXX")
|
||||
cleanup() { rm -f "$passfile" "$output.partial"; }
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
escaped=$(printf '%s' "$password" | sed 's/\\/\\\\/g; s/:/\\:/g')
|
||||
printf '%s:%s:%s:%s:%s\n' "$host" "$port" "$database" "$user" "$escaped" >"$passfile"
|
||||
chmod 0600 "$passfile"
|
||||
|
||||
PGPASSFILE=$passfile pg_dump \
|
||||
--host="$host" --port="$port" --username="$user" --dbname="$database" \
|
||||
--format=custom --compress=9 \
|
||||
--table=vectors.schema_records --table=vectors.evidence --table=vectors.memory \
|
||||
--table=public.tht_vector_migrations --file="$output.partial"
|
||||
mv "$output.partial" "$output"
|
||||
echo "Vector backup written: $output"
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)
|
||||
. "$root/deploy/vector/secret-policy.sh"
|
||||
|
||||
usage() {
|
||||
echo "usage: $0 --active-host HOST --active-database DB --active-user USER --active-password-file FILE --target-host HOST --target-database DB --target-user USER --target-password-file FILE --input FILE [--active-port PORT] [--target-port PORT] [--force-nonempty]" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
active_host= active_database= active_user= active_password_file= active_port=5432
|
||||
target_host= target_database= target_user= target_password_file= target_port=5432
|
||||
input= force=0
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--active-host) active_host=${2-}; shift 2 ;;
|
||||
--active-port) active_port=${2-}; shift 2 ;;
|
||||
--active-database) active_database=${2-}; shift 2 ;;
|
||||
--active-user) active_user=${2-}; shift 2 ;;
|
||||
--active-password-file) active_password_file=${2-}; shift 2 ;;
|
||||
--target-host) target_host=${2-}; shift 2 ;;
|
||||
--target-port) target_port=${2-}; shift 2 ;;
|
||||
--target-database) target_database=${2-}; shift 2 ;;
|
||||
--target-user) target_user=${2-}; shift 2 ;;
|
||||
--target-password-file) target_password_file=${2-}; shift 2 ;;
|
||||
--input) input=${2-}; shift 2 ;;
|
||||
--force-nonempty) force=1; shift ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
done
|
||||
for value in "$active_host" "$active_database" "$active_user" "$active_password_file" \
|
||||
"$target_host" "$target_database" "$target_user" "$target_password_file" "$input"; do
|
||||
[ -n "$value" ] || usage
|
||||
done
|
||||
[ -r "$input" ] || { echo "backup input is not readable" >&2; exit 2; }
|
||||
validate_secret_file "$active_password_file" "active source password file"
|
||||
validate_secret_file "$target_password_file" "target password file"
|
||||
|
||||
umask 077
|
||||
active_pass=$(mktemp "${TMPDIR:-/tmp}/thoth-vector-active-pgpass.XXXXXX")
|
||||
target_pass=$(mktemp "${TMPDIR:-/tmp}/thoth-vector-target-pgpass.XXXXXX")
|
||||
cleanup() { rm -f "$active_pass" "$target_pass"; }
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
make_passfile() {
|
||||
secret=$(read_secret_file "$5" "database password file")
|
||||
escaped=$(printf '%s' "$secret" | sed 's/\\/\\\\/g; s/:/\\:/g')
|
||||
printf '%s:%s:%s:%s:%s\n' "$1" "$2" "$3" "$4" "$escaped" >"$6"
|
||||
chmod 0600 "$6"
|
||||
}
|
||||
make_passfile "$active_host" "$active_port" "$active_database" "$active_user" \
|
||||
"$active_password_file" "$active_pass"
|
||||
make_passfile "$target_host" "$target_port" "$target_database" "$target_user" \
|
||||
"$target_password_file" "$target_pass"
|
||||
|
||||
identity_sql="SELECT system_identifier::text || ':' || d.oid::text FROM pg_control_system(), pg_database d WHERE d.datname = current_database()"
|
||||
active_identity=$(PGPASSFILE=$active_pass psql -XAt --host="$active_host" --port="$active_port" \
|
||||
--username="$active_user" --dbname="$active_database" --command="$identity_sql")
|
||||
target_identity=$(PGPASSFILE=$target_pass psql -XAt --host="$target_host" --port="$target_port" \
|
||||
--username="$target_user" --dbname="$target_database" --command="$identity_sql")
|
||||
[ "$active_identity" != "$target_identity" ] || {
|
||||
echo "refusing restore: active source and target are the same database" >&2; exit 2;
|
||||
}
|
||||
|
||||
object_count=$(PGPASSFILE=$target_pass psql -XAt --host="$target_host" --port="$target_port" \
|
||||
--username="$target_user" --dbname="$target_database" --command="
|
||||
SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace
|
||||
WHERE (n.nspname='vectors' OR (n.nspname='public' AND c.relname='tht_vector_migrations'))
|
||||
AND c.relkind IN ('r','p','S','v','m');")
|
||||
if [ "$object_count" != 0 ] && [ "$force" != 1 ]; then
|
||||
echo "refusing restore into non-empty target; use --force-nonempty explicitly" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
PGPASSFILE=$target_pass pg_restore --exit-on-error --clean --if-exists --no-owner \
|
||||
--host="$target_host" --port="$target_port" --username="$target_user" \
|
||||
--dbname="$target_database" "$input"
|
||||
echo "Vector restore completed into explicit target $target_host:$target_port/$target_database"
|
||||
Reference in New Issue
Block a user