fix: retire active pgvector artifacts
This commit is contained in:
@@ -61,8 +61,9 @@ for profile in local server; do
|
||||
const fs = require("fs");
|
||||
const [path, profile] = process.argv.slice(2);
|
||||
const config = JSON.parse(fs.readFileSync(path, "utf8"));
|
||||
if (Object.keys(config.services).sort().join(",") !== "core,frontend") {
|
||||
throw new Error(profile + ": install stack must be exactly core,frontend");
|
||||
const expected = "core,embedding,embedding-model-init,frontend,qdrant";
|
||||
if (Object.keys(config.services).sort().join(",") !== expected) {
|
||||
throw new Error(profile + ": install stack must be exactly " + expected);
|
||||
}
|
||||
if (!config.services.core.secrets?.some((secret) => secret.target === "thothii.secrets")) {
|
||||
throw new Error(profile + ": install stack lacks the runtime secret bundle");
|
||||
|
||||
@@ -28,13 +28,17 @@ new_fixture() {
|
||||
printf '%s\n' '// generic frontend configuration' >"$fixture/repository/frontend/vite.config.ts"
|
||||
printf '%s\n' '// explicit descriptor migration module may mention pgvector during conversion' \
|
||||
>"$fixture/repository/backend/src/workspaces/migrate-legacy.ts"
|
||||
printf '%s\n' 'language: en' 'vectors: { type: qdrant, base_url: http://qdrant:6333, collection: demo }' \
|
||||
>"$fixture/repository/deploy/workspaces/example.yaml"
|
||||
printf '%s\n' '# qdrant backup helper' >"$fixture/repository/scripts/vector-backup.sh"
|
||||
printf '%s\n' '# qdrant restore helper' >"$fixture/repository/scripts/vector-restore.sh"
|
||||
|
||||
# These are the three intentionally allowed categories from the Task 10 boundary.
|
||||
printf '%s\n' 'historical omics_portal and Chirone record' \
|
||||
>"$fixture/repository/docs/superpowers/plans/legacy.md"
|
||||
printf '%s\n' 'historical pgvector rollout note' \
|
||||
>"$fixture/repository/docs/superpowers/specs/history.md"
|
||||
printf '%s\n' 'id: psd' >"$fixture/repository/deploy/workspaces/psd.yaml.example"
|
||||
printf '%s\n' 'id: generic' >"$fixture/repository/deploy/workspaces/psd.yaml.example"
|
||||
printf '%s\n' '# migrate PSD sessions from /home/chirone' \
|
||||
>"$fixture/repository/docker/session-migrate.sh"
|
||||
}
|
||||
@@ -77,6 +81,10 @@ assert_detected scripts/run-stack.sh 'export THT_VECTOR_READER_PASSWORD_FILE=/ru
|
||||
assert_detected deploy/env/local.env.example 'THT_OLLAMA_URL=http://ollama.example.invalid:11434'
|
||||
assert_detected scripts/generate-override.sh 'THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_SOURCE=/tmp/vector-key'
|
||||
assert_detected scripts/test-contract.sh 'docker compose -f deploy/compose.local-vector.yaml --profile local-vector config'
|
||||
assert_detected deploy/workspaces/local-vector.yaml 'vectors: { type: pgvector_direct }'
|
||||
assert_detected deploy/workspaces/example.yaml 'embeddings: { base_url: ${THT_OLLAMA_URL} }'
|
||||
assert_detected scripts/vector-backup.sh 'pg_dump --format=custom'
|
||||
assert_detected scripts/vector-restore.sh 'pg_restore --single-transaction'
|
||||
|
||||
new_fixture
|
||||
mkdir -p "$fixture/bin"
|
||||
|
||||
@@ -28,7 +28,7 @@ for file in .dockerignore compose.yaml docker-compose.dev.yml frontend/vite.conf
|
||||
done
|
||||
if [[ -d deploy ]]; then
|
||||
while IFS= read -r -d '' file; do runtime_files+=("${file#./}"); done < <(
|
||||
find deploy -type f ! -path 'deploy/workspaces/*' -print0
|
||||
find deploy -type f -print0
|
||||
)
|
||||
fi
|
||||
if [[ -d docker ]]; then
|
||||
@@ -57,7 +57,7 @@ if [[ -d scripts ]]; then
|
||||
contract_test_files+=("${file#./}")
|
||||
continue
|
||||
;;
|
||||
compose-with-preflight.sh|generate-connector-secrets-override.sh|unified-deployment-smoke.sh|vector-backup.sh|vector-restore.sh|vector-rotate-bootstrap-password.sh)
|
||||
compose-with-preflight.sh|generate-connector-secrets-override.sh|unified-deployment-smoke.sh|vector-rotate-bootstrap-password.sh)
|
||||
continue
|
||||
;;
|
||||
verify-*.sh) continue ;;
|
||||
@@ -113,7 +113,7 @@ for forbidden_file in \
|
||||
done
|
||||
|
||||
forbidden='omics_portal|chirone|localllm_default|datamart-builder|compose\.production\.yaml|compose\.psd-local\.yaml|\bpsd\b'
|
||||
retired_semantic='local-vector|THT_VECTOR_(BOOTSTRAP|MIGRATOR|READER|WRITER|DATABASE|HOST|PORT|USER|ADMIN_URL|OPERATOR_ENV_FILE)|THT_OLLAMA_URL|VECTOR_API_KEY_(FILE|SOURCE)|vector-api-key|(^|[^A-Za-z0-9_])THT_VEC_(REST_URL|WRITE_REST_URL)'
|
||||
retired_semantic='local-vector|pgvector(_direct)?|pg_(dump|restore)|THT_VECTOR_([A-Z0-9_]+)|THT_OLLAMA_URL|THT_WS_[A-Z0-9_]*_(VECTOR|EMBEDDING)_[A-Z0-9_]+|VECTOR_API_KEY_(FILE|SOURCE)|EMBEDDING_API_KEY_(FILE|SOURCE)|vector-api-key|(^|[^A-Za-z0-9_])THT_VEC_(REST_URL|WRITE_REST_URL)|thoth_vector_http'
|
||||
scan_category runtime "$forbidden" "${runtime_files[@]}"
|
||||
scan_category install "$forbidden" "${install_files[@]}"
|
||||
scan_category operator "$forbidden" "${operator_files[@]}"
|
||||
@@ -132,7 +132,7 @@ for file in "${contract_test_files[@]}"; do
|
||||
esac
|
||||
contract_scan_files+=("$file")
|
||||
done
|
||||
retired_semantic_contract='docker compose[^\n]*(compose\.local-vector|compose\.preprocess-local-vector)|THT_VECTOR_(BOOTSTRAP|MIGRATOR|READER|WRITER|DATABASE|HOST|PORT|USER|ADMIN_URL|OPERATOR_ENV_FILE)=|THT_OLLAMA_URL=|THT_WS_[A-Z0-9_]*_VECTOR_(TRANSPORT|API_KEY_(FILE|SOURCE))=|vector-api-key'
|
||||
retired_semantic_contract='docker compose[^\n]*(compose\.local-vector|compose\.preprocess-local-vector)|THT_VECTOR_([A-Z0-9_]+)=|THT_OLLAMA_URL=|THT_WS_[A-Z0-9_]*_(VECTOR|EMBEDDING)_[A-Z0-9_]+=|vector-api-key|pgvector|pg_(dump|restore)'
|
||||
scan_category contract-test "$retired_semantic_contract" "${contract_scan_files[@]}"
|
||||
|
||||
if [[ -f scripts/run-stack.sh ]]; then
|
||||
|
||||
@@ -6,84 +6,103 @@ tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT TERM
|
||||
fakebin="$tmp/bin"
|
||||
mkdir "$fakebin"
|
||||
printf '%s' secret >"$tmp/password"
|
||||
chmod 0600 "$tmp/password"
|
||||
|
||||
cat >"$fakebin/pg_dump" <<'SH'
|
||||
project="thoth-task8"
|
||||
volume_name="${project}_qdrant-data"
|
||||
mountpoint="$tmp/docker-volumes/$volume_name/_data"
|
||||
mkdir -p "$mountpoint/collections/demo"
|
||||
printf '%s' before-backup >"$mountpoint/collections/demo/state.json"
|
||||
|
||||
cat >"$fakebin/docker" <<'SH'
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
for arg in "$@"; do case "$arg" in --file=*) output=${arg#--file=} ;; esac; done
|
||||
printf 'custom dump' >"$output"
|
||||
if [ -n "${RACE_OUTPUT:-}" ]; then
|
||||
printf 'concurrent owner' >"$RACE_OUTPUT"
|
||||
log_file=${DOCKER_LOG:?}
|
||||
printf '%s\n' "$*" >>"$log_file"
|
||||
|
||||
if [ "$1" = volume ] && [ "$2" = ls ]; then
|
||||
if [ "${VOLUME_LS_OUTPUT:-}" = multiple ]; then
|
||||
printf '%s\n%s\n' "${PROJECT_NAME}_qdrant-data" "${PROJECT_NAME}_qdrant-data-copy"
|
||||
exit 0
|
||||
fi
|
||||
printf '%s\n' "${PROJECT_NAME}_qdrant-data"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$1" = volume ] && [ "$2" = inspect ]; then
|
||||
printf '%s\n' "${MOUNTPOINT:?}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$1" = compose ] && [ "$2" = --project-name ]; then
|
||||
case "$4" in
|
||||
ps)
|
||||
if [ "${QDRANT_RUNNING:-1}" = 1 ]; then
|
||||
printf '%s\n' qdrant-container
|
||||
fi
|
||||
exit 0
|
||||
;;
|
||||
stop)
|
||||
exit 0
|
||||
;;
|
||||
start)
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
exit 0
|
||||
SH
|
||||
chmod 0755 "$fakebin/pg_dump"
|
||||
chmod 0755 "$fakebin/docker"
|
||||
|
||||
victim="$tmp/victim"
|
||||
output="$tmp/vector.dump"
|
||||
printf 'sentinel' >"$victim"
|
||||
ln -s "$victim" "$output.partial"
|
||||
PATH="$fakebin:$PATH" ./scripts/vector-backup.sh --host source --database thoth --user admin \
|
||||
--password-file "$tmp/password" --output "$output" >/dev/null
|
||||
test "$(cat "$victim")" = sentinel
|
||||
test "$(cat "$output")" = 'custom dump'
|
||||
test -L "$output.partial"
|
||||
backup_output="$tmp/qdrant-backup.tar"
|
||||
docker_log="$tmp/docker.log"
|
||||
PATH="$fakebin:$PATH" DOCKER_LOG="$docker_log" PROJECT_NAME="$project" MOUNTPOINT="$mountpoint" \
|
||||
./scripts/vector-backup.sh --project-name "$project" --output "$backup_output" >/dev/null
|
||||
test -s "$backup_output"
|
||||
tar -tf "$backup_output" | grep -q '^./collections/demo/state.json$'
|
||||
grep -q "volume ls --filter label=com.docker.compose.project=$project --filter label=com.docker.compose.volume=qdrant-data" "$docker_log"
|
||||
grep -q "compose --project-name $project ps --status running -q qdrant" "$docker_log"
|
||||
grep -q "compose --project-name $project stop qdrant" "$docker_log"
|
||||
grep -q "compose --project-name $project start qdrant" "$docker_log"
|
||||
|
||||
race_output="$tmp/raced.dump"
|
||||
if PATH="$fakebin:$PATH" RACE_OUTPUT="$race_output" ./scripts/vector-backup.sh \
|
||||
--host source --database thoth --user admin --password-file "$tmp/password" \
|
||||
--output "$race_output" >"$tmp/race.out" 2>"$tmp/race.err"; then
|
||||
echo "backup replaced a destination created concurrently" >&2
|
||||
existing="$tmp/existing.tar"
|
||||
printf '%s' sentinel >"$existing"
|
||||
if PATH="$fakebin:$PATH" DOCKER_LOG="$tmp/existing.log" PROJECT_NAME="$project" MOUNTPOINT="$mountpoint" \
|
||||
./scripts/vector-backup.sh --project-name "$project" --output "$existing" >"$tmp/existing.out" 2>"$tmp/existing.err"; then
|
||||
echo "backup overwrote an existing archive" >&2
|
||||
exit 1
|
||||
fi
|
||||
test "$(cat "$race_output")" = 'concurrent owner'
|
||||
if find "$tmp" -name '.raced.dump.tmp.*' -print | grep -q .; then
|
||||
echo "backup left its owned temporary archive after publication failure" >&2
|
||||
test "$(cat "$existing")" = sentinel
|
||||
|
||||
if PATH="$fakebin:$PATH" DOCKER_LOG="$tmp/ambiguous.log" PROJECT_NAME="$project" MOUNTPOINT="$mountpoint" VOLUME_LS_OUTPUT=multiple \
|
||||
./scripts/vector-backup.sh --project-name "$project" --output "$tmp/ambiguous.tar" >"$tmp/ambiguous.out" 2>"$tmp/ambiguous.err"; then
|
||||
echo "backup accepted an ambiguous qdrant-data target" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q 'exactly one qdrant-data volume' "$tmp/ambiguous.err"
|
||||
|
||||
cat >"$fakebin/psql" <<'SH'
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
case "$*" in
|
||||
*pg_control_system*)
|
||||
echo same-cluster ;;
|
||||
*) echo 0 ;;
|
||||
esac
|
||||
SH
|
||||
cat >"$fakebin/pg_restore" <<'SH'
|
||||
#!/bin/sh
|
||||
printf '%s\n' "$*" >"$RESTORE_LOG"
|
||||
SH
|
||||
chmod 0755 "$fakebin/psql" "$fakebin/pg_restore"
|
||||
printf 'archive' >"$tmp/input"
|
||||
if PATH="$fakebin:$PATH" RESTORE_LOG="$tmp/restore.log" ./scripts/vector-restore.sh \
|
||||
--active-host source --active-database active --active-user admin \
|
||||
--active-password-file "$tmp/password" --target-host target --target-database restore \
|
||||
--target-user admin --target-password-file "$tmp/password" --input "$tmp/input" \
|
||||
>"$tmp/out" 2>"$tmp/err"; then
|
||||
echo "restore accepted a target on the active PostgreSQL cluster" >&2
|
||||
restore_input="$tmp/restore.tar"
|
||||
restore_source="$tmp/restore-source"
|
||||
mkdir -p "$restore_source/collections/demo"
|
||||
printf '%s' restored >"$restore_source/collections/demo/state.json"
|
||||
tar -C "$restore_source" -cf "$restore_input" .
|
||||
|
||||
if PATH="$fakebin:$PATH" DOCKER_LOG="$tmp/restore-refuse.log" PROJECT_NAME="$project" MOUNTPOINT="$mountpoint" \
|
||||
./scripts/vector-restore.sh --project-name "$project" --input "$restore_input" --confirm-project wrong-project \
|
||||
>"$tmp/restore-refuse.out" 2>"$tmp/restore-refuse.err"; then
|
||||
echo "restore skipped explicit project confirmation" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q 'same PostgreSQL cluster' "$tmp/err"
|
||||
test ! -e "$tmp/restore.log"
|
||||
grep -q 'confirmation must match --project-name exactly' "$tmp/restore-refuse.err"
|
||||
test "$(cat "$mountpoint/collections/demo/state.json")" = before-backup
|
||||
|
||||
cat >"$fakebin/psql" <<'SH'
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
case "$*" in
|
||||
*pg_control_system*)
|
||||
case "$*" in *--host=source*) echo same-cluster ;; *) echo other-cluster ;; esac ;;
|
||||
*) echo 0 ;;
|
||||
esac
|
||||
SH
|
||||
chmod 0755 "$fakebin/psql"
|
||||
PATH="$fakebin:$PATH" RESTORE_LOG="$tmp/restore.log" ./scripts/vector-restore.sh \
|
||||
--active-host source --active-database active --active-user admin \
|
||||
--active-password-file "$tmp/password" --target-host target --target-database restore \
|
||||
--target-user admin --target-password-file "$tmp/password" --input "$tmp/input" >/dev/null
|
||||
grep -q -- '--single-transaction' "$tmp/restore.log"
|
||||
grep -q -- '--exit-on-error' "$tmp/restore.log"
|
||||
printf '%s' modified-live >"$mountpoint/collections/demo/state.json"
|
||||
restore_log="$tmp/restore-ok.log"
|
||||
PATH="$fakebin:$PATH" DOCKER_LOG="$restore_log" PROJECT_NAME="$project" MOUNTPOINT="$mountpoint" \
|
||||
./scripts/vector-restore.sh --project-name "$project" --input "$restore_input" --confirm-project "$project" >/dev/null
|
||||
test "$(cat "$mountpoint/collections/demo/state.json")" = restored
|
||||
grep -q "compose --project-name $project stop qdrant" "$restore_log"
|
||||
grep -q "compose --project-name $project start qdrant" "$restore_log"
|
||||
grep -q "volume inspect --format {{ .Mountpoint }} $volume_name" "$restore_log"
|
||||
|
||||
echo "vector backup/restore filesystem, identity, and transaction contracts passed."
|
||||
echo "qdrant backup/restore target resolution, refusal, and service-state contracts passed."
|
||||
|
||||
+54
-27
@@ -1,53 +1,80 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)
|
||||
. "$root/scripts/secret-file-utils.sh"
|
||||
|
||||
usage() {
|
||||
echo "usage: $0 --host HOST --database DB --user USER --password-file FILE --output FILE [--port PORT]" >&2
|
||||
echo "usage: $0 --project-name NAME --output FILE" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
host= database= user= password_file= output= port=5432
|
||||
project_name=
|
||||
output=
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--host) host=${2-}; shift 2 ;;
|
||||
--port) port=${2-}; shift 2 ;;
|
||||
--database) database=${2-}; shift 2 ;;
|
||||
--user) user=${2-}; shift 2 ;;
|
||||
--password-file) password_file=${2-}; shift 2 ;;
|
||||
--project-name) project_name=${2-}; shift 2 ;;
|
||||
--output) output=${2-}; shift 2 ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
done
|
||||
[ -n "$host" ] && [ -n "$database" ] && [ -n "$user" ] || usage
|
||||
[ -n "$password_file" ] && [ -n "$output" ] || usage
|
||||
validate_secret_file "$password_file" "backup password file"
|
||||
|
||||
[ -n "$project_name" ] && [ -n "$output" ] || usage
|
||||
[ ! -e "$output" ] || { echo "refusing to overwrite existing backup: $output" >&2; exit 2; }
|
||||
|
||||
output_dir=$(dirname "$output")
|
||||
output_name=$(basename "$output")
|
||||
[ -d "$output_dir" ] || { echo "backup destination directory does not exist" >&2; exit 2; }
|
||||
|
||||
password=$(read_secret_file "$password_file" "backup password file")
|
||||
resolve_volume() {
|
||||
names=$(docker volume ls \
|
||||
--filter "label=com.docker.compose.project=$project_name" \
|
||||
--filter "label=com.docker.compose.volume=qdrant-data" \
|
||||
--format '{{.Name}}')
|
||||
count=$(printf '%s\n' "$names" | sed '/^$/d' | wc -l | tr -d ' ')
|
||||
[ "$count" -eq 1 ] || {
|
||||
echo "expected exactly one qdrant-data volume for compose project $project_name" >&2
|
||||
exit 2
|
||||
}
|
||||
printf '%s\n' "$names" | sed -n '/./{p;q;}'
|
||||
}
|
||||
|
||||
resolve_mountpoint() {
|
||||
mountpoint=$(docker volume inspect --format '{{ .Mountpoint }}' "$1")
|
||||
[ -n "$mountpoint" ] || { echo "docker did not return a qdrant-data mountpoint" >&2; exit 2; }
|
||||
case "$mountpoint" in
|
||||
/*) ;;
|
||||
*) echo "qdrant-data mountpoint is not absolute: $mountpoint" >&2; exit 2 ;;
|
||||
esac
|
||||
[ -d "$mountpoint" ] || { echo "qdrant-data mountpoint is not a directory: $mountpoint" >&2; exit 2; }
|
||||
printf '%s\n' "$mountpoint"
|
||||
}
|
||||
|
||||
volume_name=$(resolve_volume)
|
||||
mountpoint=$(resolve_mountpoint "$volume_name")
|
||||
running_container=$(docker compose --project-name "$project_name" ps --status running -q qdrant)
|
||||
restart_qdrant=0
|
||||
cleanup() {
|
||||
status=$?
|
||||
if [ "${temporary_output:-}" ] && [ -e "${temporary_output:-}" ]; then
|
||||
rm -f "$temporary_output"
|
||||
fi
|
||||
if [ "$restart_qdrant" -eq 1 ]; then
|
||||
docker compose --project-name "$project_name" start qdrant >/dev/null
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
if [ -n "$running_container" ]; then
|
||||
docker compose --project-name "$project_name" stop qdrant >/dev/null
|
||||
restart_qdrant=1
|
||||
fi
|
||||
|
||||
umask 077
|
||||
passfile=$(mktemp "${TMPDIR:-/tmp}/thoth-vector-pgpass.XXXXXX")
|
||||
temporary_output=$(mktemp "$output_dir/.${output_name}.tmp.XXXXXX")
|
||||
cleanup() { rm -f "$passfile" "$temporary_output"; }
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
escaped=$(printf '%s' "$password" | sed 's/\\/\\\\/g; s/:/\\:/g')
|
||||
printf '%s:%s:%s:%s:%s\n' "$host" "$port" "$database" "$user" "$escaped" >"$passfile"
|
||||
chmod 0600 "$passfile"
|
||||
|
||||
PGPASSFILE=$passfile pg_dump \
|
||||
--host="$host" --port="$port" --username="$user" --dbname="$database" \
|
||||
--format=custom --compress=9 \
|
||||
--table=vectors.schema_records --table=vectors.evidence --table=vectors.memory \
|
||||
--table=public.tht_vector_migrations --file="$temporary_output"
|
||||
tar -C "$mountpoint" -cf "$temporary_output" .
|
||||
if ! ln "$temporary_output" "$output"; then
|
||||
echo "refusing to replace backup destination created concurrently: $output" >&2
|
||||
exit 2
|
||||
fi
|
||||
rm -f "$temporary_output"
|
||||
echo "Vector backup written: $output"
|
||||
temporary_output=
|
||||
echo "Qdrant backup written from $volume_name to $output"
|
||||
|
||||
+76
-61
@@ -1,80 +1,95 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)
|
||||
. "$root/scripts/secret-file-utils.sh"
|
||||
|
||||
usage() {
|
||||
echo "usage: $0 --active-host HOST --active-database DB --active-user USER --active-password-file FILE --target-host HOST --target-database DB --target-user USER --target-password-file FILE --input FILE [--active-port PORT] [--target-port PORT] [--force-nonempty]" >&2
|
||||
echo "usage: $0 --project-name NAME --input FILE --confirm-project NAME" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
active_host= active_database= active_user= active_password_file= active_port=5432
|
||||
target_host= target_database= target_user= target_password_file= target_port=5432
|
||||
input= force=0
|
||||
project_name=
|
||||
input=
|
||||
confirm_project=
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--active-host) active_host=${2-}; shift 2 ;;
|
||||
--active-port) active_port=${2-}; shift 2 ;;
|
||||
--active-database) active_database=${2-}; shift 2 ;;
|
||||
--active-user) active_user=${2-}; shift 2 ;;
|
||||
--active-password-file) active_password_file=${2-}; shift 2 ;;
|
||||
--target-host) target_host=${2-}; shift 2 ;;
|
||||
--target-port) target_port=${2-}; shift 2 ;;
|
||||
--target-database) target_database=${2-}; shift 2 ;;
|
||||
--target-user) target_user=${2-}; shift 2 ;;
|
||||
--target-password-file) target_password_file=${2-}; shift 2 ;;
|
||||
--project-name) project_name=${2-}; shift 2 ;;
|
||||
--input) input=${2-}; shift 2 ;;
|
||||
--force-nonempty) force=1; shift ;;
|
||||
--confirm-project) confirm_project=${2-}; shift 2 ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
done
|
||||
for value in "$active_host" "$active_database" "$active_user" "$active_password_file" \
|
||||
"$target_host" "$target_database" "$target_user" "$target_password_file" "$input"; do
|
||||
[ -n "$value" ] || usage
|
||||
done
|
||||
|
||||
[ -n "$project_name" ] && [ -n "$input" ] && [ -n "$confirm_project" ] || usage
|
||||
[ "$confirm_project" = "$project_name" ] || {
|
||||
echo "restore confirmation must match --project-name exactly" >&2
|
||||
exit 2
|
||||
}
|
||||
[ -r "$input" ] || { echo "backup input is not readable" >&2; exit 2; }
|
||||
validate_secret_file "$active_password_file" "active source password file"
|
||||
validate_secret_file "$target_password_file" "target password file"
|
||||
|
||||
umask 077
|
||||
active_pass=$(mktemp "${TMPDIR:-/tmp}/thoth-vector-active-pgpass.XXXXXX")
|
||||
target_pass=$(mktemp "${TMPDIR:-/tmp}/thoth-vector-target-pgpass.XXXXXX")
|
||||
cleanup() { rm -f "$active_pass" "$target_pass"; }
|
||||
resolve_volume() {
|
||||
names=$(docker volume ls \
|
||||
--filter "label=com.docker.compose.project=$project_name" \
|
||||
--filter "label=com.docker.compose.volume=qdrant-data" \
|
||||
--format '{{.Name}}')
|
||||
count=$(printf '%s\n' "$names" | sed '/^$/d' | wc -l | tr -d ' ')
|
||||
[ "$count" -eq 1 ] || {
|
||||
echo "expected exactly one qdrant-data volume for compose project $project_name" >&2
|
||||
exit 2
|
||||
}
|
||||
printf '%s\n' "$names" | sed -n '/./{p;q;}'
|
||||
}
|
||||
|
||||
resolve_mountpoint() {
|
||||
mountpoint=$(docker volume inspect --format '{{ .Mountpoint }}' "$1")
|
||||
[ -n "$mountpoint" ] || { echo "docker did not return a qdrant-data mountpoint" >&2; exit 2; }
|
||||
case "$mountpoint" in
|
||||
/*) ;;
|
||||
*) echo "qdrant-data mountpoint is not absolute: $mountpoint" >&2; exit 2 ;;
|
||||
esac
|
||||
[ -d "$mountpoint" ] || { echo "qdrant-data mountpoint is not a directory: $mountpoint" >&2; exit 2; }
|
||||
printf '%s\n' "$mountpoint"
|
||||
}
|
||||
|
||||
volume_name=$(resolve_volume)
|
||||
mountpoint=$(resolve_mountpoint "$volume_name")
|
||||
running_container=$(docker compose --project-name "$project_name" ps --status running -q qdrant)
|
||||
restart_qdrant=0
|
||||
staging_dir=
|
||||
extract_dir=
|
||||
|
||||
cleanup() {
|
||||
status=$?
|
||||
if [ "$status" -ne 0 ] && [ -n "${staging_dir:-}" ] && [ -d "${staging_dir:-}" ]; then
|
||||
find "$mountpoint" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +
|
||||
find "$staging_dir" -mindepth 1 -maxdepth 1 -exec mv {} "$mountpoint"/ \;
|
||||
fi
|
||||
if [ -n "${staging_dir:-}" ] && [ -d "${staging_dir:-}" ]; then
|
||||
rm -rf "$staging_dir"
|
||||
fi
|
||||
if [ -n "${extract_dir:-}" ] && [ -d "${extract_dir:-}" ]; then
|
||||
rm -rf "$extract_dir"
|
||||
fi
|
||||
if [ "$restart_qdrant" -eq 1 ]; then
|
||||
docker compose --project-name "$project_name" start qdrant >/dev/null
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
make_passfile() {
|
||||
secret=$(read_secret_file "$5" "database password file")
|
||||
escaped=$(printf '%s' "$secret" | sed 's/\\/\\\\/g; s/:/\\:/g')
|
||||
printf '%s:%s:%s:%s:%s\n' "$1" "$2" "$3" "$4" "$escaped" >"$6"
|
||||
chmod 0600 "$6"
|
||||
}
|
||||
make_passfile "$active_host" "$active_port" "$active_database" "$active_user" \
|
||||
"$active_password_file" "$active_pass"
|
||||
make_passfile "$target_host" "$target_port" "$target_database" "$target_user" \
|
||||
"$target_password_file" "$target_pass"
|
||||
|
||||
identity_sql="SELECT system_identifier::text FROM pg_control_system()"
|
||||
active_identity=$(PGPASSFILE=$active_pass psql -XAt --host="$active_host" --port="$active_port" \
|
||||
--username="$active_user" --dbname="$active_database" --command="$identity_sql")
|
||||
target_identity=$(PGPASSFILE=$target_pass psql -XAt --host="$target_host" --port="$target_port" \
|
||||
--username="$target_user" --dbname="$target_database" --command="$identity_sql")
|
||||
[ "$active_identity" != "$target_identity" ] || {
|
||||
echo "refusing restore: active source and target are on the same PostgreSQL cluster" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
object_count=$(PGPASSFILE=$target_pass psql -XAt --host="$target_host" --port="$target_port" \
|
||||
--username="$target_user" --dbname="$target_database" --command="
|
||||
SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace
|
||||
WHERE (n.nspname='vectors' OR (n.nspname='public' AND c.relname='tht_vector_migrations'))
|
||||
AND c.relkind IN ('r','p','S','v','m');")
|
||||
if [ "$object_count" != 0 ] && [ "$force" != 1 ]; then
|
||||
echo "refusing restore into non-empty target; use --force-nonempty explicitly" >&2
|
||||
exit 2
|
||||
if [ -n "$running_container" ]; then
|
||||
docker compose --project-name "$project_name" stop qdrant >/dev/null
|
||||
restart_qdrant=1
|
||||
fi
|
||||
|
||||
PGPASSFILE=$target_pass pg_restore --exit-on-error --single-transaction \
|
||||
--clean --if-exists --no-owner \
|
||||
--host="$target_host" --port="$target_port" --username="$target_user" \
|
||||
--dbname="$target_database" "$input"
|
||||
echo "Vector restore completed into explicit target $target_host:$target_port/$target_database"
|
||||
parent_dir=$(dirname "$mountpoint")
|
||||
staging_dir=$(mktemp -d "$parent_dir/.qdrant-restore-staging.XXXXXX")
|
||||
extract_dir=$(mktemp -d "${TMPDIR:-/tmp}/qdrant-restore.XXXXXX")
|
||||
tar -C "$extract_dir" -xf "$input"
|
||||
|
||||
find "$mountpoint" -mindepth 1 -maxdepth 1 -exec mv {} "$staging_dir"/ \;
|
||||
find "$extract_dir" -mindepth 1 -maxdepth 1 -exec mv {} "$mountpoint"/ \;
|
||||
|
||||
rm -rf "$staging_dir"
|
||||
staging_dir=
|
||||
rm -rf "$extract_dir"
|
||||
extract_dir=
|
||||
echo "Qdrant restore completed into $volume_name for compose project $project_name"
|
||||
|
||||
Reference in New Issue
Block a user