594 lines
22 KiB
Bash
Executable File
594 lines
22 KiB
Bash
Executable File
#!/bin/sh
|
|
set -eu
|
|
|
|
cd "$(dirname "$0")/.."
|
|
tmp=$(mktemp -d)
|
|
holder_pid=
|
|
holder_release="$tmp/holder-release"
|
|
cleanup() {
|
|
touch "$holder_release"
|
|
if [ -n "${holder_pid:-}" ]; then
|
|
kill "$holder_pid" 2>/dev/null || true
|
|
wait "$holder_pid" 2>/dev/null || true
|
|
fi
|
|
rm -rf "$tmp"
|
|
}
|
|
trap cleanup EXIT HUP INT TERM
|
|
fakebin="$tmp/bin"
|
|
mkdir "$fakebin"
|
|
|
|
project="thoth-task8"
|
|
volume_name="${project}_qdrant-data"
|
|
volume_root="$tmp/volumes"
|
|
volume_dir="$volume_root/$volume_name"
|
|
mkdir -p "$volume_dir/collections/demo"
|
|
printf '%s' before-backup >"$volume_dir/collections/demo/state.json"
|
|
|
|
cat >"$fakebin/docker" <<'SH'
|
|
#!/bin/sh
|
|
set -eu
|
|
log_file=${DOCKER_LOG:?}
|
|
printf '%s\n' "$*" >>"$log_file"
|
|
|
|
volume_dir_for() {
|
|
printf '%s/%s\n' "${VOLUME_ROOT:?}" "$1"
|
|
}
|
|
|
|
run_backup() {
|
|
volume_name=$1
|
|
backup_dir=$2
|
|
output_name=$3
|
|
if [ -n "${RUN_BLOCK_READY:-}" ]; then
|
|
: >"$RUN_BLOCK_READY"
|
|
attempts=0
|
|
while [ ! -e "${RUN_BLOCK_RELEASE:?}" ]; do
|
|
attempts=$((attempts + 1))
|
|
[ "$attempts" -lt 400 ] || exit 24
|
|
sleep 0.05
|
|
done
|
|
fi
|
|
staging=$(mktemp -d "${TMPDIR:-/tmp}/fake-qdrant-backup.XXXXXX")
|
|
mkdir -p "$staging/payload"
|
|
cat >"$staging/manifest.env" <<EOF
|
|
format=thothii-qdrant-backup-v1
|
|
project_name=${PROJECT_NAME:?}
|
|
volume_name=$volume_name
|
|
volume_role=qdrant-data
|
|
helper_image=${HELPER_IMAGE:?}
|
|
created_utc=2026-08-08T17:35:36Z
|
|
EOF
|
|
cp -R "$(volume_dir_for "$volume_name")"/. "$staging/payload"/
|
|
tar -C "$staging" -cf "$backup_dir/$output_name" manifest.env payload
|
|
rm -rf "$staging"
|
|
}
|
|
|
|
lock_state_root=${LOCK_ROOT:-$(dirname "${VOLUME_ROOT:?}")/locks}
|
|
mkdir -p "$lock_state_root"
|
|
|
|
if [ "$1" = create ]; then
|
|
shift
|
|
lock_name=
|
|
lock_owner=
|
|
lock_volume=
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--name) lock_name=$2; shift 2 ;;
|
|
--label)
|
|
case "$2" in
|
|
com.thothii.qdrant-operation-owner=*) lock_owner=${2#*=} ;;
|
|
com.thothii.qdrant-operation-volume=*) lock_volume=${2#*=} ;;
|
|
esac
|
|
shift 2
|
|
;;
|
|
*) shift ;;
|
|
esac
|
|
done
|
|
[ -n "$lock_name" ] && [ -n "$lock_owner" ] && [ -n "$lock_volume" ] || exit 25
|
|
state="$lock_state_root/$lock_name"
|
|
mkdir "$state" 2>/dev/null || exit 1
|
|
printf '%s\n' "$lock_owner" >"$state/owner"
|
|
printf '%s\n' "$lock_volume" >"$state/volume"
|
|
printf '%s\n' "$lock_name"
|
|
exit 0
|
|
fi
|
|
|
|
if [ "$1" = inspect ]; then
|
|
for lock_name in "$@"; do :; done
|
|
state="$lock_state_root/$lock_name"
|
|
[ -d "$state" ] || exit 1
|
|
printf '%s %s\n' "$(cat "$state/owner")" "$(cat "$state/volume")"
|
|
exit 0
|
|
fi
|
|
|
|
if [ "$1" = rm ]; then
|
|
for lock_name in "$@"; do :; done
|
|
state="$lock_state_root/$lock_name"
|
|
[ -d "$state" ] || exit 1
|
|
rm -rf "$state"
|
|
exit 0
|
|
fi
|
|
|
|
run_restore() {
|
|
volume_name=$1
|
|
backup_dir=$2
|
|
input_name=$3
|
|
volume_dir=$(volume_dir_for "$volume_name")
|
|
snapshot=$(mktemp -d "${TMPDIR:-/tmp}/fake-qdrant-rollback.XXXXXX")
|
|
extract=$(mktemp -d "${TMPDIR:-/tmp}/fake-qdrant-extract.XXXXXX")
|
|
cp -R "$volume_dir"/. "$snapshot"/
|
|
find "$volume_dir" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +
|
|
if [ "${RUN_FAIL_AFTER_CLEANUP:-0}" = 1 ]; then
|
|
cp -R "$snapshot"/. "$volume_dir"/
|
|
rm -rf "$snapshot" "$extract"
|
|
exit 17
|
|
fi
|
|
tar -C "$extract" -xf "$backup_dir/$input_name"
|
|
if [ -n "${RESTORE_CAPTURE_FILE:-}" ]; then
|
|
cp "$backup_dir/$input_name" "$RESTORE_CAPTURE_FILE"
|
|
fi
|
|
cp -R "$extract/payload"/. "$volume_dir"/
|
|
rm -rf "$snapshot" "$extract"
|
|
}
|
|
|
|
if [ "$1" = volume ] && [ "$2" = ls ]; then
|
|
if [ "${VOLUME_LS_OUTPUT:-single}" = multiple ]; then
|
|
printf '%s\n%s\n' "${PROJECT_NAME}_qdrant-data" "${PROJECT_NAME}_qdrant-data-copy"
|
|
exit 0
|
|
fi
|
|
printf '%s\n' "${PROJECT_NAME}_qdrant-data"
|
|
exit 0
|
|
fi
|
|
|
|
if [ "$1" = volume ] && [ "$2" = inspect ]; then
|
|
printf '%s %s %s\n' \
|
|
"${PROJECT_NAME}_qdrant-data" \
|
|
"${PROJECT_NAME}" \
|
|
"qdrant-data"
|
|
exit 0
|
|
fi
|
|
|
|
if [ "$1" = compose ] && [ "$2" = --project-name ]; then
|
|
case "$4" in
|
|
ps)
|
|
qdrant_state=${QDRANT_RUNNING:-1}
|
|
if [ -n "${QDRANT_STATE_FILE:-}" ] && [ -f "$QDRANT_STATE_FILE" ]; then
|
|
qdrant_state=$(cat "$QDRANT_STATE_FILE")
|
|
fi
|
|
if [ "$qdrant_state" = 1 ]; then
|
|
printf '%s\n' qdrant-container
|
|
fi
|
|
exit 0
|
|
;;
|
|
stop)
|
|
if [ -n "${QDRANT_STATE_FILE:-}" ]; then
|
|
printf '%s\n' 0 >"$QDRANT_STATE_FILE"
|
|
fi
|
|
if [ "${STOP_MARKS_STOPPED_THEN_FAIL:-0}" = 1 ]; then
|
|
exit 42
|
|
fi
|
|
exit 0
|
|
;;
|
|
start)
|
|
if [ -n "${QDRANT_STATE_FILE:-}" ]; then
|
|
printf '%s\n' 1 >"$QDRANT_STATE_FILE"
|
|
fi
|
|
exit 0
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
if [ "$1" = run ]; then
|
|
shift
|
|
volume_mount=
|
|
backup_mount=
|
|
backup_readonly=0
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--mount)
|
|
case "$2" in
|
|
type=volume,src=*,dst=/qdrant-data*)
|
|
volume_mount=$2
|
|
;;
|
|
type=bind,src=*,dst=/backup*)
|
|
backup_mount=$2
|
|
;;
|
|
type=bind,src=*,dst=/restore-backup,readonly)
|
|
backup_mount=$2
|
|
backup_readonly=1
|
|
;;
|
|
esac
|
|
shift 2
|
|
;;
|
|
--rm)
|
|
shift
|
|
;;
|
|
*)
|
|
break
|
|
;;
|
|
esac
|
|
done
|
|
image=$1
|
|
shift
|
|
last_arg=
|
|
for arg in "$@"; do last_arg=$arg; done
|
|
volume_name=$(printf '%s' "$volume_mount" | sed -n 's/.*src=\([^,]*\),dst=.*/\1/p')
|
|
backup_src=$(printf '%s' "$backup_mount" | sed -n 's/.*src=\([^,]*\),dst=.*/\1/p')
|
|
[ "$image" = "${HELPER_IMAGE:?}" ] || exit 19
|
|
[ -n "$volume_name" ] || exit 20
|
|
[ -n "$backup_src" ] || exit 21
|
|
case "$backup_mount" in
|
|
*dst=/backup)
|
|
run_backup "$volume_name" "$backup_src" "$last_arg"
|
|
;;
|
|
*dst=/restore-backup,readonly)
|
|
[ "$backup_readonly" -eq 1 ] || exit 22
|
|
run_restore "$volume_name" "$backup_src" "$last_arg"
|
|
;;
|
|
*)
|
|
exit 23
|
|
;;
|
|
esac
|
|
exit 0
|
|
fi
|
|
|
|
exit 0
|
|
SH
|
|
chmod 0755 "$fakebin/docker"
|
|
|
|
make_foreign_archive() {
|
|
archive_path=$1
|
|
python - "$archive_path" <<'PY'
|
|
import io, tarfile, sys
|
|
archive = sys.argv[1]
|
|
with tarfile.open(archive, "w") as tf:
|
|
manifest = b"format=wrong-format\nproject_name=foreign\nvolume_name=foreign_qdrant-data\nvolume_role=qdrant-data\nhelper_image=foreign\n"
|
|
info = tarfile.TarInfo("manifest.env")
|
|
info.size = len(manifest)
|
|
tf.addfile(info, io.BytesIO(manifest))
|
|
data = b"x"
|
|
info = tarfile.TarInfo("../escape")
|
|
info.size = len(data)
|
|
tf.addfile(info, io.BytesIO(data))
|
|
PY
|
|
}
|
|
|
|
make_symlink_archive() {
|
|
archive_path=$1
|
|
python - "$archive_path" <<'PY'
|
|
import io, tarfile, sys
|
|
archive = sys.argv[1]
|
|
with tarfile.open(archive, "w") as tf:
|
|
manifest = b"format=thothii-qdrant-backup-v1\nproject_name=thoth-task8\nvolume_name=thoth-task8_qdrant-data\nvolume_role=qdrant-data\nhelper_image=qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c\n"
|
|
info = tarfile.TarInfo("manifest.env")
|
|
info.size = len(manifest)
|
|
tf.addfile(info, io.BytesIO(manifest))
|
|
directory = tarfile.TarInfo("payload")
|
|
directory.type = tarfile.DIRTYPE
|
|
tf.addfile(directory)
|
|
link = tarfile.TarInfo("payload/collections-link")
|
|
link.type = tarfile.SYMTYPE
|
|
link.linkname = "/etc/passwd"
|
|
tf.addfile(link)
|
|
PY
|
|
}
|
|
|
|
make_duplicate_manifest_archive() {
|
|
archive_path=$1
|
|
python - "$archive_path" <<'PY'
|
|
import io, tarfile, sys
|
|
archive = sys.argv[1]
|
|
with tarfile.open(archive, "w") as tf:
|
|
manifest = b"""format=thothii-qdrant-backup-v1
|
|
project_name=thoth-task8
|
|
project_name=thoth-task8
|
|
volume_name=thoth-task8_qdrant-data
|
|
volume_role=qdrant-data
|
|
helper_image=qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c
|
|
created_utc=2026-08-08T17:35:36Z
|
|
"""
|
|
info = tarfile.TarInfo("manifest.env")
|
|
info.size = len(manifest)
|
|
tf.addfile(info, io.BytesIO(manifest))
|
|
directory = tarfile.TarInfo("payload")
|
|
directory.type = tarfile.DIRTYPE
|
|
tf.addfile(directory)
|
|
payload = b"dup"
|
|
info = tarfile.TarInfo("payload/dup.txt")
|
|
info.size = len(payload)
|
|
tf.addfile(info, io.BytesIO(payload))
|
|
PY
|
|
}
|
|
|
|
backup_output="$tmp/qdrant-backup.tar"
|
|
docker_log="$tmp/docker-backup.log"
|
|
lock_name="${project}_qdrant-data-operation-lock"
|
|
PATH="$fakebin:$PATH" DOCKER_LOG="$docker_log" PROJECT_NAME="$project" VOLUME_ROOT="$volume_root" \
|
|
HELPER_IMAGE="qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c" \
|
|
./scripts/vector-backup.sh --project-name "$project" --output "$backup_output" >/dev/null
|
|
test -s "$backup_output"
|
|
tar -tf "$backup_output" | grep -qx 'manifest.env'
|
|
tar -tf "$backup_output" | grep -qx 'payload/collections/demo/state.json'
|
|
tar -xOf "$backup_output" manifest.env | grep -qx 'format=thothii-qdrant-backup-v1'
|
|
tar -xOf "$backup_output" manifest.env | grep -qx "project_name=$project"
|
|
tar -xOf "$backup_output" manifest.env | grep -qx "volume_name=$volume_name"
|
|
grep -q "run --rm --mount type=volume,src=$volume_name,dst=/qdrant-data,readonly --mount type=bind,src=$tmp,dst=/backup" "$docker_log"
|
|
grep -q "compose --project-name $project stop qdrant" "$docker_log"
|
|
grep -q "compose --project-name $project start qdrant" "$docker_log"
|
|
grep -q "create --name $lock_name" "$docker_log"
|
|
grep -q "rm -f $lock_name" "$docker_log"
|
|
test ! -e "$tmp/locks/$lock_name"
|
|
lock_line=$(grep -n "create --name $lock_name" "$docker_log" | sed -n '1s/:.*//p')
|
|
volume_line=$(grep -n '^volume ls ' "$docker_log" | sed -n '1s/:.*//p')
|
|
[ "$lock_line" -lt "$volume_line" ] || {
|
|
echo "backup resolved volume state before acquiring the operation lock" >&2
|
|
exit 1
|
|
}
|
|
if grep -q "volume inspect --format {{ .Mountpoint }}" "$docker_log"; then
|
|
echo "backup consulted Docker mountpoints" >&2
|
|
exit 1
|
|
fi
|
|
if grep -q "prune" "$docker_log"; then
|
|
echo "backup attempted global docker cleanup" >&2
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p "$tmp/locks/$lock_name"
|
|
printf '%s\n' foreign-owner >"$tmp/locks/$lock_name/owner"
|
|
printf '%s\n' "$volume_name" >"$tmp/locks/$lock_name/volume"
|
|
foreign_lock_log="$tmp/foreign-lock.log"
|
|
if PATH="$fakebin:$PATH" DOCKER_LOG="$foreign_lock_log" PROJECT_NAME="$project" VOLUME_ROOT="$volume_root" \
|
|
HELPER_IMAGE="qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c" \
|
|
./scripts/vector-backup.sh --project-name "$project" --output "$tmp/foreign-lock.tar" \
|
|
>"$tmp/foreign-lock.out" 2>"$tmp/foreign-lock.err"; then
|
|
echo "backup ignored a foreign operation lock" >&2
|
|
exit 1
|
|
fi
|
|
grep -q 'operation already in progress' "$tmp/foreign-lock.err"
|
|
test "$(cat "$tmp/locks/$lock_name/owner")" = foreign-owner
|
|
rm -rf "$tmp/locks/$lock_name"
|
|
|
|
holder_ready="$tmp/holder-ready"
|
|
holder_output="$tmp/holder.tar"
|
|
holder_log="$tmp/holder.log"
|
|
PATH="$fakebin:$PATH" DOCKER_LOG="$holder_log" PROJECT_NAME="$project" VOLUME_ROOT="$volume_root" \
|
|
RUN_BLOCK_READY="$holder_ready" RUN_BLOCK_RELEASE="$holder_release" \
|
|
HELPER_IMAGE="qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c" \
|
|
./scripts/vector-backup.sh --project-name "$project" --output "$holder_output" >/dev/null 2>"$tmp/holder.err" &
|
|
holder_pid=$!
|
|
attempts=0
|
|
while [ ! -e "$holder_ready" ]; do
|
|
attempts=$((attempts + 1))
|
|
[ "$attempts" -lt 400 ] || { echo "lock holder did not reach copy phase" >&2; exit 1; }
|
|
sleep 0.05
|
|
done
|
|
|
|
for contender in backup restore; do
|
|
contender_log="$tmp/contender-$contender.log"
|
|
if [ "$contender" = backup ]; then
|
|
set -- ./scripts/vector-backup.sh --project-name "$project" --output "$tmp/contender.tar"
|
|
else
|
|
set -- ./scripts/vector-restore.sh --project-name "$project" --input "$backup_output" --confirm-project "$project"
|
|
fi
|
|
if PATH="$fakebin:$PATH" DOCKER_LOG="$contender_log" PROJECT_NAME="$project" VOLUME_ROOT="$volume_root" \
|
|
HELPER_IMAGE="qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c" \
|
|
"$@" >"$tmp/contender-$contender.out" 2>"$tmp/contender-$contender.err"; then
|
|
echo "$contender interleaved with an active qdrant operation" >&2
|
|
exit 1
|
|
fi
|
|
grep -q 'operation already in progress' "$tmp/contender-$contender.err"
|
|
if grep -Eq '^volume (ls|inspect)|^compose .* (stop|start) qdrant|^run ' "$contender_log"; then
|
|
echo "$contender touched qdrant state after lock contention" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
touch "$holder_release"
|
|
wait "$holder_pid"
|
|
holder_pid=
|
|
test -s "$holder_output"
|
|
test ! -e "$tmp/locks/$lock_name"
|
|
|
|
existing="$tmp/existing.tar"
|
|
printf '%s' sentinel >"$existing"
|
|
if PATH="$fakebin:$PATH" DOCKER_LOG="$tmp/existing.log" PROJECT_NAME="$project" VOLUME_ROOT="$volume_root" \
|
|
HELPER_IMAGE="qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c" \
|
|
./scripts/vector-backup.sh --project-name "$project" --output "$existing" >"$tmp/existing.out" 2>"$tmp/existing.err"; then
|
|
echo "backup overwrote an existing archive" >&2
|
|
exit 1
|
|
fi
|
|
test "$(cat "$existing")" = sentinel
|
|
|
|
if PATH="$fakebin:$PATH" DOCKER_LOG="$tmp/ambiguous.log" PROJECT_NAME="$project" VOLUME_ROOT="$volume_root" \
|
|
VOLUME_LS_OUTPUT=multiple HELPER_IMAGE="qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c" \
|
|
./scripts/vector-backup.sh --project-name "$project" --output "$tmp/ambiguous.tar" >"$tmp/ambiguous.out" 2>"$tmp/ambiguous.err"; then
|
|
echo "backup accepted an ambiguous qdrant-data target" >&2
|
|
exit 1
|
|
fi
|
|
grep -q 'exactly one qdrant-data volume' "$tmp/ambiguous.err"
|
|
|
|
foreign_archive="$tmp/foreign.tar"
|
|
make_foreign_archive "$foreign_archive"
|
|
printf '%s' before-malicious >"$volume_dir/collections/demo/state.json"
|
|
foreign_log="$tmp/foreign.log"
|
|
: >"$foreign_log"
|
|
if PATH="$fakebin:$PATH" DOCKER_LOG="$foreign_log" PROJECT_NAME="$project" VOLUME_ROOT="$volume_root" \
|
|
MOUNTPOINT="/" HELPER_IMAGE="qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c" \
|
|
./scripts/vector-restore.sh --project-name "$project" --input "$foreign_archive" --confirm-project "$project" \
|
|
>"$tmp/foreign.out" 2>"$tmp/foreign.err"; then
|
|
echo "restore accepted a malformed or foreign archive" >&2
|
|
exit 1
|
|
fi
|
|
grep -Eq 'manifest|archive|foreign|unexpected|traversal' "$tmp/foreign.err"
|
|
test "$(cat "$volume_dir/collections/demo/state.json")" = before-malicious
|
|
if grep -q "compose --project-name $project stop qdrant" "$foreign_log"; then
|
|
echo "restore stopped qdrant before archive validation" >&2
|
|
exit 1
|
|
fi
|
|
if grep -q '^run ' "$foreign_log"; then
|
|
echo "restore ran helper container before archive validation" >&2
|
|
exit 1
|
|
fi
|
|
if grep -q "volume inspect --format {{ .Mountpoint }}" "$foreign_log"; then
|
|
echo "restore consulted Docker mountpoints during malformed archive rejection" >&2
|
|
exit 1
|
|
fi
|
|
|
|
symlink_archive="$tmp/symlink.tar"
|
|
make_symlink_archive "$symlink_archive"
|
|
symlink_log="$tmp/symlink.log"
|
|
: >"$symlink_log"
|
|
if PATH="$fakebin:$PATH" DOCKER_LOG="$symlink_log" PROJECT_NAME="$project" VOLUME_ROOT="$volume_root" \
|
|
HELPER_IMAGE="qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c" \
|
|
./scripts/vector-restore.sh --project-name "$project" --input "$symlink_archive" --confirm-project "$project" \
|
|
>"$tmp/symlink.out" 2>"$tmp/symlink.err"; then
|
|
echo "restore accepted a symlink-bearing archive" >&2
|
|
exit 1
|
|
fi
|
|
grep -Eq 'symlink|hardlink|device|fifo|socket|archive' "$tmp/symlink.err"
|
|
if grep -q "compose --project-name $project stop qdrant" "$symlink_log"; then
|
|
echo "restore stopped qdrant before symlink archive rejection" >&2
|
|
exit 1
|
|
fi
|
|
|
|
duplicate_archive="$tmp/duplicate.tar"
|
|
make_duplicate_manifest_archive "$duplicate_archive"
|
|
duplicate_log="$tmp/duplicate.log"
|
|
: >"$duplicate_log"
|
|
if PATH="$fakebin:$PATH" DOCKER_LOG="$duplicate_log" PROJECT_NAME="$project" VOLUME_ROOT="$volume_root" \
|
|
HELPER_IMAGE="qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c" \
|
|
./scripts/vector-restore.sh --project-name "$project" --input "$duplicate_archive" --confirm-project "$project" \
|
|
>"$tmp/duplicate.out" 2>"$tmp/duplicate.err"; then
|
|
echo "restore accepted duplicate manifest keys" >&2
|
|
exit 1
|
|
fi
|
|
grep -Eq 'duplicate|manifest' "$tmp/duplicate.err"
|
|
if grep -q "compose --project-name $project stop qdrant" "$duplicate_log"; then
|
|
echo "restore stopped qdrant before duplicate-manifest rejection" >&2
|
|
exit 1
|
|
fi
|
|
if grep -q '^run ' "$duplicate_log"; then
|
|
echo "restore ran helper before duplicate-manifest rejection" >&2
|
|
exit 1
|
|
fi
|
|
|
|
restore_source="$tmp/restore-source"
|
|
mkdir -p "$restore_source/payload/collections/demo"
|
|
cat >"$restore_source/manifest.env" <<EOF
|
|
format=thothii-qdrant-backup-v1
|
|
project_name=$project
|
|
volume_name=$volume_name
|
|
volume_role=qdrant-data
|
|
helper_image=qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c
|
|
created_utc=2026-08-08T17:35:36Z
|
|
EOF
|
|
printf '%s' restored >"$restore_source/payload/collections/demo/state.json"
|
|
restore_input="$tmp/restore.tar"
|
|
tar -C "$restore_source" -cf "$restore_input" manifest.env payload
|
|
|
|
if PATH="$fakebin:$PATH" DOCKER_LOG="$tmp/confirm.log" PROJECT_NAME="$project" VOLUME_ROOT="$volume_root" \
|
|
HELPER_IMAGE="qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c" \
|
|
./scripts/vector-restore.sh --project-name "$project" --input "$restore_input" --confirm-project wrong-project \
|
|
>"$tmp/confirm.out" 2>"$tmp/confirm.err"; then
|
|
echo "restore skipped explicit project confirmation" >&2
|
|
exit 1
|
|
fi
|
|
grep -q 'confirmation must match --project-name exactly' "$tmp/confirm.err"
|
|
|
|
printf '%s' modified-live >"$volume_dir/collections/demo/state.json"
|
|
restore_log="$tmp/restore-ok.log"
|
|
restore_capture="$tmp/restore-captured.tar"
|
|
PATH="$fakebin:$PATH" DOCKER_LOG="$restore_log" PROJECT_NAME="$project" VOLUME_ROOT="$volume_root" \
|
|
RESTORE_CAPTURE_FILE="$restore_capture" \
|
|
HELPER_IMAGE="qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c" \
|
|
./scripts/vector-restore.sh --project-name "$project" --input "$restore_input" --confirm-project "$project" >/dev/null &
|
|
restore_pid=$!
|
|
sleep 1
|
|
printf '%s' swapped >"$restore_source/payload/collections/demo/state.json"
|
|
tar -C "$restore_source" -cf "$restore_input" manifest.env payload
|
|
wait "$restore_pid"
|
|
test "$(cat "$volume_dir/collections/demo/state.json")" = restored
|
|
tar -xOf "$restore_capture" payload/collections/demo/state.json | grep -qx 'restored'
|
|
grep -q "compose --project-name $project stop qdrant" "$restore_log"
|
|
grep -q "compose --project-name $project start qdrant" "$restore_log"
|
|
grep -Eq "run --rm --mount type=volume,src=$volume_name,dst=/qdrant-data --mount type=bind,src=.*/qdrant-archive\.[^,]*,dst=/restore-backup,readonly" "$restore_log"
|
|
if grep -q "src=$tmp,dst=/restore-backup,readonly" "$restore_log"; then
|
|
echo "restore mounted the original archive directory instead of a private copy" >&2
|
|
exit 1
|
|
fi
|
|
if grep -q "volume inspect --format {{ .Mountpoint }}" "$restore_log"; then
|
|
echo "restore consulted Docker mountpoints during normal restore" >&2
|
|
exit 1
|
|
fi
|
|
if grep -q "prune" "$restore_log"; then
|
|
echo "restore attempted global docker cleanup" >&2
|
|
exit 1
|
|
fi
|
|
if find "$tmp" -maxdepth 1 -type d -name 'qdrant-archive.*' | grep -q .; then
|
|
echo "restore left its private archive-copy directory behind" >&2
|
|
exit 1
|
|
fi
|
|
|
|
printf '%s' rollback-source >"$volume_dir/collections/demo/state.json"
|
|
rollback_log="$tmp/restore-rollback.log"
|
|
if PATH="$fakebin:$PATH" DOCKER_LOG="$rollback_log" PROJECT_NAME="$project" VOLUME_ROOT="$volume_root" \
|
|
RUN_FAIL_AFTER_CLEANUP=1 HELPER_IMAGE="qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c" \
|
|
./scripts/vector-restore.sh --project-name "$project" --input "$restore_input" --confirm-project "$project" \
|
|
>"$tmp/rollback.out" 2>"$tmp/rollback.err"; then
|
|
echo "restore ignored helper failure after cleanup" >&2
|
|
exit 1
|
|
fi
|
|
test "$(cat "$volume_dir/collections/demo/state.json")" = rollback-source
|
|
grep -q "compose --project-name $project stop qdrant" "$rollback_log"
|
|
grep -q "compose --project-name $project start qdrant" "$rollback_log"
|
|
test ! -e "$tmp/locks/$lock_name"
|
|
|
|
assert_ambiguous_stop_reconciled() {
|
|
operation=$1
|
|
operation_log="$tmp/$operation-ambiguous-stop.log"
|
|
operation_state="$tmp/$operation-qdrant-state"
|
|
foreign_lock="$tmp/locks/foreign-$operation-lock"
|
|
printf '%s\n' 1 >"$operation_state"
|
|
mkdir -p "$foreign_lock"
|
|
printf '%s\n' foreign-owner >"$foreign_lock/owner"
|
|
printf '%s\n' foreign-volume >"$foreign_lock/volume"
|
|
|
|
if [ "$operation" = backup ]; then
|
|
set -- ./scripts/vector-backup.sh --project-name "$project" \
|
|
--output "$tmp/ambiguous-stop-backup.tar"
|
|
else
|
|
set -- ./scripts/vector-restore.sh --project-name "$project" \
|
|
--input "$restore_input" --confirm-project "$project"
|
|
fi
|
|
|
|
if PATH="$fakebin:$PATH" DOCKER_LOG="$operation_log" PROJECT_NAME="$project" \
|
|
VOLUME_ROOT="$volume_root" QDRANT_STATE_FILE="$operation_state" \
|
|
STOP_MARKS_STOPPED_THEN_FAIL=1 \
|
|
HELPER_IMAGE="qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c" \
|
|
"$@" >"$tmp/$operation-ambiguous-stop.out" \
|
|
2>"$tmp/$operation-ambiguous-stop.err"; then
|
|
echo "$operation hid an ambiguous qdrant stop failure" >&2
|
|
exit 1
|
|
fi
|
|
|
|
grep -q "compose --project-name $project stop qdrant" "$operation_log"
|
|
grep -q "compose --project-name $project start qdrant" "$operation_log"
|
|
test "$(cat "$operation_state")" = 1
|
|
test ! -e "$tmp/locks/$lock_name"
|
|
test "$(cat "$foreign_lock/owner")" = foreign-owner
|
|
test "$(cat "$foreign_lock/volume")" = foreign-volume
|
|
if grep -q "rm -f foreign-$operation-lock" "$operation_log"; then
|
|
echo "$operation removed a foreign operation lock" >&2
|
|
exit 1
|
|
fi
|
|
start_line=$(grep -n "compose --project-name $project start qdrant" "$operation_log" | sed -n '1s/:.*//p')
|
|
release_line=$(grep -n "rm -f $lock_name" "$operation_log" | sed -n '1s/:.*//p')
|
|
[ "$start_line" -lt "$release_line" ] || {
|
|
echo "$operation released its lock before qdrant reconciliation" >&2
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
assert_ambiguous_stop_reconciled backup
|
|
assert_ambiguous_stop_reconciled restore
|
|
|
|
echo "qdrant backup/restore archive validation, scoped helper execution, and rollback safety passed."
|