fix: close qdrant archive restore race

This commit is contained in:
2026-08-08 19:43:54 +02:00
parent afaab5931f
commit af1e922a48
2 changed files with 136 additions and 23 deletions
+69 -2
View File
@@ -57,6 +57,9 @@ run_restore() {
exit 17
fi
tar -C "$extract" -xf "$backup_dir/$input_name"
if [ -n "${RESTORE_CAPTURE_FILE:-}" ]; then
cp "$backup_dir/$input_name" "$RESTORE_CAPTURE_FILE"
fi
cp -R "$extract/payload"/. "$volume_dir"/
rm -rf "$snapshot" "$extract"
}
@@ -187,6 +190,33 @@ with tarfile.open(archive, "w") as tf:
PY
}
make_duplicate_manifest_archive() {
archive_path=$1
python - "$archive_path" <<'PY'
import io, tarfile, sys
archive = sys.argv[1]
with tarfile.open(archive, "w") as tf:
manifest = b"""format=thothii-qdrant-backup-v1
project_name=thoth-task8
project_name=thoth-task8
volume_name=thoth-task8_qdrant-data
volume_role=qdrant-data
helper_image=qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c
created_utc=2026-08-08T17:35:36Z
"""
info = tarfile.TarInfo("manifest.env")
info.size = len(manifest)
tf.addfile(info, io.BytesIO(manifest))
directory = tarfile.TarInfo("payload")
directory.type = tarfile.DIRTYPE
tf.addfile(directory)
payload = b"dup"
info = tarfile.TarInfo("payload/dup.txt")
info.size = len(payload)
tf.addfile(info, io.BytesIO(payload))
PY
}
backup_output="$tmp/qdrant-backup.tar"
docker_log="$tmp/docker-backup.log"
PATH="$fakebin:$PATH" DOCKER_LOG="$docker_log" PROJECT_NAME="$project" VOLUME_ROOT="$volume_root" \
@@ -272,6 +302,27 @@ if grep -q "compose --project-name $project stop qdrant" "$symlink_log"; then
exit 1
fi
duplicate_archive="$tmp/duplicate.tar"
make_duplicate_manifest_archive "$duplicate_archive"
duplicate_log="$tmp/duplicate.log"
: >"$duplicate_log"
if PATH="$fakebin:$PATH" DOCKER_LOG="$duplicate_log" PROJECT_NAME="$project" VOLUME_ROOT="$volume_root" \
HELPER_IMAGE="qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c" \
./scripts/vector-restore.sh --project-name "$project" --input "$duplicate_archive" --confirm-project "$project" \
>"$tmp/duplicate.out" 2>"$tmp/duplicate.err"; then
echo "restore accepted duplicate manifest keys" >&2
exit 1
fi
grep -Eq 'duplicate|manifest' "$tmp/duplicate.err"
if grep -q "compose --project-name $project stop qdrant" "$duplicate_log"; then
echo "restore stopped qdrant before duplicate-manifest rejection" >&2
exit 1
fi
if grep -q '^run ' "$duplicate_log"; then
echo "restore ran helper before duplicate-manifest rejection" >&2
exit 1
fi
restore_source="$tmp/restore-source"
mkdir -p "$restore_source/payload/collections/demo"
cat >"$restore_source/manifest.env" <<EOF
@@ -297,13 +348,25 @@ grep -q 'confirmation must match --project-name exactly' "$tmp/confirm.err"
printf '%s' modified-live >"$volume_dir/collections/demo/state.json"
restore_log="$tmp/restore-ok.log"
restore_capture="$tmp/restore-captured.tar"
PATH="$fakebin:$PATH" DOCKER_LOG="$restore_log" PROJECT_NAME="$project" VOLUME_ROOT="$volume_root" \
RESTORE_CAPTURE_FILE="$restore_capture" \
HELPER_IMAGE="qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c" \
./scripts/vector-restore.sh --project-name "$project" --input "$restore_input" --confirm-project "$project" >/dev/null
./scripts/vector-restore.sh --project-name "$project" --input "$restore_input" --confirm-project "$project" >/dev/null &
restore_pid=$!
sleep 1
printf '%s' swapped >"$restore_source/payload/collections/demo/state.json"
tar -C "$restore_source" -cf "$restore_input" manifest.env payload
wait "$restore_pid"
test "$(cat "$volume_dir/collections/demo/state.json")" = restored
tar -xOf "$restore_capture" payload/collections/demo/state.json | grep -qx 'restored'
grep -q "compose --project-name $project stop qdrant" "$restore_log"
grep -q "compose --project-name $project start qdrant" "$restore_log"
grep -q "run --rm --mount type=volume,src=$volume_name,dst=/qdrant-data --mount type=bind,src=$tmp,dst=/restore-backup,readonly" "$restore_log"
grep -Eq "run --rm --mount type=volume,src=$volume_name,dst=/qdrant-data --mount type=bind,src=.*/qdrant-archive\.[^,]*,dst=/restore-backup,readonly" "$restore_log"
if grep -q "src=$tmp,dst=/restore-backup,readonly" "$restore_log"; then
echo "restore mounted the original archive directory instead of a private copy" >&2
exit 1
fi
if grep -q "volume inspect --format {{ .Mountpoint }}" "$restore_log"; then
echo "restore consulted Docker mountpoints during normal restore" >&2
exit 1
@@ -312,6 +375,10 @@ if grep -q "prune" "$restore_log"; then
echo "restore attempted global docker cleanup" >&2
exit 1
fi
if find "$tmp" -maxdepth 1 -type d -name 'qdrant-archive.*' | grep -q .; then
echo "restore left its private archive-copy directory behind" >&2
exit 1
fi
printf '%s' rollback-source >"$volume_dir/collections/demo/state.json"
rollback_log="$tmp/restore-rollback.log"