fix: close qdrant archive restore race
This commit is contained in:
@@ -57,6 +57,9 @@ run_restore() {
|
||||
exit 17
|
||||
fi
|
||||
tar -C "$extract" -xf "$backup_dir/$input_name"
|
||||
if [ -n "${RESTORE_CAPTURE_FILE:-}" ]; then
|
||||
cp "$backup_dir/$input_name" "$RESTORE_CAPTURE_FILE"
|
||||
fi
|
||||
cp -R "$extract/payload"/. "$volume_dir"/
|
||||
rm -rf "$snapshot" "$extract"
|
||||
}
|
||||
@@ -187,6 +190,33 @@ with tarfile.open(archive, "w") as tf:
|
||||
PY
|
||||
}
|
||||
|
||||
make_duplicate_manifest_archive() {
|
||||
archive_path=$1
|
||||
python - "$archive_path" <<'PY'
|
||||
import io, tarfile, sys
|
||||
archive = sys.argv[1]
|
||||
with tarfile.open(archive, "w") as tf:
|
||||
manifest = b"""format=thothii-qdrant-backup-v1
|
||||
project_name=thoth-task8
|
||||
project_name=thoth-task8
|
||||
volume_name=thoth-task8_qdrant-data
|
||||
volume_role=qdrant-data
|
||||
helper_image=qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c
|
||||
created_utc=2026-08-08T17:35:36Z
|
||||
"""
|
||||
info = tarfile.TarInfo("manifest.env")
|
||||
info.size = len(manifest)
|
||||
tf.addfile(info, io.BytesIO(manifest))
|
||||
directory = tarfile.TarInfo("payload")
|
||||
directory.type = tarfile.DIRTYPE
|
||||
tf.addfile(directory)
|
||||
payload = b"dup"
|
||||
info = tarfile.TarInfo("payload/dup.txt")
|
||||
info.size = len(payload)
|
||||
tf.addfile(info, io.BytesIO(payload))
|
||||
PY
|
||||
}
|
||||
|
||||
backup_output="$tmp/qdrant-backup.tar"
|
||||
docker_log="$tmp/docker-backup.log"
|
||||
PATH="$fakebin:$PATH" DOCKER_LOG="$docker_log" PROJECT_NAME="$project" VOLUME_ROOT="$volume_root" \
|
||||
@@ -272,6 +302,27 @@ if grep -q "compose --project-name $project stop qdrant" "$symlink_log"; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
duplicate_archive="$tmp/duplicate.tar"
|
||||
make_duplicate_manifest_archive "$duplicate_archive"
|
||||
duplicate_log="$tmp/duplicate.log"
|
||||
: >"$duplicate_log"
|
||||
if PATH="$fakebin:$PATH" DOCKER_LOG="$duplicate_log" PROJECT_NAME="$project" VOLUME_ROOT="$volume_root" \
|
||||
HELPER_IMAGE="qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c" \
|
||||
./scripts/vector-restore.sh --project-name "$project" --input "$duplicate_archive" --confirm-project "$project" \
|
||||
>"$tmp/duplicate.out" 2>"$tmp/duplicate.err"; then
|
||||
echo "restore accepted duplicate manifest keys" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Eq 'duplicate|manifest' "$tmp/duplicate.err"
|
||||
if grep -q "compose --project-name $project stop qdrant" "$duplicate_log"; then
|
||||
echo "restore stopped qdrant before duplicate-manifest rejection" >&2
|
||||
exit 1
|
||||
fi
|
||||
if grep -q '^run ' "$duplicate_log"; then
|
||||
echo "restore ran helper before duplicate-manifest rejection" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
restore_source="$tmp/restore-source"
|
||||
mkdir -p "$restore_source/payload/collections/demo"
|
||||
cat >"$restore_source/manifest.env" <<EOF
|
||||
@@ -297,13 +348,25 @@ grep -q 'confirmation must match --project-name exactly' "$tmp/confirm.err"
|
||||
|
||||
printf '%s' modified-live >"$volume_dir/collections/demo/state.json"
|
||||
restore_log="$tmp/restore-ok.log"
|
||||
restore_capture="$tmp/restore-captured.tar"
|
||||
PATH="$fakebin:$PATH" DOCKER_LOG="$restore_log" PROJECT_NAME="$project" VOLUME_ROOT="$volume_root" \
|
||||
RESTORE_CAPTURE_FILE="$restore_capture" \
|
||||
HELPER_IMAGE="qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c" \
|
||||
./scripts/vector-restore.sh --project-name "$project" --input "$restore_input" --confirm-project "$project" >/dev/null
|
||||
./scripts/vector-restore.sh --project-name "$project" --input "$restore_input" --confirm-project "$project" >/dev/null &
|
||||
restore_pid=$!
|
||||
sleep 1
|
||||
printf '%s' swapped >"$restore_source/payload/collections/demo/state.json"
|
||||
tar -C "$restore_source" -cf "$restore_input" manifest.env payload
|
||||
wait "$restore_pid"
|
||||
test "$(cat "$volume_dir/collections/demo/state.json")" = restored
|
||||
tar -xOf "$restore_capture" payload/collections/demo/state.json | grep -qx 'restored'
|
||||
grep -q "compose --project-name $project stop qdrant" "$restore_log"
|
||||
grep -q "compose --project-name $project start qdrant" "$restore_log"
|
||||
grep -q "run --rm --mount type=volume,src=$volume_name,dst=/qdrant-data --mount type=bind,src=$tmp,dst=/restore-backup,readonly" "$restore_log"
|
||||
grep -Eq "run --rm --mount type=volume,src=$volume_name,dst=/qdrant-data --mount type=bind,src=.*/qdrant-archive\.[^,]*,dst=/restore-backup,readonly" "$restore_log"
|
||||
if grep -q "src=$tmp,dst=/restore-backup,readonly" "$restore_log"; then
|
||||
echo "restore mounted the original archive directory instead of a private copy" >&2
|
||||
exit 1
|
||||
fi
|
||||
if grep -q "volume inspect --format {{ .Mountpoint }}" "$restore_log"; then
|
||||
echo "restore consulted Docker mountpoints during normal restore" >&2
|
||||
exit 1
|
||||
@@ -312,6 +375,10 @@ if grep -q "prune" "$restore_log"; then
|
||||
echo "restore attempted global docker cleanup" >&2
|
||||
exit 1
|
||||
fi
|
||||
if find "$tmp" -maxdepth 1 -type d -name 'qdrant-archive.*' | grep -q .; then
|
||||
echo "restore left its private archive-copy directory behind" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf '%s' rollback-source >"$volume_dir/collections/demo/state.json"
|
||||
rollback_log="$tmp/restore-rollback.log"
|
||||
|
||||
Reference in New Issue
Block a user