diff --git a/harness/tests/test_qdrant_vector_store.py b/harness/tests/test_qdrant_vector_store.py index a729f611..fe36104d 100644 --- a/harness/tests/test_qdrant_vector_store.py +++ b/harness/tests/test_qdrant_vector_store.py @@ -289,11 +289,30 @@ def test_search_filters_by_workspace_and_allowed_record_kinds(): assert query_call[2]["filter"] == { "must": [ {"key": "workspace_id", "match": {"value": "demo"}}, + {"key": "kind", "match": {"any": ["memory"]}}, {"key": "record_kind", "match": {"any": ["memory"]}}, ] } +def test_search_excludes_inconsistent_semantic_kind_in_bound_workspace(): + fake = FakeQdrantHttp() + store = _store(fake) + store.upsert("memory", [_write_record("memory:1", "memory")]) + contaminated = next(iter(fake.points.values())).copy() + contaminated["id"] = point_id("demo", "evidence", "memory:contaminated") + contaminated["payload"] = { + **contaminated["payload"], + "kind": "evidence", + "record_key": "memory:contaminated", + } + fake.points[contaminated["id"]] = contaminated + + hits = store.search(["memory"], [0.2] * 1024, limit=5, kinds=["memory"]) + + assert [hit.id for hit in hits] == ["memory:1"] + + def test_existing_hashes_health_and_exact_generation_inventory_and_delete(): fake = FakeQdrantHttp() store = _store(fake) @@ -328,6 +347,46 @@ def test_existing_hashes_health_and_exact_generation_inventory_and_delete(): assert health.dimension_compatible is True +def test_evidence_inventory_and_delete_ignore_inconsistent_semantic_kind(): + fake = FakeQdrantHttp() + store = _store(fake) + generation = "gen:" + "1" * 32 + contaminated_generation = "gen:" + "2" * 32 + store.upsert("evidence", [ + _write_record( + f"demo:{generation}:chunk:1", + "evidence", + metadata={ + "workspace_id": "demo", + "vector_generation": generation, + "document_id": "doc:1", + }, + ), + ]) + contaminated_delete = next(iter(fake.points.values())).copy() + contaminated_delete["id"] = point_id("demo", "memory", "evidence:contaminated-delete") + contaminated_delete["payload"] = { + **contaminated_delete["payload"], + "kind": "memory", + "record_key": "evidence:contaminated-delete", + } + fake.points[contaminated_delete["id"]] = contaminated_delete + contaminated_list = next(iter(fake.points.values())).copy() + contaminated_list["id"] = point_id("demo", "memory", "evidence:contaminated-list") + contaminated_list["payload"] = { + **contaminated_list["payload"], + "kind": "memory", + "record_key": "evidence:contaminated-list", + "vector_generation": contaminated_generation, + } + fake.points[contaminated_list["id"]] = contaminated_list + + assert store.list_evidence_generations("evidence", "demo") == [generation] + assert store.delete_generation("evidence", generation, "demo") == 1 + assert contaminated_delete["id"] in fake.points + assert contaminated_list["id"] in fake.points + + def test_metadata_search_rejects_a_workspace_id_different_from_the_bound_adapter(): fake = FakeQdrantHttp() store = _store(fake) @@ -444,6 +503,7 @@ def test_delete_kinds_is_workspace_scoped_and_preserves_other_semantic_kinds(): assert delete_call[2]["filter"] == { "must": [ {"key": "workspace_id", "match": {"value": "demo"}}, + {"key": "kind", "match": {"any": ["memory"]}}, {"key": "record_kind", "match": {"any": ["memory"]}}, ] } @@ -457,6 +517,26 @@ def test_delete_kinds_is_workspace_scoped_and_preserves_other_semantic_kinds(): } +def test_existing_hashes_and_delete_kinds_ignore_inconsistent_semantic_kind(): + fake = FakeQdrantHttp() + store = _store(fake) + store.upsert("memory", [_write_record("memory:1", "memory")]) + contaminated = next(iter(fake.points.values())).copy() + contaminated["id"] = point_id("demo", "evidence", "memory:contaminated") + contaminated["payload"] = { + **contaminated["payload"], + "kind": "evidence", + "record_key": "memory:contaminated", + } + fake.points[contaminated["id"]] = contaminated + + assert store.existing_hashes("memory", ["memory"]) == { + "memory:1": "sha256:" + "a" * 64, + } + assert store.delete_kinds("memory", ["memory"]) == 1 + assert contaminated["id"] in fake.points + + def test_sanitizes_timeout_and_malformed_responses(): fake = FakeQdrantHttp() store = _store(fake) diff --git a/harness/tht/adapters/vector/qdrant.py b/harness/tht/adapters/vector/qdrant.py index d3073f15..f95611f6 100644 --- a/harness/tht/adapters/vector/qdrant.py +++ b/harness/tht/adapters/vector/qdrant.py @@ -125,6 +125,7 @@ class QdrantVectorStore: if not allowed_record_kinds: return [] filter_must = self._workspace_filter() + filter_must.append(self._semantic_kind_filter(allowed_record_kinds)) filter_must.append({"key": "record_kind", "match": {"any": allowed_record_kinds}}) if metadata_filter is not None: if set(metadata_filter) != {"vector_generation", "document_ids", "workspace_id"}: @@ -166,6 +167,7 @@ class QdrantVectorStore: points = self._scroll( [ *self._workspace_filter(), + self._semantic_kind_filter(kinds), {"key": "record_kind", "match": {"any": sorted(kinds)}}, ] ) @@ -213,6 +215,7 @@ class QdrantVectorStore: validate_collection_kinds(collection, kinds) must = [ *self._workspace_filter(), + self._semantic_kind_filter(kinds), {"key": "record_kind", "match": {"any": sorted(kinds)}}, ] before = len(self._scroll(must)) @@ -231,6 +234,7 @@ class QdrantVectorStore: self._require_bound_workspace(workspace_id) must = [ *self._workspace_filter(), + {"key": "kind", "match": {"value": "evidence"}}, {"key": "record_kind", "match": {"any": ["evidence"]}}, {"key": "vector_generation", "match": {"value": generation}}, ] @@ -253,6 +257,7 @@ class QdrantVectorStore: points = self._scroll( [ *self._workspace_filter(), + {"key": "kind", "match": {"value": "evidence"}}, {"key": "record_kind", "match": {"any": ["evidence"]}}, ] ) @@ -268,6 +273,10 @@ class QdrantVectorStore: def _workspace_filter(self) -> list[dict]: return [{"key": "workspace_id", "match": {"value": self._workspace_id}}] + def _semantic_kind_filter(self, record_kinds: list[str]) -> dict: + semantic_kinds = sorted({qdrant_semantic_kind(kind) for kind in record_kinds}) + return {"key": "kind", "match": {"any": semantic_kinds}} + def _require_bound_workspace(self, workspace_id: str) -> None: if workspace_id != self._workspace_id: raise VectorStoreError("Evidence workspace namespace does not match bound workspace") diff --git a/scripts/test-vector-backup-restore-safety.sh b/scripts/test-vector-backup-restore-safety.sh index d173d224..ead2ec4d 100755 --- a/scripts/test-vector-backup-restore-safety.sh +++ b/scripts/test-vector-backup-restore-safety.sh @@ -150,12 +150,28 @@ fi if [ "$1" = compose ] && [ "$2" = --project-name ]; then case "$4" in ps) - if [ "${QDRANT_RUNNING:-1}" = 1 ]; then + qdrant_state=${QDRANT_RUNNING:-1} + if [ -n "${QDRANT_STATE_FILE:-}" ] && [ -f "$QDRANT_STATE_FILE" ]; then + qdrant_state=$(cat "$QDRANT_STATE_FILE") + fi + if [ "$qdrant_state" = 1 ]; then printf '%s\n' qdrant-container fi exit 0 ;; - stop|start) + stop) + if [ -n "${QDRANT_STATE_FILE:-}" ]; then + printf '%s\n' 0 >"$QDRANT_STATE_FILE" + fi + if [ "${STOP_MARKS_STOPPED_THEN_FAIL:-0}" = 1 ]; then + exit 42 + fi + exit 0 + ;; + start) + if [ -n "${QDRANT_STATE_FILE:-}" ]; then + printf '%s\n' 1 >"$QDRANT_STATE_FILE" + fi exit 0 ;; esac @@ -525,4 +541,53 @@ grep -q "compose --project-name $project stop qdrant" "$rollback_log" grep -q "compose --project-name $project start qdrant" "$rollback_log" test ! -e "$tmp/locks/$lock_name" +assert_ambiguous_stop_reconciled() { + operation=$1 + operation_log="$tmp/$operation-ambiguous-stop.log" + operation_state="$tmp/$operation-qdrant-state" + foreign_lock="$tmp/locks/foreign-$operation-lock" + printf '%s\n' 1 >"$operation_state" + mkdir -p "$foreign_lock" + printf '%s\n' foreign-owner >"$foreign_lock/owner" + printf '%s\n' foreign-volume >"$foreign_lock/volume" + + if [ "$operation" = backup ]; then + set -- ./scripts/vector-backup.sh --project-name "$project" \ + --output "$tmp/ambiguous-stop-backup.tar" + else + set -- ./scripts/vector-restore.sh --project-name "$project" \ + --input "$restore_input" --confirm-project "$project" + fi + + if PATH="$fakebin:$PATH" DOCKER_LOG="$operation_log" PROJECT_NAME="$project" \ + VOLUME_ROOT="$volume_root" QDRANT_STATE_FILE="$operation_state" \ + STOP_MARKS_STOPPED_THEN_FAIL=1 \ + HELPER_IMAGE="qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c" \ + "$@" >"$tmp/$operation-ambiguous-stop.out" \ + 2>"$tmp/$operation-ambiguous-stop.err"; then + echo "$operation hid an ambiguous qdrant stop failure" >&2 + exit 1 + fi + + grep -q "compose --project-name $project stop qdrant" "$operation_log" + grep -q "compose --project-name $project start qdrant" "$operation_log" + test "$(cat "$operation_state")" = 1 + test ! -e "$tmp/locks/$lock_name" + test "$(cat "$foreign_lock/owner")" = foreign-owner + test "$(cat "$foreign_lock/volume")" = foreign-volume + if grep -q "rm -f foreign-$operation-lock" "$operation_log"; then + echo "$operation removed a foreign operation lock" >&2 + exit 1 + fi + start_line=$(grep -n "compose --project-name $project start qdrant" "$operation_log" | sed -n '1s/:.*//p') + release_line=$(grep -n "rm -f $lock_name" "$operation_log" | sed -n '1s/:.*//p') + [ "$start_line" -lt "$release_line" ] || { + echo "$operation released its lock before qdrant reconciliation" >&2 + exit 1 + } +} + +assert_ambiguous_stop_reconciled backup +assert_ambiguous_stop_reconciled restore + echo "qdrant backup/restore archive validation, scoped helper execution, and rollback safety passed." diff --git a/scripts/vector-backup.sh b/scripts/vector-backup.sh index 3efcd18d..19eb0b9a 100755 --- a/scripts/vector-backup.sh +++ b/scripts/vector-backup.sh @@ -85,8 +85,8 @@ validate_volume_metadata "$volume_name" running_container=$(docker compose --project-name "$project_name" ps --status running -q qdrant) if [ -n "$running_container" ]; then - docker compose --project-name "$project_name" stop qdrant >/dev/null restart_qdrant=1 + docker compose --project-name "$project_name" stop qdrant >/dev/null fi umask 077 diff --git a/scripts/vector-restore.sh b/scripts/vector-restore.sh index 5ee3a27b..c29dd014 100755 --- a/scripts/vector-restore.sh +++ b/scripts/vector-restore.sh @@ -237,8 +237,8 @@ validate_volume_metadata "$volume_name" running_container=$(docker compose --project-name "$project_name" ps --status running -q qdrant) if [ -n "$running_container" ]; then - docker compose --project-name "$project_name" stop qdrant >/dev/null restart_qdrant=1 + docker compose --project-name "$project_name" stop qdrant >/dev/null fi docker run --rm \