fix: close semantic isolation review gaps
This commit is contained in:
@@ -289,11 +289,30 @@ def test_search_filters_by_workspace_and_allowed_record_kinds():
|
|||||||
assert query_call[2]["filter"] == {
|
assert query_call[2]["filter"] == {
|
||||||
"must": [
|
"must": [
|
||||||
{"key": "workspace_id", "match": {"value": "demo"}},
|
{"key": "workspace_id", "match": {"value": "demo"}},
|
||||||
|
{"key": "kind", "match": {"any": ["memory"]}},
|
||||||
{"key": "record_kind", "match": {"any": ["memory"]}},
|
{"key": "record_kind", "match": {"any": ["memory"]}},
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_excludes_inconsistent_semantic_kind_in_bound_workspace():
|
||||||
|
fake = FakeQdrantHttp()
|
||||||
|
store = _store(fake)
|
||||||
|
store.upsert("memory", [_write_record("memory:1", "memory")])
|
||||||
|
contaminated = next(iter(fake.points.values())).copy()
|
||||||
|
contaminated["id"] = point_id("demo", "evidence", "memory:contaminated")
|
||||||
|
contaminated["payload"] = {
|
||||||
|
**contaminated["payload"],
|
||||||
|
"kind": "evidence",
|
||||||
|
"record_key": "memory:contaminated",
|
||||||
|
}
|
||||||
|
fake.points[contaminated["id"]] = contaminated
|
||||||
|
|
||||||
|
hits = store.search(["memory"], [0.2] * 1024, limit=5, kinds=["memory"])
|
||||||
|
|
||||||
|
assert [hit.id for hit in hits] == ["memory:1"]
|
||||||
|
|
||||||
|
|
||||||
def test_existing_hashes_health_and_exact_generation_inventory_and_delete():
|
def test_existing_hashes_health_and_exact_generation_inventory_and_delete():
|
||||||
fake = FakeQdrantHttp()
|
fake = FakeQdrantHttp()
|
||||||
store = _store(fake)
|
store = _store(fake)
|
||||||
@@ -328,6 +347,46 @@ def test_existing_hashes_health_and_exact_generation_inventory_and_delete():
|
|||||||
assert health.dimension_compatible is True
|
assert health.dimension_compatible is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_evidence_inventory_and_delete_ignore_inconsistent_semantic_kind():
|
||||||
|
fake = FakeQdrantHttp()
|
||||||
|
store = _store(fake)
|
||||||
|
generation = "gen:" + "1" * 32
|
||||||
|
contaminated_generation = "gen:" + "2" * 32
|
||||||
|
store.upsert("evidence", [
|
||||||
|
_write_record(
|
||||||
|
f"demo:{generation}:chunk:1",
|
||||||
|
"evidence",
|
||||||
|
metadata={
|
||||||
|
"workspace_id": "demo",
|
||||||
|
"vector_generation": generation,
|
||||||
|
"document_id": "doc:1",
|
||||||
|
},
|
||||||
|
),
|
||||||
|
])
|
||||||
|
contaminated_delete = next(iter(fake.points.values())).copy()
|
||||||
|
contaminated_delete["id"] = point_id("demo", "memory", "evidence:contaminated-delete")
|
||||||
|
contaminated_delete["payload"] = {
|
||||||
|
**contaminated_delete["payload"],
|
||||||
|
"kind": "memory",
|
||||||
|
"record_key": "evidence:contaminated-delete",
|
||||||
|
}
|
||||||
|
fake.points[contaminated_delete["id"]] = contaminated_delete
|
||||||
|
contaminated_list = next(iter(fake.points.values())).copy()
|
||||||
|
contaminated_list["id"] = point_id("demo", "memory", "evidence:contaminated-list")
|
||||||
|
contaminated_list["payload"] = {
|
||||||
|
**contaminated_list["payload"],
|
||||||
|
"kind": "memory",
|
||||||
|
"record_key": "evidence:contaminated-list",
|
||||||
|
"vector_generation": contaminated_generation,
|
||||||
|
}
|
||||||
|
fake.points[contaminated_list["id"]] = contaminated_list
|
||||||
|
|
||||||
|
assert store.list_evidence_generations("evidence", "demo") == [generation]
|
||||||
|
assert store.delete_generation("evidence", generation, "demo") == 1
|
||||||
|
assert contaminated_delete["id"] in fake.points
|
||||||
|
assert contaminated_list["id"] in fake.points
|
||||||
|
|
||||||
|
|
||||||
def test_metadata_search_rejects_a_workspace_id_different_from_the_bound_adapter():
|
def test_metadata_search_rejects_a_workspace_id_different_from_the_bound_adapter():
|
||||||
fake = FakeQdrantHttp()
|
fake = FakeQdrantHttp()
|
||||||
store = _store(fake)
|
store = _store(fake)
|
||||||
@@ -444,6 +503,7 @@ def test_delete_kinds_is_workspace_scoped_and_preserves_other_semantic_kinds():
|
|||||||
assert delete_call[2]["filter"] == {
|
assert delete_call[2]["filter"] == {
|
||||||
"must": [
|
"must": [
|
||||||
{"key": "workspace_id", "match": {"value": "demo"}},
|
{"key": "workspace_id", "match": {"value": "demo"}},
|
||||||
|
{"key": "kind", "match": {"any": ["memory"]}},
|
||||||
{"key": "record_kind", "match": {"any": ["memory"]}},
|
{"key": "record_kind", "match": {"any": ["memory"]}},
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
@@ -457,6 +517,26 @@ def test_delete_kinds_is_workspace_scoped_and_preserves_other_semantic_kinds():
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_existing_hashes_and_delete_kinds_ignore_inconsistent_semantic_kind():
|
||||||
|
fake = FakeQdrantHttp()
|
||||||
|
store = _store(fake)
|
||||||
|
store.upsert("memory", [_write_record("memory:1", "memory")])
|
||||||
|
contaminated = next(iter(fake.points.values())).copy()
|
||||||
|
contaminated["id"] = point_id("demo", "evidence", "memory:contaminated")
|
||||||
|
contaminated["payload"] = {
|
||||||
|
**contaminated["payload"],
|
||||||
|
"kind": "evidence",
|
||||||
|
"record_key": "memory:contaminated",
|
||||||
|
}
|
||||||
|
fake.points[contaminated["id"]] = contaminated
|
||||||
|
|
||||||
|
assert store.existing_hashes("memory", ["memory"]) == {
|
||||||
|
"memory:1": "sha256:" + "a" * 64,
|
||||||
|
}
|
||||||
|
assert store.delete_kinds("memory", ["memory"]) == 1
|
||||||
|
assert contaminated["id"] in fake.points
|
||||||
|
|
||||||
|
|
||||||
def test_sanitizes_timeout_and_malformed_responses():
|
def test_sanitizes_timeout_and_malformed_responses():
|
||||||
fake = FakeQdrantHttp()
|
fake = FakeQdrantHttp()
|
||||||
store = _store(fake)
|
store = _store(fake)
|
||||||
|
|||||||
@@ -125,6 +125,7 @@ class QdrantVectorStore:
|
|||||||
if not allowed_record_kinds:
|
if not allowed_record_kinds:
|
||||||
return []
|
return []
|
||||||
filter_must = self._workspace_filter()
|
filter_must = self._workspace_filter()
|
||||||
|
filter_must.append(self._semantic_kind_filter(allowed_record_kinds))
|
||||||
filter_must.append({"key": "record_kind", "match": {"any": allowed_record_kinds}})
|
filter_must.append({"key": "record_kind", "match": {"any": allowed_record_kinds}})
|
||||||
if metadata_filter is not None:
|
if metadata_filter is not None:
|
||||||
if set(metadata_filter) != {"vector_generation", "document_ids", "workspace_id"}:
|
if set(metadata_filter) != {"vector_generation", "document_ids", "workspace_id"}:
|
||||||
@@ -166,6 +167,7 @@ class QdrantVectorStore:
|
|||||||
points = self._scroll(
|
points = self._scroll(
|
||||||
[
|
[
|
||||||
*self._workspace_filter(),
|
*self._workspace_filter(),
|
||||||
|
self._semantic_kind_filter(kinds),
|
||||||
{"key": "record_kind", "match": {"any": sorted(kinds)}},
|
{"key": "record_kind", "match": {"any": sorted(kinds)}},
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
@@ -213,6 +215,7 @@ class QdrantVectorStore:
|
|||||||
validate_collection_kinds(collection, kinds)
|
validate_collection_kinds(collection, kinds)
|
||||||
must = [
|
must = [
|
||||||
*self._workspace_filter(),
|
*self._workspace_filter(),
|
||||||
|
self._semantic_kind_filter(kinds),
|
||||||
{"key": "record_kind", "match": {"any": sorted(kinds)}},
|
{"key": "record_kind", "match": {"any": sorted(kinds)}},
|
||||||
]
|
]
|
||||||
before = len(self._scroll(must))
|
before = len(self._scroll(must))
|
||||||
@@ -231,6 +234,7 @@ class QdrantVectorStore:
|
|||||||
self._require_bound_workspace(workspace_id)
|
self._require_bound_workspace(workspace_id)
|
||||||
must = [
|
must = [
|
||||||
*self._workspace_filter(),
|
*self._workspace_filter(),
|
||||||
|
{"key": "kind", "match": {"value": "evidence"}},
|
||||||
{"key": "record_kind", "match": {"any": ["evidence"]}},
|
{"key": "record_kind", "match": {"any": ["evidence"]}},
|
||||||
{"key": "vector_generation", "match": {"value": generation}},
|
{"key": "vector_generation", "match": {"value": generation}},
|
||||||
]
|
]
|
||||||
@@ -253,6 +257,7 @@ class QdrantVectorStore:
|
|||||||
points = self._scroll(
|
points = self._scroll(
|
||||||
[
|
[
|
||||||
*self._workspace_filter(),
|
*self._workspace_filter(),
|
||||||
|
{"key": "kind", "match": {"value": "evidence"}},
|
||||||
{"key": "record_kind", "match": {"any": ["evidence"]}},
|
{"key": "record_kind", "match": {"any": ["evidence"]}},
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
@@ -268,6 +273,10 @@ class QdrantVectorStore:
|
|||||||
def _workspace_filter(self) -> list[dict]:
|
def _workspace_filter(self) -> list[dict]:
|
||||||
return [{"key": "workspace_id", "match": {"value": self._workspace_id}}]
|
return [{"key": "workspace_id", "match": {"value": self._workspace_id}}]
|
||||||
|
|
||||||
|
def _semantic_kind_filter(self, record_kinds: list[str]) -> dict:
|
||||||
|
semantic_kinds = sorted({qdrant_semantic_kind(kind) for kind in record_kinds})
|
||||||
|
return {"key": "kind", "match": {"any": semantic_kinds}}
|
||||||
|
|
||||||
def _require_bound_workspace(self, workspace_id: str) -> None:
|
def _require_bound_workspace(self, workspace_id: str) -> None:
|
||||||
if workspace_id != self._workspace_id:
|
if workspace_id != self._workspace_id:
|
||||||
raise VectorStoreError("Evidence workspace namespace does not match bound workspace")
|
raise VectorStoreError("Evidence workspace namespace does not match bound workspace")
|
||||||
|
|||||||
@@ -150,12 +150,28 @@ fi
|
|||||||
if [ "$1" = compose ] && [ "$2" = --project-name ]; then
|
if [ "$1" = compose ] && [ "$2" = --project-name ]; then
|
||||||
case "$4" in
|
case "$4" in
|
||||||
ps)
|
ps)
|
||||||
if [ "${QDRANT_RUNNING:-1}" = 1 ]; then
|
qdrant_state=${QDRANT_RUNNING:-1}
|
||||||
|
if [ -n "${QDRANT_STATE_FILE:-}" ] && [ -f "$QDRANT_STATE_FILE" ]; then
|
||||||
|
qdrant_state=$(cat "$QDRANT_STATE_FILE")
|
||||||
|
fi
|
||||||
|
if [ "$qdrant_state" = 1 ]; then
|
||||||
printf '%s\n' qdrant-container
|
printf '%s\n' qdrant-container
|
||||||
fi
|
fi
|
||||||
exit 0
|
exit 0
|
||||||
;;
|
;;
|
||||||
stop|start)
|
stop)
|
||||||
|
if [ -n "${QDRANT_STATE_FILE:-}" ]; then
|
||||||
|
printf '%s\n' 0 >"$QDRANT_STATE_FILE"
|
||||||
|
fi
|
||||||
|
if [ "${STOP_MARKS_STOPPED_THEN_FAIL:-0}" = 1 ]; then
|
||||||
|
exit 42
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
start)
|
||||||
|
if [ -n "${QDRANT_STATE_FILE:-}" ]; then
|
||||||
|
printf '%s\n' 1 >"$QDRANT_STATE_FILE"
|
||||||
|
fi
|
||||||
exit 0
|
exit 0
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
@@ -525,4 +541,53 @@ grep -q "compose --project-name $project stop qdrant" "$rollback_log"
|
|||||||
grep -q "compose --project-name $project start qdrant" "$rollback_log"
|
grep -q "compose --project-name $project start qdrant" "$rollback_log"
|
||||||
test ! -e "$tmp/locks/$lock_name"
|
test ! -e "$tmp/locks/$lock_name"
|
||||||
|
|
||||||
|
assert_ambiguous_stop_reconciled() {
|
||||||
|
operation=$1
|
||||||
|
operation_log="$tmp/$operation-ambiguous-stop.log"
|
||||||
|
operation_state="$tmp/$operation-qdrant-state"
|
||||||
|
foreign_lock="$tmp/locks/foreign-$operation-lock"
|
||||||
|
printf '%s\n' 1 >"$operation_state"
|
||||||
|
mkdir -p "$foreign_lock"
|
||||||
|
printf '%s\n' foreign-owner >"$foreign_lock/owner"
|
||||||
|
printf '%s\n' foreign-volume >"$foreign_lock/volume"
|
||||||
|
|
||||||
|
if [ "$operation" = backup ]; then
|
||||||
|
set -- ./scripts/vector-backup.sh --project-name "$project" \
|
||||||
|
--output "$tmp/ambiguous-stop-backup.tar"
|
||||||
|
else
|
||||||
|
set -- ./scripts/vector-restore.sh --project-name "$project" \
|
||||||
|
--input "$restore_input" --confirm-project "$project"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if PATH="$fakebin:$PATH" DOCKER_LOG="$operation_log" PROJECT_NAME="$project" \
|
||||||
|
VOLUME_ROOT="$volume_root" QDRANT_STATE_FILE="$operation_state" \
|
||||||
|
STOP_MARKS_STOPPED_THEN_FAIL=1 \
|
||||||
|
HELPER_IMAGE="qdrant/qdrant:v1.18.2@sha256:75eab8c4ba42096724fdcfde8b4de0b5713d529dde32f285a1f86fdcb2c9e50c" \
|
||||||
|
"$@" >"$tmp/$operation-ambiguous-stop.out" \
|
||||||
|
2>"$tmp/$operation-ambiguous-stop.err"; then
|
||||||
|
echo "$operation hid an ambiguous qdrant stop failure" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
grep -q "compose --project-name $project stop qdrant" "$operation_log"
|
||||||
|
grep -q "compose --project-name $project start qdrant" "$operation_log"
|
||||||
|
test "$(cat "$operation_state")" = 1
|
||||||
|
test ! -e "$tmp/locks/$lock_name"
|
||||||
|
test "$(cat "$foreign_lock/owner")" = foreign-owner
|
||||||
|
test "$(cat "$foreign_lock/volume")" = foreign-volume
|
||||||
|
if grep -q "rm -f foreign-$operation-lock" "$operation_log"; then
|
||||||
|
echo "$operation removed a foreign operation lock" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
start_line=$(grep -n "compose --project-name $project start qdrant" "$operation_log" | sed -n '1s/:.*//p')
|
||||||
|
release_line=$(grep -n "rm -f $lock_name" "$operation_log" | sed -n '1s/:.*//p')
|
||||||
|
[ "$start_line" -lt "$release_line" ] || {
|
||||||
|
echo "$operation released its lock before qdrant reconciliation" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_ambiguous_stop_reconciled backup
|
||||||
|
assert_ambiguous_stop_reconciled restore
|
||||||
|
|
||||||
echo "qdrant backup/restore archive validation, scoped helper execution, and rollback safety passed."
|
echo "qdrant backup/restore archive validation, scoped helper execution, and rollback safety passed."
|
||||||
|
|||||||
@@ -85,8 +85,8 @@ validate_volume_metadata "$volume_name"
|
|||||||
running_container=$(docker compose --project-name "$project_name" ps --status running -q qdrant)
|
running_container=$(docker compose --project-name "$project_name" ps --status running -q qdrant)
|
||||||
|
|
||||||
if [ -n "$running_container" ]; then
|
if [ -n "$running_container" ]; then
|
||||||
docker compose --project-name "$project_name" stop qdrant >/dev/null
|
|
||||||
restart_qdrant=1
|
restart_qdrant=1
|
||||||
|
docker compose --project-name "$project_name" stop qdrant >/dev/null
|
||||||
fi
|
fi
|
||||||
|
|
||||||
umask 077
|
umask 077
|
||||||
|
|||||||
@@ -237,8 +237,8 @@ validate_volume_metadata "$volume_name"
|
|||||||
running_container=$(docker compose --project-name "$project_name" ps --status running -q qdrant)
|
running_container=$(docker compose --project-name "$project_name" ps --status running -q qdrant)
|
||||||
|
|
||||||
if [ -n "$running_container" ]; then
|
if [ -n "$running_container" ]; then
|
||||||
docker compose --project-name "$project_name" stop qdrant >/dev/null
|
|
||||||
restart_qdrant=1
|
restart_qdrant=1
|
||||||
|
docker compose --project-name "$project_name" stop qdrant >/dev/null
|
||||||
fi
|
fi
|
||||||
|
|
||||||
docker run --rm \
|
docker run --rm \
|
||||||
|
|||||||
Reference in New Issue
Block a user