feat(evidence): use server-side Qdrant BM25 retrieval

harness/tht/adapters/vector/qdrant.py

@@ -24,6 +24,7 @@ from tht.vectorstore.store import VectorHit, hit_from_metadata
 
 _GENERATION = re.compile(r"gen:[0-9a-f]{32}")
 _WORKSPACE = re.compile(r"[a-z][a-z0-9_-]{0,63}")
+_BM25_LANGUAGES = frozenset({"english", "italian"})
 _KEYWORD_INDEXES = (
 
     "content_hash",
@@ -127,6 +128,8 @@ class QdrantVectorStore:
         limit: int,
         kinds: list[str] | None = None,
         metadata_filter: dict[str, object] | None = None,
+        query_text: str | None = None,
+        query_language: str | None = None,
     ) -> list[VectorHit]:
         require_positive_limit(limit)
         self._validate_embedding(embedding, query=True)
@@ -155,16 +158,43 @@ class QdrantVectorStore:
                 {"key": "vector_generation", "match": {"value": generation}},
                 {"key": "document_id", "match": {"any": document_ids}},
             ])
-        response = self._call(
-            "POST",
-            f"/collections/{self._collection}/points/query",
-            {
-                "vector": embedding,
-                "limit": limit,
-                "with_payload": True,
-                "filter": {"must": filter_must},
-            },
-        )
+        if query_text is None:
+            if allowed_record_kinds == ["evidence"]:
+                raise VectorStoreError("Evidence hybrid query text is required")
+            response = self._call(
+                "POST",
+                f"/collections/{self._collection}/points/query",
+                {
+                    "vector": embedding,
+                    "limit": limit,
+                    "with_payload": True,
+                    "filter": {"must": filter_must},
+                },
+            )
+        else:
+            if allowed_record_kinds != ["evidence"]:
+                raise VectorStoreError("Hybrid BM25 is only available for Evidence")
+            if query_text.strip() == "" or query_language not in _BM25_LANGUAGES:
+                raise VectorStoreError("Evidence BM25 query is invalid")
+            shared_filter = {"must": filter_must}
+            response = self._call(
+                "POST",
+                f"/collections/{self._collection}/points/query",
+                {
+                    "prefetch": [
+                        {"query": embedding, "limit": limit * 2, "filter": shared_filter},
+                        {
+                            "query": self._bm25_document(query_text, query_language),
+                            "using": "bm25",
+                            "limit": limit * 2,
+                            "filter": shared_filter,
+                        },
+                    ],
+                    "query": {"rrf": {}},
+                    "limit": limit,
+                    "with_payload": True,
+                },
+            )
         points = response.get("result", {}).get("points")
         if not isinstance(points, list):
             raise VectorStoreError("Qdrant returned malformed query response")
@@ -201,6 +231,14 @@ class QdrantVectorStore:
             validate_collection_kinds(collection, [write_record.record.kind])
             self._validate_embedding(write_record.embedding, query=False)
             semantic_kind = qdrant_semantic_kind(write_record.record.kind)
+            vector: list[float] | dict = write_record.embedding
+            if write_record.sparse_text is not None:
+                if semantic_kind != "evidence" or write_record.sparse_language not in _BM25_LANGUAGES:
+                    raise VectorStoreError("Evidence BM25 document is invalid")
+                vector = {
+                    "": write_record.embedding,
+                    "bm25": self._bm25_document(write_record.sparse_text, write_record.sparse_language),
+                }
             points.append(
                 {
                     "id": point_id(
@@ -209,7 +247,7 @@ class QdrantVectorStore:
                         write_record.record.id,
                         self._workspace_revision if semantic_kind in ("schema_table", "schema_column", "evidence") else None,
                     ),
-                    "vector": write_record.embedding,
+                    "vector": vector,
                     "payload": qdrant_payload(
                         write_record.record,
                         content_hash=write_record.content_hash,
@@ -291,6 +329,14 @@ class QdrantVectorStore:
     def _workspace_filter(self) -> list[dict]:
         return [{"key": "workspace_id", "match": {"value": self._workspace_id}}]
 
+    @staticmethod
+    def _bm25_document(text: str, language: str) -> dict:
+        return {
+            "text": text,
+            "model": "qdrant/bm25",
+            "options": {"language": language},
+        }
+
     def _revision_filter(self, kinds: list[str]) -> list[dict]:
         if self._workspace_revision is None:
             return []