feat(evidence): use server-side Qdrant BM25 retrieval

harness/tht/evidence/corpus/pipeline.py

@@ -123,7 +123,7 @@ class CorpusPipeline:
         self, *, store: CorpusStore, sources: list[EvidenceSource], embedder,
         vector_store: VectorStore, embedding_model: str, embedding_dimensions: int,
         chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3,
-        workspace_id: str | None = None,
+        workspace_id: str | None = None, sparse_language: str = "italian",
     ) -> None:
         self.store = store
         self.sources = sources
@@ -137,6 +137,9 @@ class CorpusPipeline:
             raise ValueError("retain_published_generations must be at least 1")
         self.retain_published_generations = retain_published_generations
         self.workspace_id = workspace_id
+        if sparse_language not in {"english", "italian"}:
+            raise ValueError("unsupported Qdrant BM25 language")
+        self.sparse_language = sparse_language
 
     def _assert_workspace_binding(self) -> None:
         manifest = self.store.active_manifest()
@@ -795,9 +798,8 @@ class CorpusPipeline:
             except Exception:
                 logger.debug("Failed to delete the unpublished vector generation", exc_info=True)
 
-    @staticmethod
     def _vector_record(
-        chunk: CanonicalChunk, embedding: list[float], generation: str, workspace_id: str,
+        self, chunk: CanonicalChunk, embedding: list[float], generation: str, workspace_id: str,
     ):
         record = VectorRecord(
             id=f"{workspace_id}:{generation}:{chunk.chunk_id}",
@@ -810,4 +812,10 @@ class CorpusPipeline:
                 "vector_generation": generation,
             },
         )
-        return VectorWriteRecord(record=record, embedding=embedding, content_hash=chunk.content_hash)
+        return VectorWriteRecord(
+            record=record,
+            embedding=embedding,
+            content_hash=chunk.content_hash,
+            sparse_text=chunk.content,
+            sparse_language=self.sparse_language,
+        )

harness/tht/evidence/preprocessing.py

@@ -25,6 +25,7 @@ def build_preprocessing_pipeline(
     pipeline_version: str,
     retain_published_generations: int = 3,
     workspace_id: str | None = None,
+    sparse_language: str = "italian",
 ) -> CorpusPipeline:
     """Construct preprocessing from the bounded infrastructure supplied by core."""
     return CorpusPipeline(
@@ -38,6 +39,7 @@ def build_preprocessing_pipeline(
         pipeline_version=pipeline_version,
         retain_published_generations=retain_published_generations,
         workspace_id=workspace_id,
+        sparse_language=sparse_language,
     )
 
 

harness/tht/evidence/search.py

@@ -3,6 +3,7 @@
 import re
 
 from tht.evidence.corpus.store import CorpusStore
+from tht.ports.vector import VectorStoreError
 
 
 class CorpusWorkspaceMismatchError(RuntimeError):
@@ -12,12 +13,27 @@ class CorpusWorkspaceMismatchError(RuntimeError):
 class ActiveEvidenceSearcher:
     """Searcher facade that enforces ACTIVE generation predicates before LIMIT."""
 
-    def __init__(self, corpus: CorpusStore, delegate, expected_workspace_id: str | None = None):
+    def __init__(
+        self,
+        corpus: CorpusStore,
+        delegate,
+        expected_workspace_id: str | None = None,
+        evidence_language: str = "italian",
+    ):
         self.corpus = corpus
         self.delegate = delegate
         self.expected_workspace_id = expected_workspace_id
+        self.evidence_language = evidence_language
 
-    def search(self, embedding, top_n=10, kinds=None, metadata_filter=None):
+    def search(
+        self,
+        embedding,
+        top_n=10,
+        kinds=None,
+        metadata_filter=None,
+        query_text=None,
+        query_language=None,
+    ):
         requested = set(kinds) if kinds is not None else {
             "schema_table", "schema_column", "evidence", "memory", "solved_question",
         }
@@ -59,9 +75,13 @@ class ActiveEvidenceSearcher:
                         generation = mapping.get(document.document_id, manifest.vector_generation)
                         if generation:
                             by_generation.setdefault(generation, []).append(document.document_id)
+                    if by_generation and (not isinstance(query_text, str) or query_text.strip() == ""):
+                        raise VectorStoreError("Evidence hybrid query text is required")
                     for generation, document_ids in sorted(by_generation.items()):
                         hits.extend(self.delegate.search(
                             embedding, top_n=top_n, kinds=["evidence"],
+                            query_text=query_text,
+                            query_language=query_language or self.evidence_language,
                             metadata_filter={
                                 "vector_generation": generation,
                                 "document_ids": sorted(document_ids),
@@ -73,7 +93,11 @@ class ActiveEvidenceSearcher:
 
 def active_searcher(cfg, delegate, *, workspace_id: str | None = None):
     corpus_root = cfg.paths.artifacts.parent / "corpus"
-    return ActiveEvidenceSearcher(CorpusStore(corpus_root), delegate, workspace_id)
+    languages = {"en": "english", "it": "italian"}
+    language = languages.get(getattr(cfg, "language", "en"))
+    if language is None:
+        raise VectorStoreError("workspace language is unsupported for Qdrant BM25")
+    return ActiveEvidenceSearcher(CorpusStore(corpus_root), delegate, workspace_id, language)
 
 
 def validate_corpus_workspace(cfg, workspace_id: str) -> None: