mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(evidence): use server-side Qdrant BM25 retrieval

Browse Source
This commit is contained in:
marcopan
2026-08-24 18:15:33 +02:00
parent 0e9add09a9
commit 29d41ac258
15 changed files with 253 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/tht/adapters/vector/qdrant.py
+57 -11
View File
@@ -24,6 +24,7 @@ from tht.vectorstore.store import VectorHit, hit_from_metadata
_GENERATION = re.compile(r"gen:[0-9a-f]{32}")
_WORKSPACE = re.compile(r"[a-z][a-z0-9_-]{0,63}")
_BM25_LANGUAGES = frozenset({"english", "italian"})
_KEYWORD_INDEXES = (
"content_hash",
@@ -127,6 +128,8 @@ class QdrantVectorStore:
limit: int,
kinds: list[str] | None = None,
metadata_filter: dict[str, object] | None = None,
query_text: str | None = None,
query_language: str | None = None,
) -> list[VectorHit]:
require_positive_limit(limit)
self._validate_embedding(embedding, query=True)
@@ -155,16 +158,43 @@ class QdrantVectorStore:
{"key": "vector_generation", "match": {"value": generation}},
{"key": "document_id", "match": {"any": document_ids}},
])
response = self._call(
"POST",
f"/collections/{self._collection}/points/query",
{
"vector": embedding,
"limit": limit,
"with_payload": True,
"filter": {"must": filter_must},
},
)
if query_text is None:
if allowed_record_kinds == ["evidence"]:
raise VectorStoreError("Evidence hybrid query text is required")
response = self._call(
"POST",
f"/collections/{self._collection}/points/query",
{
"vector": embedding,
"limit": limit,
"with_payload": True,
"filter": {"must": filter_must},
},
)
else:
if allowed_record_kinds != ["evidence"]:
raise VectorStoreError("Hybrid BM25 is only available for Evidence")
if query_text.strip() == "" or query_language not in _BM25_LANGUAGES:
raise VectorStoreError("Evidence BM25 query is invalid")
shared_filter = {"must": filter_must}
response = self._call(
"POST",
f"/collections/{self._collection}/points/query",
{
"prefetch": [
{"query": embedding, "limit": limit * 2, "filter": shared_filter},
{
"query": self._bm25_document(query_text, query_language),
"using": "bm25",
"limit": limit * 2,
"filter": shared_filter,
},
],
"query": {"rrf": {}},
"limit": limit,
"with_payload": True,
},
)
points = response.get("result", {}).get("points")
if not isinstance(points, list):
raise VectorStoreError("Qdrant returned malformed query response")
@@ -201,6 +231,14 @@ class QdrantVectorStore:
validate_collection_kinds(collection, [write_record.record.kind])
self._validate_embedding(write_record.embedding, query=False)
semantic_kind = qdrant_semantic_kind(write_record.record.kind)
vector: list[float] | dict = write_record.embedding
if write_record.sparse_text is not None:
if semantic_kind != "evidence" or write_record.sparse_language not in _BM25_LANGUAGES:
raise VectorStoreError("Evidence BM25 document is invalid")
vector = {
"": write_record.embedding,
"bm25": self._bm25_document(write_record.sparse_text, write_record.sparse_language),
}
points.append(
{
"id": point_id(
@@ -209,7 +247,7 @@ class QdrantVectorStore:
write_record.record.id,
self._workspace_revision if semantic_kind in ("schema_table", "schema_column", "evidence") else None,
),
"vector": write_record.embedding,
"vector": vector,
"payload": qdrant_payload(
write_record.record,
content_hash=write_record.content_hash,
@@ -291,6 +329,14 @@ class QdrantVectorStore:
def _workspace_filter(self) -> list[dict]:
return [{"key": "workspace_id", "match": {"value": self._workspace_id}}]
@staticmethod
def _bm25_document(text: str, language: str) -> dict:
return {
"text": text,
"model": "qdrant/bm25",
"options": {"language": language},
}
def _revision_filter(self, kinds: list[str]) -> list[dict]:
if self._workspace_revision is None:
return []
harness/tht/cli/preprocess_cmd.py
+10
View File
@@ -14,6 +14,14 @@ from tht.cli.config_cmd import CONFIG_OPT
preprocess_app = typer.Typer(help="Materialize versioned preprocessing artifacts")
def _bm25_language(workspace_language: str) -> str:
languages = {"en": "english", "it": "italian"}
try:
return languages[workspace_language]
except KeyError as exc:
raise ValueError("workspace language is unsupported for Qdrant BM25") from exc
def _evidence_json_context(config: Path):
from tht.cli.schema_cmd import _load_config_or_exit
@@ -112,6 +120,7 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None =
chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
pipeline_version="evidence-v1",
retain_published_generations=cfg.vector.retain_published_generations,
sparse_language=_bm25_language(cfg.language),
)
def fingerprint(value: str) -> str:
return "sha256:" + hashlib.sha256(value.encode()).hexdigest()
@@ -145,6 +154,7 @@ def gc_from_config(config: Path, *, dry_run: bool = False):
chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
pipeline_version="evidence-v1",
retain_published_generations=cfg.vector.retain_published_generations,
sparse_language=_bm25_language(cfg.language),
)
pipeline.workspace_id = cfg._workspace_id
return pipeline.gc(workspace_root=corpus_root.parent, dry_run=dry_run)
harness/tht/cli/vector_cmd.py
+11 -1
View File
@@ -55,10 +55,20 @@ def open_searcher(cfg):
store = build_vector_store(cfg)
class AdapterSearcher:
def search(self, query_vec, top_n=10, kinds=None, metadata_filter=None):
def search(
self,
query_vec,
top_n=10,
kinds=None,
metadata_filter=None,
query_text=None,
query_language=None,
):
return store.search(
tables_for_kinds(kinds), query_vec, limit=top_n, kinds=kinds,
metadata_filter=metadata_filter,
query_text=query_text,
query_language=query_language,
)
return AdapterSearcher()
harness/tht/evidence/corpus/pipeline.py
+12 -4
View File
@@ -123,7 +123,7 @@ class CorpusPipeline:
self, *, store: CorpusStore, sources: list[EvidenceSource], embedder,
vector_store: VectorStore, embedding_model: str, embedding_dimensions: int,
chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3,
workspace_id: str | None = None,
workspace_id: str | None = None, sparse_language: str = "italian",
) -> None:
self.store = store
self.sources = sources
@@ -137,6 +137,9 @@ class CorpusPipeline:
raise ValueError("retain_published_generations must be at least 1")
self.retain_published_generations = retain_published_generations
self.workspace_id = workspace_id
if sparse_language not in {"english", "italian"}:
raise ValueError("unsupported Qdrant BM25 language")
self.sparse_language = sparse_language
def _assert_workspace_binding(self) -> None:
manifest = self.store.active_manifest()
@@ -795,9 +798,8 @@ class CorpusPipeline:
except Exception:
logger.debug("Failed to delete the unpublished vector generation", exc_info=True)
@staticmethod
def _vector_record(
chunk: CanonicalChunk, embedding: list[float], generation: str, workspace_id: str,
self, chunk: CanonicalChunk, embedding: list[float], generation: str, workspace_id: str,
):
record = VectorRecord(
id=f"{workspace_id}:{generation}:{chunk.chunk_id}",
@@ -810,4 +812,10 @@ class CorpusPipeline:
"vector_generation": generation,
},
)
return VectorWriteRecord(record=record, embedding=embedding, content_hash=chunk.content_hash)
return VectorWriteRecord(
record=record,
embedding=embedding,
content_hash=chunk.content_hash,
sparse_text=chunk.content,
sparse_language=self.sparse_language,
)
harness/tht/evidence/preprocessing.py
+2
View File
@@ -25,6 +25,7 @@ def build_preprocessing_pipeline(
pipeline_version: str,
retain_published_generations: int = 3,
workspace_id: str | None = None,
sparse_language: str = "italian",
) -> CorpusPipeline:
"""Construct preprocessing from the bounded infrastructure supplied by core."""
return CorpusPipeline(
@@ -38,6 +39,7 @@ def build_preprocessing_pipeline(
pipeline_version=pipeline_version,
retain_published_generations=retain_published_generations,
workspace_id=workspace_id,
sparse_language=sparse_language,
)
harness/tht/evidence/search.py
+27 -3
View File
@@ -3,6 +3,7 @@
import re
from tht.evidence.corpus.store import CorpusStore
from tht.ports.vector import VectorStoreError
class CorpusWorkspaceMismatchError(RuntimeError):
@@ -12,12 +13,27 @@ class CorpusWorkspaceMismatchError(RuntimeError):
class ActiveEvidenceSearcher:
"""Searcher facade that enforces ACTIVE generation predicates before LIMIT."""
def __init__(self, corpus: CorpusStore, delegate, expected_workspace_id: str | None = None):
def __init__(
self,
corpus: CorpusStore,
delegate,
expected_workspace_id: str | None = None,
evidence_language: str = "italian",
):
self.corpus = corpus
self.delegate = delegate
self.expected_workspace_id = expected_workspace_id
self.evidence_language = evidence_language
def search(self, embedding, top_n=10, kinds=None, metadata_filter=None):
def search(
self,
embedding,
top_n=10,
kinds=None,
metadata_filter=None,
query_text=None,
query_language=None,
):
requested = set(kinds) if kinds is not None else {
"schema_table", "schema_column", "evidence", "memory", "solved_question",
}
@@ -59,9 +75,13 @@ class ActiveEvidenceSearcher:
generation = mapping.get(document.document_id, manifest.vector_generation)
if generation:
by_generation.setdefault(generation, []).append(document.document_id)
if by_generation and (not isinstance(query_text, str) or query_text.strip() == ""):
raise VectorStoreError("Evidence hybrid query text is required")
for generation, document_ids in sorted(by_generation.items()):
hits.extend(self.delegate.search(
embedding, top_n=top_n, kinds=["evidence"],
query_text=query_text,
query_language=query_language or self.evidence_language,
metadata_filter={
"vector_generation": generation,
"document_ids": sorted(document_ids),
@@ -73,7 +93,11 @@ class ActiveEvidenceSearcher:
def active_searcher(cfg, delegate, *, workspace_id: str | None = None):
corpus_root = cfg.paths.artifacts.parent / "corpus"
return ActiveEvidenceSearcher(CorpusStore(corpus_root), delegate, workspace_id)
languages = {"en": "english", "it": "italian"}
language = languages.get(getattr(cfg, "language", "en"))
if language is None:
raise VectorStoreError("workspace language is unsupported for Qdrant BM25")
return ActiveEvidenceSearcher(CorpusStore(corpus_root), delegate, workspace_id, language)
def validate_corpus_workspace(cfg, workspace_id: str) -> None:
harness/tht/ports/vector.py
+4
View File
@@ -39,6 +39,8 @@ class VectorWriteRecord:
record: VectorRecord
embedding: list[float]
content_hash: str
sparse_text: str | None = None
sparse_language: str | None = None
class VectorStoreError(Exception):
@@ -74,6 +76,8 @@ class VectorStore(Protocol):
limit: int,
kinds: list[str] | None = None,
metadata_filter: dict[str, object] | None = None,
query_text: str | None = None,
query_language: str | None = None,
) -> list[VectorHit]: ...
def existing_hashes(self, collection: str, kinds: list[str]) -> dict[str, str]: ...
harness/tht/search/__init__.py
+4 -1
View File
@@ -112,7 +112,10 @@ def combined_search(
if query_vec is None:
query_vec = embedder.embed_query(keyword)
vector_hits = store.search(query_vec, top_n=top * 2, kinds=kinds)
search_kwargs = {"top_n": top * 2, "kinds": kinds}
if kinds is not None and "evidence" in kinds:
search_kwargs["query_text"] = keyword
vector_hits = store.search(query_vec, **search_kwargs)
rankings["vector"] = [(_vector_key(h), h.similarity) for h in vector_hits]
by_key = {_vector_key(h): h for h in vector_hits}