mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(evidence): use server-side Qdrant BM25 retrieval

Browse Source
This commit is contained in:
marcopan
2026-08-24 18:15:33 +02:00
parent 0e9add09a9
commit 29d41ac258
15 changed files with 253 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/workspaces/evidence/preprocessing.ts
+3 -4
View File
@@ -13,7 +13,6 @@ export interface EvidenceJobState {
export interface EvidencePreprocessingDependencies {
runStage(argv: string[]): Promise<Record<string, unknown>>;
persistJob(): void;
semanticPreflight(): Promise<{ ok: true } | { ok: false; code: SemanticFailureCode }>;
evidencePreflight(): Promise<{ ok: true } | { ok: false; code: SemanticFailureCode }>;
requireRunId(value: unknown): string;
numberRecord(value: unknown): Record<string, number> | undefined;
@@ -139,9 +138,9 @@ export async function preprocessEvidence(
}
const policy = evidencePolicy(request.evidence, request.httpPrivateHostAllowlist);
if (policy) return policy;
const semantic = await deps.evidencePreflight();
if (!semantic.ok) {
return { status: "failed", code: semantic.code, runId: request.job.runId };
const preflight = await deps.evidencePreflight();
if (!preflight.ok) {
return { status: "failed", code: preflight.code, runId: request.job.runId };
}
if (request.job.completedStages.includes("evidence") && !request.dryRun) {
return {
backend/src/workspaces/preprocessing-service.ts
-1
View File
@@ -412,7 +412,6 @@ export class WorkspacePreprocessingService {
return {
runStage: async (argv) => await this.runJsonStage(scope.runtime, argv),
persistJob: () => this.state(scope.runtime.workspaceId).writeJob(scope.job),
semanticPreflight: async () => await this.deps.semanticPreflight(scope.runtime.workspace),
evidencePreflight: async () => await this.deps.evidencePreflight(scope.runtime.workspace),
requireRunId: (value) => this.requireRunId(value),
numberRecord: (value) => this.numberRecord(value),
backend/test/workspaces/evidence/preprocessing.test.ts
-4
View File
@@ -39,13 +39,11 @@ function job(overrides: Partial<EvidenceJobState> = {}): EvidenceJobState {
function dependencies(payload: Record<string, unknown> = {}): EvidencePreprocessingDependencies & {
runStage: ReturnType<typeof vi.fn>;
persistJob: ReturnType<typeof vi.fn>;
semanticPreflight: ReturnType<typeof vi.fn>;
evidencePreflight: ReturnType<typeof vi.fn>;
} {
return {
runStage: vi.fn(async () => payload),
persistJob: vi.fn(),
semanticPreflight: vi.fn(async () => ({ ok: true as const })),
evidencePreflight: vi.fn(async () => ({ ok: true as const })),
requireRunId(value) {
if (typeof value !== "string" || !/^[0-9a-f]{32}$/.test(value)) {
@@ -72,7 +70,6 @@ test("Evidence maintenance preflights the additive BM25 contract before starting
);
expect(deps.evidencePreflight).toHaveBeenCalledOnce();
expect(deps.semanticPreflight).not.toHaveBeenCalled();
expect(deps.runStage).toHaveBeenCalledWith([
"preprocess", "evidence", "--resume", "b".repeat(32), "--json", "-c", "/dev/fd/3",
]);
@@ -104,7 +101,6 @@ test("owns Evidence egress refusal before shared semantic infrastructure", async
);
expect(result).toEqual({ status: "failed", code: "egress_policy_refused" });
expect(deps.semanticPreflight).not.toHaveBeenCalled();
expect(deps.runStage).not.toHaveBeenCalled();
expect(deps.persistJob).not.toHaveBeenCalled();
});
harness/tests/test_corpus_pipeline.py
+24 -3
View File
@@ -277,7 +277,7 @@ def test_active_searcher_without_active_fails_closed_for_evidence(tmp_path):
from tht.evidence.search import active_searcher
class Delegate:
def search(self, embedding, top_n=10, kinds=None, metadata_filter=None):
def search(self, embedding, top_n=10, kinds=None, metadata_filter=None, **kwargs):
return ["legacy"]
cfg = SimpleNamespace(paths=SimpleNamespace(artifacts=tmp_path / "artifacts"))
@@ -332,14 +332,14 @@ def test_active_evidence_query_holds_lock_against_publish(tmp_path):
published = threading.Event()
class Delegate:
def search(self, embedding, top_n=10, kinds=None, metadata_filter=None):
def search(self, embedding, top_n=10, kinds=None, metadata_filter=None, **kwargs):
entered.set()
assert release.wait(5)
return [SimpleNamespace(id="active", similarity=1.0)]
search = threading.Thread(
target=lambda: ActiveEvidenceSearcher(store, Delegate()).search(
[1.0], kinds=["evidence"]
[1.0], kinds=["evidence"], query_text="old"
)
)
search.start()
@@ -360,6 +360,17 @@ def test_active_evidence_query_holds_lock_against_publish(tmp_path):
assert published.is_set()
def test_active_evidence_search_refuses_a_dense_only_fallback(tmp_path):
from tht.evidence.search import ActiveEvidenceSearcher
from tht.ports.vector import VectorStoreError
current = pipeline(tmp_path, Source([(item("one", "a"), "cardiomiopatia")]), vectors=Vectors())
current.run()
with pytest.raises(VectorStoreError, match="hybrid query text"):
ActiveEvidenceSearcher(current.store, object()).search([1.0], kinds=["evidence"])
def test_pipeline_result_dump_does_not_deepcopy_frozen_metadata():
manifest = CorpusManifest(metadata={"nested": {"value": ["safe"]}})
payload = PipelineResult(
@@ -716,6 +727,16 @@ def test_dimension_mismatch_fails_before_vector_write_and_publish(tmp_path):
assert candidate.store.active_generation() is None
def test_pipeline_marks_each_evidence_fragment_for_server_side_italian_bm25(tmp_path):
vectors = Vectors()
pipeline(tmp_path, Source([(item("one", "a"), "ricovero cardiologico")]), vectors=vectors).run()
assert [(record.sparse_text, record.sparse_language) for record in vectors.records] == [
("ricovero cardiologico", "italian"),
]
def test_dry_run_and_failed_acquire_never_change_active(tmp_path):
one = item("one", "a")
active = pipeline(tmp_path, Source([(one, "old")])).run().generation
harness/tests/test_evidence_facade_contract.py
+1
View File
@@ -133,6 +133,7 @@ def test_preprocessing_factory_forwards_only_evidence_pipeline_dependencies(monk
"pipeline_version": "evidence-v1",
"retain_published_generations": 2,
"workspace_id": None,
"sparse_language": "italian",
}
pipeline = build_preprocessing_pipeline(**dependencies)
harness/tests/test_preprocess_cli.py
+3
View File
@@ -236,11 +236,13 @@ def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_pat
chunk_policy,
pipeline_version,
retain_published_generations,
sparse_language,
):
calls["init"] = {
"embedding_model": embedding_model,
"embedding_dimensions": embedding_dimensions,
"pipeline_version": pipeline_version,
"sparse_language": sparse_language,
}
def run_as_job(self, **kwargs):
@@ -257,6 +259,7 @@ def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_pat
command.run_from_config(config)
assert calls["init"]["sparse_language"] == "english"
assert calls["run_as_job"]["workspace_id"] == "psd-clinical"
assert calls["run_as_job"]["input_fingerprint"] != calls["run_as_job"]["config_fingerprint"]
harness/tests/test_qdrant_vector_store.py
+95 -2
View File
@@ -86,7 +86,8 @@ class FakeQdrantHttp:
if method == "POST" and path == "/collections/workspace-semantic/points/query":
if self.malformed_query:
return FakeResponse(200, {"result": {"points": "nope"}})
wanted = _match_points(self.points.values(), json["filter"])
filter_value = json["filter"] if "filter" in json else json["prefetch"][0]["filter"]
wanted = _match_points(self.points.values(), filter_value)
scored = sorted(
(
{
@@ -150,7 +151,7 @@ def _match_clause(payload, clause):
raise AssertionError(clause)
def _write_record(record_id: str, kind: str, *, metadata=None):
def _write_record(record_id: str, kind: str, *, metadata=None, sparse_text=None, sparse_language=None):
return VectorWriteRecord(
record=VectorRecord(
id=record_id,
@@ -162,6 +163,8 @@ def _write_record(record_id: str, kind: str, *, metadata=None):
),
embedding=[0.1] * 1024,
content_hash="sha256:" + "a" * 64,
sparse_text=sparse_text,
sparse_language=sparse_language,
)
@@ -358,6 +361,89 @@ def test_upsert_serializes_qdrant_point_payloads(record, semantic_kind):
assert point["payload"]["content_hash"] == record.content_hash
def test_evidence_upsert_sends_dense_and_server_side_italian_bm25():
fake = FakeQdrantHttp()
store = _store(fake)
record = _write_record(
"demo:gen:11111111111111111111111111111111:chunk:1",
"evidence",
metadata={
"workspace_id": "demo",
"vector_generation": "gen:11111111111111111111111111111111",
"document_id": "doc:abc",
},
sparse_text="ricovero per cardiomiopatia dilatativa",
sparse_language="italian",
)
store.upsert("evidence", [record])
point = next(iter(fake.points.values()))
assert point["vector"] == {
"": record.embedding,
"bm25": {
"text": "ricovero per cardiomiopatia dilatativa",
"model": "qdrant/bm25",
"options": {"language": "italian"},
},
}
def test_evidence_search_uses_filtered_dense_and_bm25_prefetches_with_default_rrf():
fake = FakeQdrantHttp()
store = _store(fake)
generation = "gen:" + "1" * 32
store.upsert("evidence", [
_write_record(
f"demo:{generation}:chunk:1",
"evidence",
metadata={"workspace_id": "demo", "vector_generation": generation, "document_id": "doc:abc"},
sparse_text="ricovero per cardiomiopatia dilatativa",
sparse_language="italian",
)
])
store.search(
["evidence"], [0.2] * 1024, limit=10, kinds=["evidence"],
query_text="cardiomiopatia", query_language="italian",
metadata_filter={"workspace_id": "demo", "vector_generation": generation, "document_ids": ["doc:abc"]},
)
query = next(call[2] for call in reversed(fake.calls) if call[1].endswith("/points/query"))
assert query["query"] == {"rrf": {}}
assert query["limit"] == 10
assert query["prefetch"] == [
{
"query": [0.2] * 1024,
"limit": 20,
"filter": {"must": [
{"key": "workspace_id", "match": {"value": "demo"}},
{"key": "workspace_revision", "match": {"value": "a" * 40}},
{"key": "kind", "match": {"any": ["evidence"]}},
{"key": "record_kind", "match": {"any": ["evidence"]}},
{"key": "vector_generation", "match": {"value": generation}},
{"key": "document_id", "match": {"any": ["doc:abc"]}},
]},
},
{
"query": {
"text": "cardiomiopatia", "model": "qdrant/bm25",
"options": {"language": "italian"},
},
"using": "bm25",
"limit": 20,
"filter": {"must": [
{"key": "workspace_id", "match": {"value": "demo"}},
{"key": "workspace_revision", "match": {"value": "a" * 40}},
{"key": "kind", "match": {"any": ["evidence"]}},
{"key": "record_kind", "match": {"any": ["evidence"]}},
{"key": "vector_generation", "match": {"value": generation}},
{"key": "document_id", "match": {"any": ["doc:abc"]}},
]},
},
]
def test_search_filters_by_workspace_and_allowed_record_kinds():
fake = FakeQdrantHttp()
store = _store(fake)
@@ -384,6 +470,13 @@ def test_search_filters_by_workspace_and_allowed_record_kinds():
}
def test_evidence_search_refuses_dense_only_fallback():
store = _store(FakeQdrantHttp())
with pytest.raises(VectorStoreError, match="hybrid query text"):
store.search(["evidence"], [0.2] * 1024, limit=5, kinds=["evidence"])
def test_search_excludes_inconsistent_semantic_kind_in_bound_workspace():
fake = FakeQdrantHttp()
store = _store(fake)
harness/tht/adapters/vector/qdrant.py
+57 -11
View File
@@ -24,6 +24,7 @@ from tht.vectorstore.store import VectorHit, hit_from_metadata
_GENERATION = re.compile(r"gen:[0-9a-f]{32}")
_WORKSPACE = re.compile(r"[a-z][a-z0-9_-]{0,63}")
_BM25_LANGUAGES = frozenset({"english", "italian"})
_KEYWORD_INDEXES = (
"content_hash",
@@ -127,6 +128,8 @@ class QdrantVectorStore:
limit: int,
kinds: list[str] | None = None,
metadata_filter: dict[str, object] | None = None,
query_text: str | None = None,
query_language: str | None = None,
) -> list[VectorHit]:
require_positive_limit(limit)
self._validate_embedding(embedding, query=True)
@@ -155,16 +158,43 @@ class QdrantVectorStore:
{"key": "vector_generation", "match": {"value": generation}},
{"key": "document_id", "match": {"any": document_ids}},
])
response = self._call(
"POST",
f"/collections/{self._collection}/points/query",
{
"vector": embedding,
"limit": limit,
"with_payload": True,
"filter": {"must": filter_must},
},
)
if query_text is None:
if allowed_record_kinds == ["evidence"]:
raise VectorStoreError("Evidence hybrid query text is required")
response = self._call(
"POST",
f"/collections/{self._collection}/points/query",
{
"vector": embedding,
"limit": limit,
"with_payload": True,
"filter": {"must": filter_must},
},
)
else:
if allowed_record_kinds != ["evidence"]:
raise VectorStoreError("Hybrid BM25 is only available for Evidence")
if query_text.strip() == "" or query_language not in _BM25_LANGUAGES:
raise VectorStoreError("Evidence BM25 query is invalid")
shared_filter = {"must": filter_must}
response = self._call(
"POST",
f"/collections/{self._collection}/points/query",
{
"prefetch": [
{"query": embedding, "limit": limit * 2, "filter": shared_filter},
{
"query": self._bm25_document(query_text, query_language),
"using": "bm25",
"limit": limit * 2,
"filter": shared_filter,
},
],
"query": {"rrf": {}},
"limit": limit,
"with_payload": True,
},
)
points = response.get("result", {}).get("points")
if not isinstance(points, list):
raise VectorStoreError("Qdrant returned malformed query response")
@@ -201,6 +231,14 @@ class QdrantVectorStore:
validate_collection_kinds(collection, [write_record.record.kind])
self._validate_embedding(write_record.embedding, query=False)
semantic_kind = qdrant_semantic_kind(write_record.record.kind)
vector: list[float] | dict = write_record.embedding
if write_record.sparse_text is not None:
if semantic_kind != "evidence" or write_record.sparse_language not in _BM25_LANGUAGES:
raise VectorStoreError("Evidence BM25 document is invalid")
vector = {
"": write_record.embedding,
"bm25": self._bm25_document(write_record.sparse_text, write_record.sparse_language),
}
points.append(
{
"id": point_id(
@@ -209,7 +247,7 @@ class QdrantVectorStore:
write_record.record.id,
self._workspace_revision if semantic_kind in ("schema_table", "schema_column", "evidence") else None,
),
"vector": write_record.embedding,
"vector": vector,
"payload": qdrant_payload(
write_record.record,
content_hash=write_record.content_hash,
@@ -291,6 +329,14 @@ class QdrantVectorStore:
def _workspace_filter(self) -> list[dict]:
return [{"key": "workspace_id", "match": {"value": self._workspace_id}}]
@staticmethod
def _bm25_document(text: str, language: str) -> dict:
return {
"text": text,
"model": "qdrant/bm25",
"options": {"language": language},
}
def _revision_filter(self, kinds: list[str]) -> list[dict]:
if self._workspace_revision is None:
return []
harness/tht/cli/preprocess_cmd.py
+10
View File
@@ -14,6 +14,14 @@ from tht.cli.config_cmd import CONFIG_OPT
preprocess_app = typer.Typer(help="Materialize versioned preprocessing artifacts")
def _bm25_language(workspace_language: str) -> str:
languages = {"en": "english", "it": "italian"}
try:
return languages[workspace_language]
except KeyError as exc:
raise ValueError("workspace language is unsupported for Qdrant BM25") from exc
def _evidence_json_context(config: Path):
from tht.cli.schema_cmd import _load_config_or_exit
@@ -112,6 +120,7 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None =
chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
pipeline_version="evidence-v1",
retain_published_generations=cfg.vector.retain_published_generations,
sparse_language=_bm25_language(cfg.language),
)
def fingerprint(value: str) -> str:
return "sha256:" + hashlib.sha256(value.encode()).hexdigest()
@@ -145,6 +154,7 @@ def gc_from_config(config: Path, *, dry_run: bool = False):
chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
pipeline_version="evidence-v1",
retain_published_generations=cfg.vector.retain_published_generations,
sparse_language=_bm25_language(cfg.language),
)
pipeline.workspace_id = cfg._workspace_id
return pipeline.gc(workspace_root=corpus_root.parent, dry_run=dry_run)
harness/tht/cli/vector_cmd.py
+11 -1
View File
@@ -55,10 +55,20 @@ def open_searcher(cfg):
store = build_vector_store(cfg)
class AdapterSearcher:
def search(self, query_vec, top_n=10, kinds=None, metadata_filter=None):
def search(
self,
query_vec,
top_n=10,
kinds=None,
metadata_filter=None,
query_text=None,
query_language=None,
):
return store.search(
tables_for_kinds(kinds), query_vec, limit=top_n, kinds=kinds,
metadata_filter=metadata_filter,
query_text=query_text,
query_language=query_language,
)
return AdapterSearcher()
harness/tht/evidence/corpus/pipeline.py
+12 -4
View File
@@ -123,7 +123,7 @@ class CorpusPipeline:
self, *, store: CorpusStore, sources: list[EvidenceSource], embedder,
vector_store: VectorStore, embedding_model: str, embedding_dimensions: int,
chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3,
workspace_id: str | None = None,
workspace_id: str | None = None, sparse_language: str = "italian",
) -> None:
self.store = store
self.sources = sources
@@ -137,6 +137,9 @@ class CorpusPipeline:
raise ValueError("retain_published_generations must be at least 1")
self.retain_published_generations = retain_published_generations
self.workspace_id = workspace_id
if sparse_language not in {"english", "italian"}:
raise ValueError("unsupported Qdrant BM25 language")
self.sparse_language = sparse_language
def _assert_workspace_binding(self) -> None:
manifest = self.store.active_manifest()
@@ -795,9 +798,8 @@ class CorpusPipeline:
except Exception:
logger.debug("Failed to delete the unpublished vector generation", exc_info=True)
@staticmethod
def _vector_record(
chunk: CanonicalChunk, embedding: list[float], generation: str, workspace_id: str,
self, chunk: CanonicalChunk, embedding: list[float], generation: str, workspace_id: str,
):
record = VectorRecord(
id=f"{workspace_id}:{generation}:{chunk.chunk_id}",
@@ -810,4 +812,10 @@ class CorpusPipeline:
"vector_generation": generation,
},
)
return VectorWriteRecord(record=record, embedding=embedding, content_hash=chunk.content_hash)
return VectorWriteRecord(
record=record,
embedding=embedding,
content_hash=chunk.content_hash,
sparse_text=chunk.content,
sparse_language=self.sparse_language,
)
harness/tht/evidence/preprocessing.py
+2
View File
@@ -25,6 +25,7 @@ def build_preprocessing_pipeline(
pipeline_version: str,
retain_published_generations: int = 3,
workspace_id: str | None = None,
sparse_language: str = "italian",
) -> CorpusPipeline:
"""Construct preprocessing from the bounded infrastructure supplied by core."""
return CorpusPipeline(
@@ -38,6 +39,7 @@ def build_preprocessing_pipeline(
pipeline_version=pipeline_version,
retain_published_generations=retain_published_generations,
workspace_id=workspace_id,
sparse_language=sparse_language,
)
harness/tht/evidence/search.py
+27 -3
View File
@@ -3,6 +3,7 @@
import re
from tht.evidence.corpus.store import CorpusStore
from tht.ports.vector import VectorStoreError
class CorpusWorkspaceMismatchError(RuntimeError):
@@ -12,12 +13,27 @@ class CorpusWorkspaceMismatchError(RuntimeError):
class ActiveEvidenceSearcher:
"""Searcher facade that enforces ACTIVE generation predicates before LIMIT."""
def __init__(self, corpus: CorpusStore, delegate, expected_workspace_id: str | None = None):
def __init__(
self,
corpus: CorpusStore,
delegate,
expected_workspace_id: str | None = None,
evidence_language: str = "italian",
):
self.corpus = corpus
self.delegate = delegate
self.expected_workspace_id = expected_workspace_id
self.evidence_language = evidence_language
def search(self, embedding, top_n=10, kinds=None, metadata_filter=None):
def search(
self,
embedding,
top_n=10,
kinds=None,
metadata_filter=None,
query_text=None,
query_language=None,
):
requested = set(kinds) if kinds is not None else {
"schema_table", "schema_column", "evidence", "memory", "solved_question",
}
@@ -59,9 +75,13 @@ class ActiveEvidenceSearcher:
generation = mapping.get(document.document_id, manifest.vector_generation)
if generation:
by_generation.setdefault(generation, []).append(document.document_id)
if by_generation and (not isinstance(query_text, str) or query_text.strip() == ""):
raise VectorStoreError("Evidence hybrid query text is required")
for generation, document_ids in sorted(by_generation.items()):
hits.extend(self.delegate.search(
embedding, top_n=top_n, kinds=["evidence"],
query_text=query_text,
query_language=query_language or self.evidence_language,
metadata_filter={
"vector_generation": generation,
"document_ids": sorted(document_ids),
@@ -73,7 +93,11 @@ class ActiveEvidenceSearcher:
def active_searcher(cfg, delegate, *, workspace_id: str | None = None):
corpus_root = cfg.paths.artifacts.parent / "corpus"
return ActiveEvidenceSearcher(CorpusStore(corpus_root), delegate, workspace_id)
languages = {"en": "english", "it": "italian"}
language = languages.get(getattr(cfg, "language", "en"))
if language is None:
raise VectorStoreError("workspace language is unsupported for Qdrant BM25")
return ActiveEvidenceSearcher(CorpusStore(corpus_root), delegate, workspace_id, language)
def validate_corpus_workspace(cfg, workspace_id: str) -> None:
harness/tht/ports/vector.py
+4
View File
@@ -39,6 +39,8 @@ class VectorWriteRecord:
record: VectorRecord
embedding: list[float]
content_hash: str
sparse_text: str | None = None
sparse_language: str | None = None
class VectorStoreError(Exception):
@@ -74,6 +76,8 @@ class VectorStore(Protocol):
limit: int,
kinds: list[str] | None = None,
metadata_filter: dict[str, object] | None = None,
query_text: str | None = None,
query_language: str | None = None,
) -> list[VectorHit]: ...
def existing_hashes(self, collection: str, kinds: list[str]) -> dict[str, str]: ...
harness/tht/search/__init__.py
+4 -1
View File
@@ -112,7 +112,10 @@ def combined_search(
if query_vec is None:
query_vec = embedder.embed_query(keyword)
vector_hits = store.search(query_vec, top_n=top * 2, kinds=kinds)
search_kwargs = {"top_n": top * 2, "kinds": kinds}
if kinds is not None and "evidence" in kinds:
search_kwargs["query_text"] = keyword
vector_hits = store.search(query_vec, **search_kwargs)
rankings["vector"] = [(_vector_key(h), h.similarity) for h in vector_hits]
by_key = {_vector_key(h): h for h in vector_hits}