feat(evidence): use server-side Qdrant BM25 retrieval

This commit is contained in:
2026-08-24 18:15:33 +02:00
parent 0e9add09a9
commit 29d41ac258
15 changed files with 253 additions and 34 deletions
@@ -13,7 +13,6 @@ export interface EvidenceJobState {
export interface EvidencePreprocessingDependencies { export interface EvidencePreprocessingDependencies {
runStage(argv: string[]): Promise<Record<string, unknown>>; runStage(argv: string[]): Promise<Record<string, unknown>>;
persistJob(): void; persistJob(): void;
semanticPreflight(): Promise<{ ok: true } | { ok: false; code: SemanticFailureCode }>;
evidencePreflight(): Promise<{ ok: true } | { ok: false; code: SemanticFailureCode }>; evidencePreflight(): Promise<{ ok: true } | { ok: false; code: SemanticFailureCode }>;
requireRunId(value: unknown): string; requireRunId(value: unknown): string;
numberRecord(value: unknown): Record<string, number> | undefined; numberRecord(value: unknown): Record<string, number> | undefined;
@@ -139,9 +138,9 @@ export async function preprocessEvidence(
} }
const policy = evidencePolicy(request.evidence, request.httpPrivateHostAllowlist); const policy = evidencePolicy(request.evidence, request.httpPrivateHostAllowlist);
if (policy) return policy; if (policy) return policy;
const semantic = await deps.evidencePreflight(); const preflight = await deps.evidencePreflight();
if (!semantic.ok) { if (!preflight.ok) {
return { status: "failed", code: semantic.code, runId: request.job.runId }; return { status: "failed", code: preflight.code, runId: request.job.runId };
} }
if (request.job.completedStages.includes("evidence") && !request.dryRun) { if (request.job.completedStages.includes("evidence") && !request.dryRun) {
return { return {
@@ -412,7 +412,6 @@ export class WorkspacePreprocessingService {
return { return {
runStage: async (argv) => await this.runJsonStage(scope.runtime, argv), runStage: async (argv) => await this.runJsonStage(scope.runtime, argv),
persistJob: () => this.state(scope.runtime.workspaceId).writeJob(scope.job), persistJob: () => this.state(scope.runtime.workspaceId).writeJob(scope.job),
semanticPreflight: async () => await this.deps.semanticPreflight(scope.runtime.workspace),
evidencePreflight: async () => await this.deps.evidencePreflight(scope.runtime.workspace), evidencePreflight: async () => await this.deps.evidencePreflight(scope.runtime.workspace),
requireRunId: (value) => this.requireRunId(value), requireRunId: (value) => this.requireRunId(value),
numberRecord: (value) => this.numberRecord(value), numberRecord: (value) => this.numberRecord(value),
@@ -39,13 +39,11 @@ function job(overrides: Partial<EvidenceJobState> = {}): EvidenceJobState {
function dependencies(payload: Record<string, unknown> = {}): EvidencePreprocessingDependencies & { function dependencies(payload: Record<string, unknown> = {}): EvidencePreprocessingDependencies & {
runStage: ReturnType<typeof vi.fn>; runStage: ReturnType<typeof vi.fn>;
persistJob: ReturnType<typeof vi.fn>; persistJob: ReturnType<typeof vi.fn>;
semanticPreflight: ReturnType<typeof vi.fn>;
evidencePreflight: ReturnType<typeof vi.fn>; evidencePreflight: ReturnType<typeof vi.fn>;
} { } {
return { return {
runStage: vi.fn(async () => payload), runStage: vi.fn(async () => payload),
persistJob: vi.fn(), persistJob: vi.fn(),
semanticPreflight: vi.fn(async () => ({ ok: true as const })),
evidencePreflight: vi.fn(async () => ({ ok: true as const })), evidencePreflight: vi.fn(async () => ({ ok: true as const })),
requireRunId(value) { requireRunId(value) {
if (typeof value !== "string" || !/^[0-9a-f]{32}$/.test(value)) { if (typeof value !== "string" || !/^[0-9a-f]{32}$/.test(value)) {
@@ -72,7 +70,6 @@ test("Evidence maintenance preflights the additive BM25 contract before starting
); );
expect(deps.evidencePreflight).toHaveBeenCalledOnce(); expect(deps.evidencePreflight).toHaveBeenCalledOnce();
expect(deps.semanticPreflight).not.toHaveBeenCalled();
expect(deps.runStage).toHaveBeenCalledWith([ expect(deps.runStage).toHaveBeenCalledWith([
"preprocess", "evidence", "--resume", "b".repeat(32), "--json", "-c", "/dev/fd/3", "preprocess", "evidence", "--resume", "b".repeat(32), "--json", "-c", "/dev/fd/3",
]); ]);
@@ -104,7 +101,6 @@ test("owns Evidence egress refusal before shared semantic infrastructure", async
); );
expect(result).toEqual({ status: "failed", code: "egress_policy_refused" }); expect(result).toEqual({ status: "failed", code: "egress_policy_refused" });
expect(deps.semanticPreflight).not.toHaveBeenCalled();
expect(deps.runStage).not.toHaveBeenCalled(); expect(deps.runStage).not.toHaveBeenCalled();
expect(deps.persistJob).not.toHaveBeenCalled(); expect(deps.persistJob).not.toHaveBeenCalled();
}); });
+24 -3
View File
@@ -277,7 +277,7 @@ def test_active_searcher_without_active_fails_closed_for_evidence(tmp_path):
from tht.evidence.search import active_searcher from tht.evidence.search import active_searcher
class Delegate: class Delegate:
def search(self, embedding, top_n=10, kinds=None, metadata_filter=None): def search(self, embedding, top_n=10, kinds=None, metadata_filter=None, **kwargs):
return ["legacy"] return ["legacy"]
cfg = SimpleNamespace(paths=SimpleNamespace(artifacts=tmp_path / "artifacts")) cfg = SimpleNamespace(paths=SimpleNamespace(artifacts=tmp_path / "artifacts"))
@@ -332,14 +332,14 @@ def test_active_evidence_query_holds_lock_against_publish(tmp_path):
published = threading.Event() published = threading.Event()
class Delegate: class Delegate:
def search(self, embedding, top_n=10, kinds=None, metadata_filter=None): def search(self, embedding, top_n=10, kinds=None, metadata_filter=None, **kwargs):
entered.set() entered.set()
assert release.wait(5) assert release.wait(5)
return [SimpleNamespace(id="active", similarity=1.0)] return [SimpleNamespace(id="active", similarity=1.0)]
search = threading.Thread( search = threading.Thread(
target=lambda: ActiveEvidenceSearcher(store, Delegate()).search( target=lambda: ActiveEvidenceSearcher(store, Delegate()).search(
[1.0], kinds=["evidence"] [1.0], kinds=["evidence"], query_text="old"
) )
) )
search.start() search.start()
@@ -360,6 +360,17 @@ def test_active_evidence_query_holds_lock_against_publish(tmp_path):
assert published.is_set() assert published.is_set()
def test_active_evidence_search_refuses_a_dense_only_fallback(tmp_path):
from tht.evidence.search import ActiveEvidenceSearcher
from tht.ports.vector import VectorStoreError
current = pipeline(tmp_path, Source([(item("one", "a"), "cardiomiopatia")]), vectors=Vectors())
current.run()
with pytest.raises(VectorStoreError, match="hybrid query text"):
ActiveEvidenceSearcher(current.store, object()).search([1.0], kinds=["evidence"])
def test_pipeline_result_dump_does_not_deepcopy_frozen_metadata(): def test_pipeline_result_dump_does_not_deepcopy_frozen_metadata():
manifest = CorpusManifest(metadata={"nested": {"value": ["safe"]}}) manifest = CorpusManifest(metadata={"nested": {"value": ["safe"]}})
payload = PipelineResult( payload = PipelineResult(
@@ -716,6 +727,16 @@ def test_dimension_mismatch_fails_before_vector_write_and_publish(tmp_path):
assert candidate.store.active_generation() is None assert candidate.store.active_generation() is None
def test_pipeline_marks_each_evidence_fragment_for_server_side_italian_bm25(tmp_path):
vectors = Vectors()
pipeline(tmp_path, Source([(item("one", "a"), "ricovero cardiologico")]), vectors=vectors).run()
assert [(record.sparse_text, record.sparse_language) for record in vectors.records] == [
("ricovero cardiologico", "italian"),
]
def test_dry_run_and_failed_acquire_never_change_active(tmp_path): def test_dry_run_and_failed_acquire_never_change_active(tmp_path):
one = item("one", "a") one = item("one", "a")
active = pipeline(tmp_path, Source([(one, "old")])).run().generation active = pipeline(tmp_path, Source([(one, "old")])).run().generation
@@ -133,6 +133,7 @@ def test_preprocessing_factory_forwards_only_evidence_pipeline_dependencies(monk
"pipeline_version": "evidence-v1", "pipeline_version": "evidence-v1",
"retain_published_generations": 2, "retain_published_generations": 2,
"workspace_id": None, "workspace_id": None,
"sparse_language": "italian",
} }
pipeline = build_preprocessing_pipeline(**dependencies) pipeline = build_preprocessing_pipeline(**dependencies)
+3
View File
@@ -236,11 +236,13 @@ def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_pat
chunk_policy, chunk_policy,
pipeline_version, pipeline_version,
retain_published_generations, retain_published_generations,
sparse_language,
): ):
calls["init"] = { calls["init"] = {
"embedding_model": embedding_model, "embedding_model": embedding_model,
"embedding_dimensions": embedding_dimensions, "embedding_dimensions": embedding_dimensions,
"pipeline_version": pipeline_version, "pipeline_version": pipeline_version,
"sparse_language": sparse_language,
} }
def run_as_job(self, **kwargs): def run_as_job(self, **kwargs):
@@ -257,6 +259,7 @@ def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_pat
command.run_from_config(config) command.run_from_config(config)
assert calls["init"]["sparse_language"] == "english"
assert calls["run_as_job"]["workspace_id"] == "psd-clinical" assert calls["run_as_job"]["workspace_id"] == "psd-clinical"
assert calls["run_as_job"]["input_fingerprint"] != calls["run_as_job"]["config_fingerprint"] assert calls["run_as_job"]["input_fingerprint"] != calls["run_as_job"]["config_fingerprint"]
+95 -2
View File
@@ -86,7 +86,8 @@ class FakeQdrantHttp:
if method == "POST" and path == "/collections/workspace-semantic/points/query": if method == "POST" and path == "/collections/workspace-semantic/points/query":
if self.malformed_query: if self.malformed_query:
return FakeResponse(200, {"result": {"points": "nope"}}) return FakeResponse(200, {"result": {"points": "nope"}})
wanted = _match_points(self.points.values(), json["filter"]) filter_value = json["filter"] if "filter" in json else json["prefetch"][0]["filter"]
wanted = _match_points(self.points.values(), filter_value)
scored = sorted( scored = sorted(
( (
{ {
@@ -150,7 +151,7 @@ def _match_clause(payload, clause):
raise AssertionError(clause) raise AssertionError(clause)
def _write_record(record_id: str, kind: str, *, metadata=None): def _write_record(record_id: str, kind: str, *, metadata=None, sparse_text=None, sparse_language=None):
return VectorWriteRecord( return VectorWriteRecord(
record=VectorRecord( record=VectorRecord(
id=record_id, id=record_id,
@@ -162,6 +163,8 @@ def _write_record(record_id: str, kind: str, *, metadata=None):
), ),
embedding=[0.1] * 1024, embedding=[0.1] * 1024,
content_hash="sha256:" + "a" * 64, content_hash="sha256:" + "a" * 64,
sparse_text=sparse_text,
sparse_language=sparse_language,
) )
@@ -358,6 +361,89 @@ def test_upsert_serializes_qdrant_point_payloads(record, semantic_kind):
assert point["payload"]["content_hash"] == record.content_hash assert point["payload"]["content_hash"] == record.content_hash
def test_evidence_upsert_sends_dense_and_server_side_italian_bm25():
fake = FakeQdrantHttp()
store = _store(fake)
record = _write_record(
"demo:gen:11111111111111111111111111111111:chunk:1",
"evidence",
metadata={
"workspace_id": "demo",
"vector_generation": "gen:11111111111111111111111111111111",
"document_id": "doc:abc",
},
sparse_text="ricovero per cardiomiopatia dilatativa",
sparse_language="italian",
)
store.upsert("evidence", [record])
point = next(iter(fake.points.values()))
assert point["vector"] == {
"": record.embedding,
"bm25": {
"text": "ricovero per cardiomiopatia dilatativa",
"model": "qdrant/bm25",
"options": {"language": "italian"},
},
}
def test_evidence_search_uses_filtered_dense_and_bm25_prefetches_with_default_rrf():
fake = FakeQdrantHttp()
store = _store(fake)
generation = "gen:" + "1" * 32
store.upsert("evidence", [
_write_record(
f"demo:{generation}:chunk:1",
"evidence",
metadata={"workspace_id": "demo", "vector_generation": generation, "document_id": "doc:abc"},
sparse_text="ricovero per cardiomiopatia dilatativa",
sparse_language="italian",
)
])
store.search(
["evidence"], [0.2] * 1024, limit=10, kinds=["evidence"],
query_text="cardiomiopatia", query_language="italian",
metadata_filter={"workspace_id": "demo", "vector_generation": generation, "document_ids": ["doc:abc"]},
)
query = next(call[2] for call in reversed(fake.calls) if call[1].endswith("/points/query"))
assert query["query"] == {"rrf": {}}
assert query["limit"] == 10
assert query["prefetch"] == [
{
"query": [0.2] * 1024,
"limit": 20,
"filter": {"must": [
{"key": "workspace_id", "match": {"value": "demo"}},
{"key": "workspace_revision", "match": {"value": "a" * 40}},
{"key": "kind", "match": {"any": ["evidence"]}},
{"key": "record_kind", "match": {"any": ["evidence"]}},
{"key": "vector_generation", "match": {"value": generation}},
{"key": "document_id", "match": {"any": ["doc:abc"]}},
]},
},
{
"query": {
"text": "cardiomiopatia", "model": "qdrant/bm25",
"options": {"language": "italian"},
},
"using": "bm25",
"limit": 20,
"filter": {"must": [
{"key": "workspace_id", "match": {"value": "demo"}},
{"key": "workspace_revision", "match": {"value": "a" * 40}},
{"key": "kind", "match": {"any": ["evidence"]}},
{"key": "record_kind", "match": {"any": ["evidence"]}},
{"key": "vector_generation", "match": {"value": generation}},
{"key": "document_id", "match": {"any": ["doc:abc"]}},
]},
},
]
def test_search_filters_by_workspace_and_allowed_record_kinds(): def test_search_filters_by_workspace_and_allowed_record_kinds():
fake = FakeQdrantHttp() fake = FakeQdrantHttp()
store = _store(fake) store = _store(fake)
@@ -384,6 +470,13 @@ def test_search_filters_by_workspace_and_allowed_record_kinds():
} }
def test_evidence_search_refuses_dense_only_fallback():
store = _store(FakeQdrantHttp())
with pytest.raises(VectorStoreError, match="hybrid query text"):
store.search(["evidence"], [0.2] * 1024, limit=5, kinds=["evidence"])
def test_search_excludes_inconsistent_semantic_kind_in_bound_workspace(): def test_search_excludes_inconsistent_semantic_kind_in_bound_workspace():
fake = FakeQdrantHttp() fake = FakeQdrantHttp()
store = _store(fake) store = _store(fake)
+57 -11
View File
@@ -24,6 +24,7 @@ from tht.vectorstore.store import VectorHit, hit_from_metadata
_GENERATION = re.compile(r"gen:[0-9a-f]{32}") _GENERATION = re.compile(r"gen:[0-9a-f]{32}")
_WORKSPACE = re.compile(r"[a-z][a-z0-9_-]{0,63}") _WORKSPACE = re.compile(r"[a-z][a-z0-9_-]{0,63}")
_BM25_LANGUAGES = frozenset({"english", "italian"})
_KEYWORD_INDEXES = ( _KEYWORD_INDEXES = (
"content_hash", "content_hash",
@@ -127,6 +128,8 @@ class QdrantVectorStore:
limit: int, limit: int,
kinds: list[str] | None = None, kinds: list[str] | None = None,
metadata_filter: dict[str, object] | None = None, metadata_filter: dict[str, object] | None = None,
query_text: str | None = None,
query_language: str | None = None,
) -> list[VectorHit]: ) -> list[VectorHit]:
require_positive_limit(limit) require_positive_limit(limit)
self._validate_embedding(embedding, query=True) self._validate_embedding(embedding, query=True)
@@ -155,16 +158,43 @@ class QdrantVectorStore:
{"key": "vector_generation", "match": {"value": generation}}, {"key": "vector_generation", "match": {"value": generation}},
{"key": "document_id", "match": {"any": document_ids}}, {"key": "document_id", "match": {"any": document_ids}},
]) ])
response = self._call( if query_text is None:
"POST", if allowed_record_kinds == ["evidence"]:
f"/collections/{self._collection}/points/query", raise VectorStoreError("Evidence hybrid query text is required")
{ response = self._call(
"vector": embedding, "POST",
"limit": limit, f"/collections/{self._collection}/points/query",
"with_payload": True, {
"filter": {"must": filter_must}, "vector": embedding,
}, "limit": limit,
) "with_payload": True,
"filter": {"must": filter_must},
},
)
else:
if allowed_record_kinds != ["evidence"]:
raise VectorStoreError("Hybrid BM25 is only available for Evidence")
if query_text.strip() == "" or query_language not in _BM25_LANGUAGES:
raise VectorStoreError("Evidence BM25 query is invalid")
shared_filter = {"must": filter_must}
response = self._call(
"POST",
f"/collections/{self._collection}/points/query",
{
"prefetch": [
{"query": embedding, "limit": limit * 2, "filter": shared_filter},
{
"query": self._bm25_document(query_text, query_language),
"using": "bm25",
"limit": limit * 2,
"filter": shared_filter,
},
],
"query": {"rrf": {}},
"limit": limit,
"with_payload": True,
},
)
points = response.get("result", {}).get("points") points = response.get("result", {}).get("points")
if not isinstance(points, list): if not isinstance(points, list):
raise VectorStoreError("Qdrant returned malformed query response") raise VectorStoreError("Qdrant returned malformed query response")
@@ -201,6 +231,14 @@ class QdrantVectorStore:
validate_collection_kinds(collection, [write_record.record.kind]) validate_collection_kinds(collection, [write_record.record.kind])
self._validate_embedding(write_record.embedding, query=False) self._validate_embedding(write_record.embedding, query=False)
semantic_kind = qdrant_semantic_kind(write_record.record.kind) semantic_kind = qdrant_semantic_kind(write_record.record.kind)
vector: list[float] | dict = write_record.embedding
if write_record.sparse_text is not None:
if semantic_kind != "evidence" or write_record.sparse_language not in _BM25_LANGUAGES:
raise VectorStoreError("Evidence BM25 document is invalid")
vector = {
"": write_record.embedding,
"bm25": self._bm25_document(write_record.sparse_text, write_record.sparse_language),
}
points.append( points.append(
{ {
"id": point_id( "id": point_id(
@@ -209,7 +247,7 @@ class QdrantVectorStore:
write_record.record.id, write_record.record.id,
self._workspace_revision if semantic_kind in ("schema_table", "schema_column", "evidence") else None, self._workspace_revision if semantic_kind in ("schema_table", "schema_column", "evidence") else None,
), ),
"vector": write_record.embedding, "vector": vector,
"payload": qdrant_payload( "payload": qdrant_payload(
write_record.record, write_record.record,
content_hash=write_record.content_hash, content_hash=write_record.content_hash,
@@ -291,6 +329,14 @@ class QdrantVectorStore:
def _workspace_filter(self) -> list[dict]: def _workspace_filter(self) -> list[dict]:
return [{"key": "workspace_id", "match": {"value": self._workspace_id}}] return [{"key": "workspace_id", "match": {"value": self._workspace_id}}]
@staticmethod
def _bm25_document(text: str, language: str) -> dict:
return {
"text": text,
"model": "qdrant/bm25",
"options": {"language": language},
}
def _revision_filter(self, kinds: list[str]) -> list[dict]: def _revision_filter(self, kinds: list[str]) -> list[dict]:
if self._workspace_revision is None: if self._workspace_revision is None:
return [] return []
+10
View File
@@ -14,6 +14,14 @@ from tht.cli.config_cmd import CONFIG_OPT
preprocess_app = typer.Typer(help="Materialize versioned preprocessing artifacts") preprocess_app = typer.Typer(help="Materialize versioned preprocessing artifacts")
def _bm25_language(workspace_language: str) -> str:
languages = {"en": "english", "it": "italian"}
try:
return languages[workspace_language]
except KeyError as exc:
raise ValueError("workspace language is unsupported for Qdrant BM25") from exc
def _evidence_json_context(config: Path): def _evidence_json_context(config: Path):
from tht.cli.schema_cmd import _load_config_or_exit from tht.cli.schema_cmd import _load_config_or_exit
@@ -112,6 +120,7 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None =
chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars), chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
pipeline_version="evidence-v1", pipeline_version="evidence-v1",
retain_published_generations=cfg.vector.retain_published_generations, retain_published_generations=cfg.vector.retain_published_generations,
sparse_language=_bm25_language(cfg.language),
) )
def fingerprint(value: str) -> str: def fingerprint(value: str) -> str:
return "sha256:" + hashlib.sha256(value.encode()).hexdigest() return "sha256:" + hashlib.sha256(value.encode()).hexdigest()
@@ -145,6 +154,7 @@ def gc_from_config(config: Path, *, dry_run: bool = False):
chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars), chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
pipeline_version="evidence-v1", pipeline_version="evidence-v1",
retain_published_generations=cfg.vector.retain_published_generations, retain_published_generations=cfg.vector.retain_published_generations,
sparse_language=_bm25_language(cfg.language),
) )
pipeline.workspace_id = cfg._workspace_id pipeline.workspace_id = cfg._workspace_id
return pipeline.gc(workspace_root=corpus_root.parent, dry_run=dry_run) return pipeline.gc(workspace_root=corpus_root.parent, dry_run=dry_run)
+11 -1
View File
@@ -55,10 +55,20 @@ def open_searcher(cfg):
store = build_vector_store(cfg) store = build_vector_store(cfg)
class AdapterSearcher: class AdapterSearcher:
def search(self, query_vec, top_n=10, kinds=None, metadata_filter=None): def search(
self,
query_vec,
top_n=10,
kinds=None,
metadata_filter=None,
query_text=None,
query_language=None,
):
return store.search( return store.search(
tables_for_kinds(kinds), query_vec, limit=top_n, kinds=kinds, tables_for_kinds(kinds), query_vec, limit=top_n, kinds=kinds,
metadata_filter=metadata_filter, metadata_filter=metadata_filter,
query_text=query_text,
query_language=query_language,
) )
return AdapterSearcher() return AdapterSearcher()
+12 -4
View File
@@ -123,7 +123,7 @@ class CorpusPipeline:
self, *, store: CorpusStore, sources: list[EvidenceSource], embedder, self, *, store: CorpusStore, sources: list[EvidenceSource], embedder,
vector_store: VectorStore, embedding_model: str, embedding_dimensions: int, vector_store: VectorStore, embedding_model: str, embedding_dimensions: int,
chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3, chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3,
workspace_id: str | None = None, workspace_id: str | None = None, sparse_language: str = "italian",
) -> None: ) -> None:
self.store = store self.store = store
self.sources = sources self.sources = sources
@@ -137,6 +137,9 @@ class CorpusPipeline:
raise ValueError("retain_published_generations must be at least 1") raise ValueError("retain_published_generations must be at least 1")
self.retain_published_generations = retain_published_generations self.retain_published_generations = retain_published_generations
self.workspace_id = workspace_id self.workspace_id = workspace_id
if sparse_language not in {"english", "italian"}:
raise ValueError("unsupported Qdrant BM25 language")
self.sparse_language = sparse_language
def _assert_workspace_binding(self) -> None: def _assert_workspace_binding(self) -> None:
manifest = self.store.active_manifest() manifest = self.store.active_manifest()
@@ -795,9 +798,8 @@ class CorpusPipeline:
except Exception: except Exception:
logger.debug("Failed to delete the unpublished vector generation", exc_info=True) logger.debug("Failed to delete the unpublished vector generation", exc_info=True)
@staticmethod
def _vector_record( def _vector_record(
chunk: CanonicalChunk, embedding: list[float], generation: str, workspace_id: str, self, chunk: CanonicalChunk, embedding: list[float], generation: str, workspace_id: str,
): ):
record = VectorRecord( record = VectorRecord(
id=f"{workspace_id}:{generation}:{chunk.chunk_id}", id=f"{workspace_id}:{generation}:{chunk.chunk_id}",
@@ -810,4 +812,10 @@ class CorpusPipeline:
"vector_generation": generation, "vector_generation": generation,
}, },
) )
return VectorWriteRecord(record=record, embedding=embedding, content_hash=chunk.content_hash) return VectorWriteRecord(
record=record,
embedding=embedding,
content_hash=chunk.content_hash,
sparse_text=chunk.content,
sparse_language=self.sparse_language,
)
+2
View File
@@ -25,6 +25,7 @@ def build_preprocessing_pipeline(
pipeline_version: str, pipeline_version: str,
retain_published_generations: int = 3, retain_published_generations: int = 3,
workspace_id: str | None = None, workspace_id: str | None = None,
sparse_language: str = "italian",
) -> CorpusPipeline: ) -> CorpusPipeline:
"""Construct preprocessing from the bounded infrastructure supplied by core.""" """Construct preprocessing from the bounded infrastructure supplied by core."""
return CorpusPipeline( return CorpusPipeline(
@@ -38,6 +39,7 @@ def build_preprocessing_pipeline(
pipeline_version=pipeline_version, pipeline_version=pipeline_version,
retain_published_generations=retain_published_generations, retain_published_generations=retain_published_generations,
workspace_id=workspace_id, workspace_id=workspace_id,
sparse_language=sparse_language,
) )
+27 -3
View File
@@ -3,6 +3,7 @@
import re import re
from tht.evidence.corpus.store import CorpusStore from tht.evidence.corpus.store import CorpusStore
from tht.ports.vector import VectorStoreError
class CorpusWorkspaceMismatchError(RuntimeError): class CorpusWorkspaceMismatchError(RuntimeError):
@@ -12,12 +13,27 @@ class CorpusWorkspaceMismatchError(RuntimeError):
class ActiveEvidenceSearcher: class ActiveEvidenceSearcher:
"""Searcher facade that enforces ACTIVE generation predicates before LIMIT.""" """Searcher facade that enforces ACTIVE generation predicates before LIMIT."""
def __init__(self, corpus: CorpusStore, delegate, expected_workspace_id: str | None = None): def __init__(
self,
corpus: CorpusStore,
delegate,
expected_workspace_id: str | None = None,
evidence_language: str = "italian",
):
self.corpus = corpus self.corpus = corpus
self.delegate = delegate self.delegate = delegate
self.expected_workspace_id = expected_workspace_id self.expected_workspace_id = expected_workspace_id
self.evidence_language = evidence_language
def search(self, embedding, top_n=10, kinds=None, metadata_filter=None): def search(
self,
embedding,
top_n=10,
kinds=None,
metadata_filter=None,
query_text=None,
query_language=None,
):
requested = set(kinds) if kinds is not None else { requested = set(kinds) if kinds is not None else {
"schema_table", "schema_column", "evidence", "memory", "solved_question", "schema_table", "schema_column", "evidence", "memory", "solved_question",
} }
@@ -59,9 +75,13 @@ class ActiveEvidenceSearcher:
generation = mapping.get(document.document_id, manifest.vector_generation) generation = mapping.get(document.document_id, manifest.vector_generation)
if generation: if generation:
by_generation.setdefault(generation, []).append(document.document_id) by_generation.setdefault(generation, []).append(document.document_id)
if by_generation and (not isinstance(query_text, str) or query_text.strip() == ""):
raise VectorStoreError("Evidence hybrid query text is required")
for generation, document_ids in sorted(by_generation.items()): for generation, document_ids in sorted(by_generation.items()):
hits.extend(self.delegate.search( hits.extend(self.delegate.search(
embedding, top_n=top_n, kinds=["evidence"], embedding, top_n=top_n, kinds=["evidence"],
query_text=query_text,
query_language=query_language or self.evidence_language,
metadata_filter={ metadata_filter={
"vector_generation": generation, "vector_generation": generation,
"document_ids": sorted(document_ids), "document_ids": sorted(document_ids),
@@ -73,7 +93,11 @@ class ActiveEvidenceSearcher:
def active_searcher(cfg, delegate, *, workspace_id: str | None = None): def active_searcher(cfg, delegate, *, workspace_id: str | None = None):
corpus_root = cfg.paths.artifacts.parent / "corpus" corpus_root = cfg.paths.artifacts.parent / "corpus"
return ActiveEvidenceSearcher(CorpusStore(corpus_root), delegate, workspace_id) languages = {"en": "english", "it": "italian"}
language = languages.get(getattr(cfg, "language", "en"))
if language is None:
raise VectorStoreError("workspace language is unsupported for Qdrant BM25")
return ActiveEvidenceSearcher(CorpusStore(corpus_root), delegate, workspace_id, language)
def validate_corpus_workspace(cfg, workspace_id: str) -> None: def validate_corpus_workspace(cfg, workspace_id: str) -> None:
+4
View File
@@ -39,6 +39,8 @@ class VectorWriteRecord:
record: VectorRecord record: VectorRecord
embedding: list[float] embedding: list[float]
content_hash: str content_hash: str
sparse_text: str | None = None
sparse_language: str | None = None
class VectorStoreError(Exception): class VectorStoreError(Exception):
@@ -74,6 +76,8 @@ class VectorStore(Protocol):
limit: int, limit: int,
kinds: list[str] | None = None, kinds: list[str] | None = None,
metadata_filter: dict[str, object] | None = None, metadata_filter: dict[str, object] | None = None,
query_text: str | None = None,
query_language: str | None = None,
) -> list[VectorHit]: ... ) -> list[VectorHit]: ...
def existing_hashes(self, collection: str, kinds: list[str]) -> dict[str, str]: ... def existing_hashes(self, collection: str, kinds: list[str]) -> dict[str, str]: ...
+4 -1
View File
@@ -112,7 +112,10 @@ def combined_search(
if query_vec is None: if query_vec is None:
query_vec = embedder.embed_query(keyword) query_vec = embedder.embed_query(keyword)
vector_hits = store.search(query_vec, top_n=top * 2, kinds=kinds) search_kwargs = {"top_n": top * 2, "kinds": kinds}
if kinds is not None and "evidence" in kinds:
search_kwargs["query_text"] = keyword
vector_hits = store.search(query_vec, **search_kwargs)
rankings["vector"] = [(_vector_key(h), h.similarity) for h in vector_hits] rankings["vector"] = [(_vector_key(h), h.similarity) for h in vector_hits]
by_key = {_vector_key(h): h for h in vector_hits} by_key = {_vector_key(h): h for h in vector_hits}