diff --git a/backend/src/workspaces/evidence/preprocessing.ts b/backend/src/workspaces/evidence/preprocessing.ts index 48a734d5..7e09c894 100644 --- a/backend/src/workspaces/evidence/preprocessing.ts +++ b/backend/src/workspaces/evidence/preprocessing.ts @@ -13,7 +13,6 @@ export interface EvidenceJobState { export interface EvidencePreprocessingDependencies { runStage(argv: string[]): Promise>; persistJob(): void; - semanticPreflight(): Promise<{ ok: true } | { ok: false; code: SemanticFailureCode }>; evidencePreflight(): Promise<{ ok: true } | { ok: false; code: SemanticFailureCode }>; requireRunId(value: unknown): string; numberRecord(value: unknown): Record | undefined; @@ -139,9 +138,9 @@ export async function preprocessEvidence( } const policy = evidencePolicy(request.evidence, request.httpPrivateHostAllowlist); if (policy) return policy; - const semantic = await deps.evidencePreflight(); - if (!semantic.ok) { - return { status: "failed", code: semantic.code, runId: request.job.runId }; + const preflight = await deps.evidencePreflight(); + if (!preflight.ok) { + return { status: "failed", code: preflight.code, runId: request.job.runId }; } if (request.job.completedStages.includes("evidence") && !request.dryRun) { return { diff --git a/backend/src/workspaces/preprocessing-service.ts b/backend/src/workspaces/preprocessing-service.ts index 6f4ab1ca..f7dbc4f0 100644 --- a/backend/src/workspaces/preprocessing-service.ts +++ b/backend/src/workspaces/preprocessing-service.ts @@ -412,7 +412,6 @@ export class WorkspacePreprocessingService { return { runStage: async (argv) => await this.runJsonStage(scope.runtime, argv), persistJob: () => this.state(scope.runtime.workspaceId).writeJob(scope.job), - semanticPreflight: async () => await this.deps.semanticPreflight(scope.runtime.workspace), evidencePreflight: async () => await this.deps.evidencePreflight(scope.runtime.workspace), requireRunId: (value) => this.requireRunId(value), numberRecord: (value) => this.numberRecord(value), diff --git a/backend/test/workspaces/evidence/preprocessing.test.ts b/backend/test/workspaces/evidence/preprocessing.test.ts index b13e9932..143ca260 100644 --- a/backend/test/workspaces/evidence/preprocessing.test.ts +++ b/backend/test/workspaces/evidence/preprocessing.test.ts @@ -39,13 +39,11 @@ function job(overrides: Partial = {}): EvidenceJobState { function dependencies(payload: Record = {}): EvidencePreprocessingDependencies & { runStage: ReturnType; persistJob: ReturnType; - semanticPreflight: ReturnType; evidencePreflight: ReturnType; } { return { runStage: vi.fn(async () => payload), persistJob: vi.fn(), - semanticPreflight: vi.fn(async () => ({ ok: true as const })), evidencePreflight: vi.fn(async () => ({ ok: true as const })), requireRunId(value) { if (typeof value !== "string" || !/^[0-9a-f]{32}$/.test(value)) { @@ -72,7 +70,6 @@ test("Evidence maintenance preflights the additive BM25 contract before starting ); expect(deps.evidencePreflight).toHaveBeenCalledOnce(); - expect(deps.semanticPreflight).not.toHaveBeenCalled(); expect(deps.runStage).toHaveBeenCalledWith([ "preprocess", "evidence", "--resume", "b".repeat(32), "--json", "-c", "/dev/fd/3", ]); @@ -104,7 +101,6 @@ test("owns Evidence egress refusal before shared semantic infrastructure", async ); expect(result).toEqual({ status: "failed", code: "egress_policy_refused" }); - expect(deps.semanticPreflight).not.toHaveBeenCalled(); expect(deps.runStage).not.toHaveBeenCalled(); expect(deps.persistJob).not.toHaveBeenCalled(); }); diff --git a/harness/tests/test_corpus_pipeline.py b/harness/tests/test_corpus_pipeline.py index 0940fccd..101343a4 100644 --- a/harness/tests/test_corpus_pipeline.py +++ b/harness/tests/test_corpus_pipeline.py @@ -277,7 +277,7 @@ def test_active_searcher_without_active_fails_closed_for_evidence(tmp_path): from tht.evidence.search import active_searcher class Delegate: - def search(self, embedding, top_n=10, kinds=None, metadata_filter=None): + def search(self, embedding, top_n=10, kinds=None, metadata_filter=None, **kwargs): return ["legacy"] cfg = SimpleNamespace(paths=SimpleNamespace(artifacts=tmp_path / "artifacts")) @@ -332,14 +332,14 @@ def test_active_evidence_query_holds_lock_against_publish(tmp_path): published = threading.Event() class Delegate: - def search(self, embedding, top_n=10, kinds=None, metadata_filter=None): + def search(self, embedding, top_n=10, kinds=None, metadata_filter=None, **kwargs): entered.set() assert release.wait(5) return [SimpleNamespace(id="active", similarity=1.0)] search = threading.Thread( target=lambda: ActiveEvidenceSearcher(store, Delegate()).search( - [1.0], kinds=["evidence"] + [1.0], kinds=["evidence"], query_text="old" ) ) search.start() @@ -360,6 +360,17 @@ def test_active_evidence_query_holds_lock_against_publish(tmp_path): assert published.is_set() +def test_active_evidence_search_refuses_a_dense_only_fallback(tmp_path): + from tht.evidence.search import ActiveEvidenceSearcher + from tht.ports.vector import VectorStoreError + + current = pipeline(tmp_path, Source([(item("one", "a"), "cardiomiopatia")]), vectors=Vectors()) + current.run() + + with pytest.raises(VectorStoreError, match="hybrid query text"): + ActiveEvidenceSearcher(current.store, object()).search([1.0], kinds=["evidence"]) + + def test_pipeline_result_dump_does_not_deepcopy_frozen_metadata(): manifest = CorpusManifest(metadata={"nested": {"value": ["safe"]}}) payload = PipelineResult( @@ -716,6 +727,16 @@ def test_dimension_mismatch_fails_before_vector_write_and_publish(tmp_path): assert candidate.store.active_generation() is None +def test_pipeline_marks_each_evidence_fragment_for_server_side_italian_bm25(tmp_path): + vectors = Vectors() + + pipeline(tmp_path, Source([(item("one", "a"), "ricovero cardiologico")]), vectors=vectors).run() + + assert [(record.sparse_text, record.sparse_language) for record in vectors.records] == [ + ("ricovero cardiologico", "italian"), + ] + + def test_dry_run_and_failed_acquire_never_change_active(tmp_path): one = item("one", "a") active = pipeline(tmp_path, Source([(one, "old")])).run().generation diff --git a/harness/tests/test_evidence_facade_contract.py b/harness/tests/test_evidence_facade_contract.py index f69b12a7..93b20330 100644 --- a/harness/tests/test_evidence_facade_contract.py +++ b/harness/tests/test_evidence_facade_contract.py @@ -133,6 +133,7 @@ def test_preprocessing_factory_forwards_only_evidence_pipeline_dependencies(monk "pipeline_version": "evidence-v1", "retain_published_generations": 2, "workspace_id": None, + "sparse_language": "italian", } pipeline = build_preprocessing_pipeline(**dependencies) diff --git a/harness/tests/test_preprocess_cli.py b/harness/tests/test_preprocess_cli.py index 7bba5994..ea75b0d2 100644 --- a/harness/tests/test_preprocess_cli.py +++ b/harness/tests/test_preprocess_cli.py @@ -236,11 +236,13 @@ def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_pat chunk_policy, pipeline_version, retain_published_generations, + sparse_language, ): calls["init"] = { "embedding_model": embedding_model, "embedding_dimensions": embedding_dimensions, "pipeline_version": pipeline_version, + "sparse_language": sparse_language, } def run_as_job(self, **kwargs): @@ -257,6 +259,7 @@ def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_pat command.run_from_config(config) + assert calls["init"]["sparse_language"] == "english" assert calls["run_as_job"]["workspace_id"] == "psd-clinical" assert calls["run_as_job"]["input_fingerprint"] != calls["run_as_job"]["config_fingerprint"] diff --git a/harness/tests/test_qdrant_vector_store.py b/harness/tests/test_qdrant_vector_store.py index 1175668e..0d61db10 100644 --- a/harness/tests/test_qdrant_vector_store.py +++ b/harness/tests/test_qdrant_vector_store.py @@ -86,7 +86,8 @@ class FakeQdrantHttp: if method == "POST" and path == "/collections/workspace-semantic/points/query": if self.malformed_query: return FakeResponse(200, {"result": {"points": "nope"}}) - wanted = _match_points(self.points.values(), json["filter"]) + filter_value = json["filter"] if "filter" in json else json["prefetch"][0]["filter"] + wanted = _match_points(self.points.values(), filter_value) scored = sorted( ( { @@ -150,7 +151,7 @@ def _match_clause(payload, clause): raise AssertionError(clause) -def _write_record(record_id: str, kind: str, *, metadata=None): +def _write_record(record_id: str, kind: str, *, metadata=None, sparse_text=None, sparse_language=None): return VectorWriteRecord( record=VectorRecord( id=record_id, @@ -162,6 +163,8 @@ def _write_record(record_id: str, kind: str, *, metadata=None): ), embedding=[0.1] * 1024, content_hash="sha256:" + "a" * 64, + sparse_text=sparse_text, + sparse_language=sparse_language, ) @@ -358,6 +361,89 @@ def test_upsert_serializes_qdrant_point_payloads(record, semantic_kind): assert point["payload"]["content_hash"] == record.content_hash +def test_evidence_upsert_sends_dense_and_server_side_italian_bm25(): + fake = FakeQdrantHttp() + store = _store(fake) + record = _write_record( + "demo:gen:11111111111111111111111111111111:chunk:1", + "evidence", + metadata={ + "workspace_id": "demo", + "vector_generation": "gen:11111111111111111111111111111111", + "document_id": "doc:abc", + }, + sparse_text="ricovero per cardiomiopatia dilatativa", + sparse_language="italian", + ) + + store.upsert("evidence", [record]) + + point = next(iter(fake.points.values())) + assert point["vector"] == { + "": record.embedding, + "bm25": { + "text": "ricovero per cardiomiopatia dilatativa", + "model": "qdrant/bm25", + "options": {"language": "italian"}, + }, + } + + +def test_evidence_search_uses_filtered_dense_and_bm25_prefetches_with_default_rrf(): + fake = FakeQdrantHttp() + store = _store(fake) + generation = "gen:" + "1" * 32 + store.upsert("evidence", [ + _write_record( + f"demo:{generation}:chunk:1", + "evidence", + metadata={"workspace_id": "demo", "vector_generation": generation, "document_id": "doc:abc"}, + sparse_text="ricovero per cardiomiopatia dilatativa", + sparse_language="italian", + ) + ]) + + store.search( + ["evidence"], [0.2] * 1024, limit=10, kinds=["evidence"], + query_text="cardiomiopatia", query_language="italian", + metadata_filter={"workspace_id": "demo", "vector_generation": generation, "document_ids": ["doc:abc"]}, + ) + + query = next(call[2] for call in reversed(fake.calls) if call[1].endswith("/points/query")) + assert query["query"] == {"rrf": {}} + assert query["limit"] == 10 + assert query["prefetch"] == [ + { + "query": [0.2] * 1024, + "limit": 20, + "filter": {"must": [ + {"key": "workspace_id", "match": {"value": "demo"}}, + {"key": "workspace_revision", "match": {"value": "a" * 40}}, + {"key": "kind", "match": {"any": ["evidence"]}}, + {"key": "record_kind", "match": {"any": ["evidence"]}}, + {"key": "vector_generation", "match": {"value": generation}}, + {"key": "document_id", "match": {"any": ["doc:abc"]}}, + ]}, + }, + { + "query": { + "text": "cardiomiopatia", "model": "qdrant/bm25", + "options": {"language": "italian"}, + }, + "using": "bm25", + "limit": 20, + "filter": {"must": [ + {"key": "workspace_id", "match": {"value": "demo"}}, + {"key": "workspace_revision", "match": {"value": "a" * 40}}, + {"key": "kind", "match": {"any": ["evidence"]}}, + {"key": "record_kind", "match": {"any": ["evidence"]}}, + {"key": "vector_generation", "match": {"value": generation}}, + {"key": "document_id", "match": {"any": ["doc:abc"]}}, + ]}, + }, + ] + + def test_search_filters_by_workspace_and_allowed_record_kinds(): fake = FakeQdrantHttp() store = _store(fake) @@ -384,6 +470,13 @@ def test_search_filters_by_workspace_and_allowed_record_kinds(): } +def test_evidence_search_refuses_dense_only_fallback(): + store = _store(FakeQdrantHttp()) + + with pytest.raises(VectorStoreError, match="hybrid query text"): + store.search(["evidence"], [0.2] * 1024, limit=5, kinds=["evidence"]) + + def test_search_excludes_inconsistent_semantic_kind_in_bound_workspace(): fake = FakeQdrantHttp() store = _store(fake) diff --git a/harness/tht/adapters/vector/qdrant.py b/harness/tht/adapters/vector/qdrant.py index b4f0cdad..aded7b04 100644 --- a/harness/tht/adapters/vector/qdrant.py +++ b/harness/tht/adapters/vector/qdrant.py @@ -24,6 +24,7 @@ from tht.vectorstore.store import VectorHit, hit_from_metadata _GENERATION = re.compile(r"gen:[0-9a-f]{32}") _WORKSPACE = re.compile(r"[a-z][a-z0-9_-]{0,63}") +_BM25_LANGUAGES = frozenset({"english", "italian"}) _KEYWORD_INDEXES = ( "content_hash", @@ -127,6 +128,8 @@ class QdrantVectorStore: limit: int, kinds: list[str] | None = None, metadata_filter: dict[str, object] | None = None, + query_text: str | None = None, + query_language: str | None = None, ) -> list[VectorHit]: require_positive_limit(limit) self._validate_embedding(embedding, query=True) @@ -155,16 +158,43 @@ class QdrantVectorStore: {"key": "vector_generation", "match": {"value": generation}}, {"key": "document_id", "match": {"any": document_ids}}, ]) - response = self._call( - "POST", - f"/collections/{self._collection}/points/query", - { - "vector": embedding, - "limit": limit, - "with_payload": True, - "filter": {"must": filter_must}, - }, - ) + if query_text is None: + if allowed_record_kinds == ["evidence"]: + raise VectorStoreError("Evidence hybrid query text is required") + response = self._call( + "POST", + f"/collections/{self._collection}/points/query", + { + "vector": embedding, + "limit": limit, + "with_payload": True, + "filter": {"must": filter_must}, + }, + ) + else: + if allowed_record_kinds != ["evidence"]: + raise VectorStoreError("Hybrid BM25 is only available for Evidence") + if query_text.strip() == "" or query_language not in _BM25_LANGUAGES: + raise VectorStoreError("Evidence BM25 query is invalid") + shared_filter = {"must": filter_must} + response = self._call( + "POST", + f"/collections/{self._collection}/points/query", + { + "prefetch": [ + {"query": embedding, "limit": limit * 2, "filter": shared_filter}, + { + "query": self._bm25_document(query_text, query_language), + "using": "bm25", + "limit": limit * 2, + "filter": shared_filter, + }, + ], + "query": {"rrf": {}}, + "limit": limit, + "with_payload": True, + }, + ) points = response.get("result", {}).get("points") if not isinstance(points, list): raise VectorStoreError("Qdrant returned malformed query response") @@ -201,6 +231,14 @@ class QdrantVectorStore: validate_collection_kinds(collection, [write_record.record.kind]) self._validate_embedding(write_record.embedding, query=False) semantic_kind = qdrant_semantic_kind(write_record.record.kind) + vector: list[float] | dict = write_record.embedding + if write_record.sparse_text is not None: + if semantic_kind != "evidence" or write_record.sparse_language not in _BM25_LANGUAGES: + raise VectorStoreError("Evidence BM25 document is invalid") + vector = { + "": write_record.embedding, + "bm25": self._bm25_document(write_record.sparse_text, write_record.sparse_language), + } points.append( { "id": point_id( @@ -209,7 +247,7 @@ class QdrantVectorStore: write_record.record.id, self._workspace_revision if semantic_kind in ("schema_table", "schema_column", "evidence") else None, ), - "vector": write_record.embedding, + "vector": vector, "payload": qdrant_payload( write_record.record, content_hash=write_record.content_hash, @@ -291,6 +329,14 @@ class QdrantVectorStore: def _workspace_filter(self) -> list[dict]: return [{"key": "workspace_id", "match": {"value": self._workspace_id}}] + @staticmethod + def _bm25_document(text: str, language: str) -> dict: + return { + "text": text, + "model": "qdrant/bm25", + "options": {"language": language}, + } + def _revision_filter(self, kinds: list[str]) -> list[dict]: if self._workspace_revision is None: return [] diff --git a/harness/tht/cli/preprocess_cmd.py b/harness/tht/cli/preprocess_cmd.py index 7e852914..ab3a64b4 100644 --- a/harness/tht/cli/preprocess_cmd.py +++ b/harness/tht/cli/preprocess_cmd.py @@ -14,6 +14,14 @@ from tht.cli.config_cmd import CONFIG_OPT preprocess_app = typer.Typer(help="Materialize versioned preprocessing artifacts") +def _bm25_language(workspace_language: str) -> str: + languages = {"en": "english", "it": "italian"} + try: + return languages[workspace_language] + except KeyError as exc: + raise ValueError("workspace language is unsupported for Qdrant BM25") from exc + + def _evidence_json_context(config: Path): from tht.cli.schema_cmd import _load_config_or_exit @@ -112,6 +120,7 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None = chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars), pipeline_version="evidence-v1", retain_published_generations=cfg.vector.retain_published_generations, + sparse_language=_bm25_language(cfg.language), ) def fingerprint(value: str) -> str: return "sha256:" + hashlib.sha256(value.encode()).hexdigest() @@ -145,6 +154,7 @@ def gc_from_config(config: Path, *, dry_run: bool = False): chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars), pipeline_version="evidence-v1", retain_published_generations=cfg.vector.retain_published_generations, + sparse_language=_bm25_language(cfg.language), ) pipeline.workspace_id = cfg._workspace_id return pipeline.gc(workspace_root=corpus_root.parent, dry_run=dry_run) diff --git a/harness/tht/cli/vector_cmd.py b/harness/tht/cli/vector_cmd.py index f3db51c5..41a92dff 100644 --- a/harness/tht/cli/vector_cmd.py +++ b/harness/tht/cli/vector_cmd.py @@ -55,10 +55,20 @@ def open_searcher(cfg): store = build_vector_store(cfg) class AdapterSearcher: - def search(self, query_vec, top_n=10, kinds=None, metadata_filter=None): + def search( + self, + query_vec, + top_n=10, + kinds=None, + metadata_filter=None, + query_text=None, + query_language=None, + ): return store.search( tables_for_kinds(kinds), query_vec, limit=top_n, kinds=kinds, metadata_filter=metadata_filter, + query_text=query_text, + query_language=query_language, ) return AdapterSearcher() diff --git a/harness/tht/evidence/corpus/pipeline.py b/harness/tht/evidence/corpus/pipeline.py index 37bf7c31..969858d5 100644 --- a/harness/tht/evidence/corpus/pipeline.py +++ b/harness/tht/evidence/corpus/pipeline.py @@ -123,7 +123,7 @@ class CorpusPipeline: self, *, store: CorpusStore, sources: list[EvidenceSource], embedder, vector_store: VectorStore, embedding_model: str, embedding_dimensions: int, chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3, - workspace_id: str | None = None, + workspace_id: str | None = None, sparse_language: str = "italian", ) -> None: self.store = store self.sources = sources @@ -137,6 +137,9 @@ class CorpusPipeline: raise ValueError("retain_published_generations must be at least 1") self.retain_published_generations = retain_published_generations self.workspace_id = workspace_id + if sparse_language not in {"english", "italian"}: + raise ValueError("unsupported Qdrant BM25 language") + self.sparse_language = sparse_language def _assert_workspace_binding(self) -> None: manifest = self.store.active_manifest() @@ -795,9 +798,8 @@ class CorpusPipeline: except Exception: logger.debug("Failed to delete the unpublished vector generation", exc_info=True) - @staticmethod def _vector_record( - chunk: CanonicalChunk, embedding: list[float], generation: str, workspace_id: str, + self, chunk: CanonicalChunk, embedding: list[float], generation: str, workspace_id: str, ): record = VectorRecord( id=f"{workspace_id}:{generation}:{chunk.chunk_id}", @@ -810,4 +812,10 @@ class CorpusPipeline: "vector_generation": generation, }, ) - return VectorWriteRecord(record=record, embedding=embedding, content_hash=chunk.content_hash) + return VectorWriteRecord( + record=record, + embedding=embedding, + content_hash=chunk.content_hash, + sparse_text=chunk.content, + sparse_language=self.sparse_language, + ) diff --git a/harness/tht/evidence/preprocessing.py b/harness/tht/evidence/preprocessing.py index ca7a726a..be2f269b 100644 --- a/harness/tht/evidence/preprocessing.py +++ b/harness/tht/evidence/preprocessing.py @@ -25,6 +25,7 @@ def build_preprocessing_pipeline( pipeline_version: str, retain_published_generations: int = 3, workspace_id: str | None = None, + sparse_language: str = "italian", ) -> CorpusPipeline: """Construct preprocessing from the bounded infrastructure supplied by core.""" return CorpusPipeline( @@ -38,6 +39,7 @@ def build_preprocessing_pipeline( pipeline_version=pipeline_version, retain_published_generations=retain_published_generations, workspace_id=workspace_id, + sparse_language=sparse_language, ) diff --git a/harness/tht/evidence/search.py b/harness/tht/evidence/search.py index 108dc229..ec7210b9 100644 --- a/harness/tht/evidence/search.py +++ b/harness/tht/evidence/search.py @@ -3,6 +3,7 @@ import re from tht.evidence.corpus.store import CorpusStore +from tht.ports.vector import VectorStoreError class CorpusWorkspaceMismatchError(RuntimeError): @@ -12,12 +13,27 @@ class CorpusWorkspaceMismatchError(RuntimeError): class ActiveEvidenceSearcher: """Searcher facade that enforces ACTIVE generation predicates before LIMIT.""" - def __init__(self, corpus: CorpusStore, delegate, expected_workspace_id: str | None = None): + def __init__( + self, + corpus: CorpusStore, + delegate, + expected_workspace_id: str | None = None, + evidence_language: str = "italian", + ): self.corpus = corpus self.delegate = delegate self.expected_workspace_id = expected_workspace_id + self.evidence_language = evidence_language - def search(self, embedding, top_n=10, kinds=None, metadata_filter=None): + def search( + self, + embedding, + top_n=10, + kinds=None, + metadata_filter=None, + query_text=None, + query_language=None, + ): requested = set(kinds) if kinds is not None else { "schema_table", "schema_column", "evidence", "memory", "solved_question", } @@ -59,9 +75,13 @@ class ActiveEvidenceSearcher: generation = mapping.get(document.document_id, manifest.vector_generation) if generation: by_generation.setdefault(generation, []).append(document.document_id) + if by_generation and (not isinstance(query_text, str) or query_text.strip() == ""): + raise VectorStoreError("Evidence hybrid query text is required") for generation, document_ids in sorted(by_generation.items()): hits.extend(self.delegate.search( embedding, top_n=top_n, kinds=["evidence"], + query_text=query_text, + query_language=query_language or self.evidence_language, metadata_filter={ "vector_generation": generation, "document_ids": sorted(document_ids), @@ -73,7 +93,11 @@ class ActiveEvidenceSearcher: def active_searcher(cfg, delegate, *, workspace_id: str | None = None): corpus_root = cfg.paths.artifacts.parent / "corpus" - return ActiveEvidenceSearcher(CorpusStore(corpus_root), delegate, workspace_id) + languages = {"en": "english", "it": "italian"} + language = languages.get(getattr(cfg, "language", "en")) + if language is None: + raise VectorStoreError("workspace language is unsupported for Qdrant BM25") + return ActiveEvidenceSearcher(CorpusStore(corpus_root), delegate, workspace_id, language) def validate_corpus_workspace(cfg, workspace_id: str) -> None: diff --git a/harness/tht/ports/vector.py b/harness/tht/ports/vector.py index e2bdef21..f16c20b1 100644 --- a/harness/tht/ports/vector.py +++ b/harness/tht/ports/vector.py @@ -39,6 +39,8 @@ class VectorWriteRecord: record: VectorRecord embedding: list[float] content_hash: str + sparse_text: str | None = None + sparse_language: str | None = None class VectorStoreError(Exception): @@ -74,6 +76,8 @@ class VectorStore(Protocol): limit: int, kinds: list[str] | None = None, metadata_filter: dict[str, object] | None = None, + query_text: str | None = None, + query_language: str | None = None, ) -> list[VectorHit]: ... def existing_hashes(self, collection: str, kinds: list[str]) -> dict[str, str]: ... diff --git a/harness/tht/search/__init__.py b/harness/tht/search/__init__.py index e976d93c..e572bde9 100644 --- a/harness/tht/search/__init__.py +++ b/harness/tht/search/__init__.py @@ -112,7 +112,10 @@ def combined_search( if query_vec is None: query_vec = embedder.embed_query(keyword) - vector_hits = store.search(query_vec, top_n=top * 2, kinds=kinds) + search_kwargs = {"top_n": top * 2, "kinds": kinds} + if kinds is not None and "evidence" in kinds: + search_kwargs["query_text"] = keyword + vector_hits = store.search(query_vec, **search_kwargs) rankings["vector"] = [(_vector_key(h), h.similarity) for h in vector_hits] by_key = {_vector_key(h): h for h in vector_hits}