mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(evidence): filter active data in every search

Browse Source
This commit is contained in:
marcopan
2026-07-12 05:40:45 +02:00
parent 92b1d3ccc3
commit 6656a69630
3 changed files with 118 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/tht/search/evidence.py
+34 -35
View File
@@ -11,26 +11,40 @@ class ActiveEvidenceSearcher:
self.delegate = delegate
def search(self, embedding, top_n=10, kinds=None, metadata_filter=None):
if kinds != ["evidence"]:
return self.delegate.search(embedding, top_n=top_n, kinds=kinds)
manifest = self.corpus.active_manifest()
if manifest is None:
return []
by_generation: dict[str, list[str]] = {}
mapping = dict(manifest.metadata.get("document_generations", {}))
for document in manifest.documents:
generation = mapping.get(document.document_id, manifest.vector_generation)
if generation:
by_generation.setdefault(generation, []).append(document.document_id)
hits = []
for generation, document_ids in by_generation.items():
hits.extend(self.delegate.search(
embedding, top_n=top_n, kinds=["evidence"],
metadata_filter={
"vector_generation": generation,
"document_ids": document_ids,
},
))
requested = set(kinds) if kinds is not None else {
"schema_table", "schema_column", "evidence", "memory", "solved_question",
}
include_evidence = "evidence" in requested
other_kinds = sorted(requested - {"evidence"})
if not include_evidence:
kwargs = {"top_n": top_n, "kinds": kinds}
if metadata_filter is not None:
kwargs["metadata_filter"] = metadata_filter
return self.delegate.search(embedding, **kwargs)
with self.corpus.writer_lock():
hits = []
if other_kinds:
kwargs = {"top_n": top_n, "kinds": other_kinds}
if metadata_filter is not None:
kwargs["metadata_filter"] = metadata_filter
hits.extend(self.delegate.search(embedding, **kwargs))
if include_evidence:
manifest = self.corpus.active_manifest()
if manifest is not None:
by_generation: dict[str, list[str]] = {}
mapping = dict(manifest.metadata.get("document_generations", {}))
for document in manifest.documents:
generation = mapping.get(document.document_id, manifest.vector_generation)
if generation:
by_generation.setdefault(generation, []).append(document.document_id)
for generation, document_ids in sorted(by_generation.items()):
hits.extend(self.delegate.search(
embedding, top_n=top_n, kinds=["evidence"],
metadata_filter={
"vector_generation": generation,
"document_ids": sorted(document_ids),
},
))
return sorted(hits, key=lambda hit: (-hit.similarity, hit.id))[:top_n]
@@ -39,21 +53,6 @@ def active_searcher(cfg, delegate):
return ActiveEvidenceSearcher(CorpusStore(corpus_root), delegate)
def active_evidence_hits(store: CorpusStore, vector_store, embedding, *, limit: int):
manifest = store.active_manifest()
if manifest is None or manifest.vector_generation is None:
return []
document_generations = dict(manifest.metadata.get("document_generations", {}))
active_documents = {document.document_id for document in manifest.documents}
hits = vector_store.search(["evidence"], embedding, limit=limit, kinds=["evidence"])
return [
hit for hit in hits
if hit.metadata.get("document_id") in active_documents
and hit.metadata.get("vector_generation")
== document_generations.get(hit.metadata.get("document_id"), manifest.vector_generation)
]
def resolve_evidence_file(
store: CorpusStore, evidence_id: str, *, materialized_root=None,
) -> str: