139 lines
6.0 KiB
Python
139 lines
6.0 KiB
Python
"""Evidence-owned runtime lookup bound to the atomically active corpus generation."""
|
|
|
|
import re
|
|
|
|
from tht.evidence.corpus.store import CorpusStore
|
|
|
|
|
|
class CorpusWorkspaceMismatchError(RuntimeError):
|
|
"""The configured workspace does not own the persisted corpus."""
|
|
|
|
|
|
class ActiveEvidenceSearcher:
|
|
"""Searcher facade that enforces ACTIVE generation predicates before LIMIT."""
|
|
|
|
def __init__(self, corpus: CorpusStore, delegate, expected_workspace_id: str | None = None):
|
|
self.corpus = corpus
|
|
self.delegate = delegate
|
|
self.expected_workspace_id = expected_workspace_id
|
|
|
|
def search(self, embedding, top_n=10, kinds=None, metadata_filter=None):
|
|
requested = set(kinds) if kinds is not None else {
|
|
"schema_table", "schema_column", "evidence", "memory", "solved_question",
|
|
}
|
|
include_evidence = "evidence" in requested
|
|
other_kinds = sorted(requested - {"evidence"})
|
|
with self.corpus.writer_lock():
|
|
manifest = self.corpus.active_manifest()
|
|
persisted_workspace = manifest.metadata.get("workspace_id") if manifest else None
|
|
if manifest is not None and (
|
|
not isinstance(persisted_workspace, str)
|
|
or re.fullmatch(r"[a-z][a-z0-9_-]{0,63}", persisted_workspace) is None
|
|
):
|
|
raise CorpusWorkspaceMismatchError(
|
|
"corpus workspace ownership is missing or invalid; use a new corpus root or rebuild"
|
|
)
|
|
if manifest is not None and self.expected_workspace_id is not None and (
|
|
persisted_workspace != self.expected_workspace_id
|
|
):
|
|
raise CorpusWorkspaceMismatchError(
|
|
"corpus belongs to a different workspace; use a new corpus root or rebuild"
|
|
)
|
|
if not include_evidence:
|
|
kwargs = {"top_n": top_n, "kinds": kinds}
|
|
if metadata_filter is not None:
|
|
kwargs["metadata_filter"] = metadata_filter
|
|
return self.delegate.search(embedding, **kwargs)
|
|
hits = []
|
|
if other_kinds:
|
|
kwargs = {"top_n": top_n, "kinds": other_kinds}
|
|
if metadata_filter is not None:
|
|
kwargs["metadata_filter"] = metadata_filter
|
|
hits.extend(self.delegate.search(embedding, **kwargs))
|
|
if include_evidence:
|
|
workspace_id = manifest.metadata.get("workspace_id") if manifest else None
|
|
if manifest is not None and isinstance(workspace_id, str):
|
|
by_generation: dict[str, list[str]] = {}
|
|
mapping = dict(manifest.metadata.get("document_generations", {}))
|
|
for document in manifest.documents:
|
|
generation = mapping.get(document.document_id, manifest.vector_generation)
|
|
if generation:
|
|
by_generation.setdefault(generation, []).append(document.document_id)
|
|
for generation, document_ids in sorted(by_generation.items()):
|
|
hits.extend(self.delegate.search(
|
|
embedding, top_n=top_n, kinds=["evidence"],
|
|
metadata_filter={
|
|
"vector_generation": generation,
|
|
"document_ids": sorted(document_ids),
|
|
"workspace_id": workspace_id,
|
|
},
|
|
))
|
|
return sorted(hits, key=lambda hit: (-hit.similarity, hit.id))[:top_n]
|
|
|
|
|
|
def active_searcher(cfg, delegate, *, workspace_id: str | None = None):
|
|
corpus_root = cfg.paths.artifacts.parent / "corpus"
|
|
return ActiveEvidenceSearcher(CorpusStore(corpus_root), delegate, workspace_id)
|
|
|
|
|
|
def validate_corpus_workspace(cfg, workspace_id: str) -> None:
|
|
"""Fail before downstream retrieval setup when configured corpus ownership differs."""
|
|
corpus = CorpusStore(cfg.paths.artifacts.parent / "corpus")
|
|
with corpus.writer_lock():
|
|
manifest = corpus.active_manifest()
|
|
if manifest is None:
|
|
return
|
|
persisted = manifest.metadata.get("workspace_id")
|
|
if not isinstance(persisted, str) or re.fullmatch(
|
|
r"[a-z][a-z0-9_-]{0,63}", persisted
|
|
) is None:
|
|
raise CorpusWorkspaceMismatchError(
|
|
"corpus workspace ownership is missing or invalid; use a new corpus root or rebuild"
|
|
)
|
|
if persisted != workspace_id:
|
|
raise CorpusWorkspaceMismatchError(
|
|
"corpus belongs to a different workspace; use a new corpus root or rebuild"
|
|
)
|
|
|
|
|
|
def resolve_citation(
|
|
store: CorpusStore, evidence_id: str, *, materialized_root=None,
|
|
) -> str:
|
|
with store.writer_lock():
|
|
manifest = store.active_manifest()
|
|
if manifest is None:
|
|
return ""
|
|
for document in manifest.documents:
|
|
frontmatter = document.metadata.get("frontmatter", {})
|
|
identifiers = {document.document_id, document.source_id, str(frontmatter.get("id", ""))}
|
|
if evidence_id in identifiers:
|
|
root = materialized_root or (store.root / "runtime")
|
|
filename = document.document_id.removeprefix("doc:") + ".md"
|
|
path = store.materialize_document(
|
|
document.document_id, root / filename, generation=manifest.manifest_id,
|
|
)
|
|
return str(path) if path else ""
|
|
return ""
|
|
|
|
|
|
def build_retrieval_entries(results, *, excerpt_chars: int) -> list[dict]:
|
|
"""Project ordered Evidence search hits into the retrieval-pack shape."""
|
|
return [
|
|
{
|
|
"title": result.label,
|
|
"status": result.status,
|
|
"excerpt": result.content[:excerpt_chars],
|
|
}
|
|
for result in results
|
|
]
|
|
|
|
|
|
__all__ = [
|
|
"ActiveEvidenceSearcher",
|
|
"CorpusWorkspaceMismatchError",
|
|
"active_searcher",
|
|
"build_retrieval_entries",
|
|
"resolve_citation",
|
|
"validate_corpus_workspace",
|
|
]
|