feat(evidence): evaluate retrieval with a small fixture

harness/tht/adapters/vector/qdrant.py

@@ -132,6 +132,7 @@ class QdrantVectorStore:
         metadata_filter: dict[str, object] | None = None,
         query_text: str | None = None,
         query_language: str | None = None,
+        retrieval_mode: str = "fused",
     ) -> list[VectorHit]:
         require_positive_limit(limit)
         self._validate_embedding(embedding, query=True)
@@ -185,7 +186,40 @@ class QdrantVectorStore:
                 if not isinstance(values, list) or not all(isinstance(item, str) for item in values):
                     raise VectorStoreError("Invalid vector metadata filter")
                 filter_must.extend({"key": payload_key, "match": {"value": item}} for item in values)
-        if query_text is None:
+        if retrieval_mode not in {"fused", "dense", "bm25"}:
+            raise VectorStoreError("Evidence retrieval mode is invalid")
+        if retrieval_mode == "dense":
+            if allowed_record_kinds != ["evidence"]:
+                raise VectorStoreError("Evidence branch diagnostics are only available for Evidence")
+            response = self._call(
+                "POST",
+                f"/collections/{self._collection}/points/query",
+                {
+                    "vector": embedding,
+                    "limit": limit,
+                    "with_payload": True,
+                    "filter": {"must": filter_must},
+                },
+            )
+        elif retrieval_mode == "bm25":
+            if allowed_record_kinds != ["evidence"]:
+                raise VectorStoreError("Evidence branch diagnostics are only available for Evidence")
+            if query_text is None or query_text.strip() == "" or query_language not in _BM25_LANGUAGES:
+                raise VectorStoreError("Evidence BM25 query is invalid")
+            self._ensure_collection(strict=False, require_bm25=True)
+            shared_filter = {"must": filter_must}
+            response = self._call(
+                "POST",
+                f"/collections/{self._collection}/points/query",
+                {
+                    "query": self._bm25_document(query_text, query_language),
+                    "using": "bm25",
+                    "limit": limit,
+                    "with_payload": True,
+                    "filter": shared_filter,
+                },
+            )
+        elif query_text is None:
             if allowed_record_kinds == ["evidence"]:
                 raise VectorStoreError("Evidence hybrid query text is required")
             response = self._call(

harness/tht/cli/evidence_cmd.py

@@ -10,6 +10,7 @@ from typing import Annotated
 
 import typer
 
+from tht.cli.config_cmd import CONFIG_OPT
 from tht.evidence import (
     EvidencePreparationError,
     PiEvidenceRestructurer,
@@ -63,6 +64,48 @@ def _findings_payload(findings) -> list[dict[str, object]]:
     return [finding.__dict__ for finding in findings]
 
 
+def evaluate_from_config(
+    workspace_root: Path,
+    config: Path,
+    *,
+    generation: str | None = None,
+) -> dict[str, object]:
+    """Evaluate one stored Evidence generation without changing corpus or vectors."""
+    from tht.adapters.factory import build_vector_store
+    from tht.cli.schema_cmd import _load_config_or_exit
+    from tht.cli.vector_cmd import make_embedder
+    from tht.evidence.canonical import load_curated_tree
+    from tht.evidence.corpus.store import CorpusStore
+    from tht.evidence.evaluation import evaluate_retrieval, load_evaluation_fixture
+
+    cfg = _load_config_or_exit(config)
+    store = CorpusStore(cfg.paths.artifacts.parent / "corpus")
+    manifest = store.manifest(generation) if generation is not None else store.active_manifest()
+    if manifest is None:
+        raise RuntimeError("active Evidence generation is unavailable")
+    document_generations = manifest.metadata.get("document_generations")
+    if not isinstance(document_generations, dict):
+        raise TypeError("Evidence generation is invalid")
+    language = {"en": "english", "it": "italian"}.get(cfg.language)
+    if language is None:
+        raise RuntimeError("workspace language is unsupported for Qdrant BM25")
+    report = evaluate_retrieval(
+        load_evaluation_fixture(workspace_root / "evidence" / "evaluation.yaml"),
+        workspace_revision=cfg._workspace_revision,
+        vector_generation=manifest.vector_generation,
+        document_generations=document_generations,
+        workspace_id=cfg._workspace_id,
+        language=language,
+        searcher=build_vector_store(cfg),
+        embedder=make_embedder(cfg.embeddings),
+        expected_kinds={
+            evidence.id: evidence.kind
+            for evidence in load_curated_tree(workspace_root / "evidence" / "curated")
+        },
+    )
+    return report.model_dump()
+
+
 @evidence_app.command("prepare")
 def prepare_cmd(
     workspace_root: Path,
@@ -106,6 +149,36 @@ def validate_cmd(
     raise typer.Exit(code=1)
 
 
+@evidence_app.command("evaluate")
+def evaluate_cmd(
+    workspace_root: Path,
+    config: Path = CONFIG_OPT,
+    generation: str | None = typer.Option(None, "--generation"),
+    json_output: bool = typer.Option(False, "--json"),
+) -> None:
+    """Evaluate active or selected Evidence retrieval generation without publishing it."""
+    root = _canonical_worktree(workspace_root)
+    try:
+        payload = evaluate_from_config(root, config, generation=generation)
+    except Exception:  # noqa: BLE001 - CLI reports a safe operational failure.
+        _emit({
+            "schemaVersion": 1,
+            "operation": "evidence_evaluate",
+            "status": "failed",
+            "code": "evaluation_failed",
+        }, json_output)
+        raise typer.Exit(code=1) from None
+    payload = {
+        "schemaVersion": 1,
+        "operation": "evidence_evaluate",
+        "status": "passed" if payload["passed"] else "failed",
+        **payload,
+    }
+    _emit(payload, json_output)
+    if not payload["passed"]:
+        raise typer.Exit(code=1)
+
+
 @evidence_app.command("resolve")
 def resolve_cmd(
     workspace_root: Path,

harness/tht/cli/preprocess_cmd.py

@@ -28,6 +28,53 @@ def _evidence_json_context(config: Path):
     return _load_config_or_exit(config)
 
 
+def _evaluation_workspace_root(cfg) -> Path:
+    evidence = cfg.evidence
+    if evidence is None:
+        raise RuntimeError("Evidence evaluation fixture is unavailable")
+    if evidence.source_root is not None:
+        return evidence.source_root
+    filesystem_roots = [source.root for source in evidence.sources if source.type == "filesystem"]
+    if len(filesystem_roots) != 1:
+        raise RuntimeError("Evidence evaluation requires one filesystem workspace source")
+    root = filesystem_roots[0]
+    if root.name == "curated":
+        root = root.parent
+    if root.name == "evidence":
+        return root.parent
+    return root
+
+
+def _candidate_evaluator(cfg, *, vector_store, embedder):
+    """Bind candidate publication to the same read-only retrieval evaluator as the CLI."""
+    from tht.evidence.canonical import load_curated_tree
+    from tht.evidence.evaluation import evaluate_retrieval, load_evaluation_fixture
+
+    workspace_root = _evaluation_workspace_root(cfg)
+    language = _bm25_language(cfg.language)
+
+    def evaluate(manifest):
+        document_generations = manifest.metadata.get("document_generations")
+        if not isinstance(document_generations, dict):
+            raise TypeError("candidate Evidence generation is invalid")
+        return evaluate_retrieval(
+            load_evaluation_fixture(workspace_root / "evidence" / "evaluation.yaml"),
+            workspace_revision=cfg._workspace_revision,
+            vector_generation=manifest.vector_generation,
+            document_generations=document_generations,
+            workspace_id=cfg._workspace_id,
+            language=language,
+            searcher=vector_store,
+            embedder=embedder,
+            expected_kinds={
+                evidence.id: evidence.kind
+                for evidence in load_curated_tree(workspace_root / "evidence" / "curated")
+            },
+        )
+
+    return evaluate
+
+
 def _evidence_json_payload(cfg, payload: dict, *, code: str, error: str | None = None) -> dict:
     value = {
         **payload,
@@ -112,15 +159,18 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None =
     if cfg.embeddings is None:
         raise RuntimeError("embeddings are not configured")
     corpus_root = cfg.paths.artifacts.parent / "corpus"
+    vector_store = build_vector_store(cfg, require_write=True)
+    embedder = make_embedder(cfg.embeddings)
     pipeline = build_preprocessing_pipeline(
         store=CorpusStore(corpus_root), sources=build_sources(cfg.evidence),
-        embedder=make_embedder(cfg.embeddings),
-        vector_store=build_vector_store(cfg, require_write=True),
+        embedder=embedder,
+        vector_store=vector_store,
         embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim,
         chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
         pipeline_version="evidence-v1",
         retain_published_generations=cfg.vector.retain_published_generations,
         sparse_language=_bm25_language(cfg.language),
+        candidate_evaluator=_candidate_evaluator(cfg, vector_store=vector_store, embedder=embedder),
     )
     def fingerprint(value: str) -> str:
         return "sha256:" + hashlib.sha256(value.encode()).hexdigest()

harness/tht/evidence/corpus/pipeline.py

@@ -7,7 +7,7 @@ import json
 import logging
 import re
 import uuid
-from collections.abc import Mapping, Sequence
+from collections.abc import Callable, Mapping, Sequence
 from dataclasses import asdict, dataclass, field
 from datetime import UTC
 from pathlib import Path
@@ -127,6 +127,7 @@ class CorpusPipeline:
         vector_store: VectorStore, embedding_model: str, embedding_dimensions: int,
         chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3,
         workspace_id: str | None = None, sparse_language: str = "italian",
+        candidate_evaluator: Callable[[CorpusManifest], object] | None = None,
     ) -> None:
         self.store = store
         self.sources = sources
@@ -143,6 +144,14 @@ class CorpusPipeline:
         if sparse_language not in {"english", "italian"}:
             raise ValueError("unsupported Qdrant BM25 language")
         self.sparse_language = sparse_language
+        self.candidate_evaluator = candidate_evaluator
+
+    def _evaluate_candidate(self, manifest: CorpusManifest) -> None:
+        if self.candidate_evaluator is None:
+            return
+        report = self.candidate_evaluator(manifest)
+        if getattr(report, "passed", False) is not True:
+            raise PipelineError("candidate retrieval evaluation failed")
 
     def _assert_workspace_binding(self) -> None:
         manifest = self.store.active_manifest()
@@ -658,6 +667,7 @@ class CorpusPipeline:
                     raise
             generation = read(context, "plan.json")["generation"]
             try:
+                self._evaluate_candidate(CorpusManifest.model_validate(read(context, "manifest.json")))
                 self.store.publish(generation)
             except Exception:
                 compensate(context)
@@ -791,6 +801,7 @@ class CorpusPipeline:
                     manifest, {document.document_id: document.content for document in documents},
                     generation=generation,
                 )
+            self._evaluate_candidate(manifest)
             self.store.publish(staged)
             self.gc(workspace_root=self.store.root.parent)
         except AtomicContentTooLargeError as error:

harness/tht/evidence/evaluation.py

@@ -0,0 +1,303 @@
+"""Read-only retrieval evaluation for published and candidate Evidence generations."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+
+from tht.evidence.canonical import EVIDENCE_PURPOSES
+from tht.evidence.search import EvidenceSearchContext, render_evidence_query
+
+_PROFILES = frozenset({"lexical", "semantic", "mixed"})
+
+
+class EvaluationFixtureError(ValueError):
+    """The versioned retrieval fixture is not safe to use as a publication gate."""
+
+
+class EvaluationError(RuntimeError):
+    """The configured Evidence generation could not be evaluated safely."""
+
+
+@dataclass(frozen=True)
+class EvaluationQuery:
+    query_id: str
+    query: str
+    profile: str
+    purpose: str
+    expected: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class EvaluationFixture:
+    queries: tuple[EvaluationQuery, ...]
+
+
+@dataclass(frozen=True)
+class ExpectedEvidenceReport:
+    evidence_id: str
+    kind: str | None
+    dense_rank: int | None
+    bm25_rank: int | None
+    fused_rank: int | None
+
+
+@dataclass(frozen=True)
+class EvaluationQueryReport:
+    query_id: str
+    profile: str
+    purpose: str
+    hit_at_5: bool
+    hit_at_10: bool
+    missing_expected: tuple[str, ...]
+    empty_result: bool
+    expected: tuple[ExpectedEvidenceReport, ...]
+
+
+@dataclass(frozen=True)
+class EvaluationReport:
+    workspace_revision: str
+    vector_generation: str | None
+    rrf: dict[str, object]
+    passed: bool
+    queries: tuple[EvaluationQueryReport, ...]
+    counts_by_expected_kind: dict[str, int]
+
+    def model_dump(self) -> dict[str, object]:
+        return {
+            "workspaceRevision": self.workspace_revision,
+            "vectorGeneration": self.vector_generation,
+            "rrf": self.rrf,
+            "passed": self.passed,
+            "countsByExpectedKind": self.counts_by_expected_kind,
+            "queries": [
+                {
+                    "id": query.query_id,
+                    "profile": query.profile,
+                    "purpose": query.purpose,
+                    "hitAt5": query.hit_at_5,
+                    "hitAt10": query.hit_at_10,
+                    "missingExpected": list(query.missing_expected),
+                    "emptyResult": query.empty_result,
+                    "expected": [
+                        {
+                            "evidenceId": expected.evidence_id,
+                            "kind": expected.kind,
+                            "denseRank": expected.dense_rank,
+                            "bm25Rank": expected.bm25_rank,
+                            "fusedRank": expected.fused_rank,
+                        }
+                        for expected in query.expected
+                    ],
+                }
+                for query in self.queries
+            ],
+        }
+
+
+def load_evaluation_fixture(path: Path) -> EvaluationFixture:
+    """Load the deliberately small, complete v1 evaluation fixture."""
+    try:
+        raw = yaml.safe_load(path.read_text(encoding="utf-8"))
+    except (OSError, UnicodeError, yaml.YAMLError) as error:
+        raise EvaluationFixtureError("evaluation fixture is unreadable") from error
+    if not isinstance(raw, dict) or set(raw) != {"schema_version", "queries"}:
+        raise EvaluationFixtureError("evaluation fixture schema is invalid")
+    if raw.get("schema_version") != 1 or not isinstance(raw.get("queries"), list):
+        raise EvaluationFixtureError("evaluation fixture schema is invalid")
+
+    queries: list[EvaluationQuery] = []
+    errors: set[str] = set()
+    ids: set[str] = set()
+    profiles: set[str] = set()
+    for entry in raw["queries"]:
+        entry_errors: set[str] = set()
+        if not isinstance(entry, dict) or set(entry) != {"id", "query", "profile", "purpose", "expected"}:
+            errors.add("schema")
+            continue
+        query_id = entry["id"]
+        query = entry["query"]
+        profile = entry["profile"]
+        purpose = entry["purpose"]
+        expected = entry["expected"]
+        if not isinstance(query_id, str) or not query_id.strip() or query_id in ids:
+            entry_errors.add("duplicate")
+        else:
+            ids.add(query_id)
+        if not isinstance(query, str) or not query.strip():
+            entry_errors.add("query")
+        if profile not in _PROFILES:
+            entry_errors.add("profile")
+        else:
+            profiles.add(profile)
+        if purpose not in EVIDENCE_PURPOSES:
+            entry_errors.add("purpose")
+        if (
+            not isinstance(expected, list)
+            or not expected
+            or any(not isinstance(value, str) or not value.strip() for value in expected)
+        ):
+            entry_errors.add("expected")
+        errors.update(entry_errors)
+        if not entry_errors:
+            queries.append(EvaluationQuery(query_id, query, profile, purpose, tuple(expected)))
+    missing_profiles = _PROFILES - profiles
+    if missing_profiles:
+        errors.update(missing_profiles)
+    if errors:
+        raise EvaluationFixtureError(" ".join(sorted(errors)))
+    return EvaluationFixture(tuple(queries))
+
+
+def _ranked_evidence(hits) -> tuple[dict[str, int], dict[str, str]]:
+    ranks: dict[str, int] = {}
+    kinds: dict[str, str] = {}
+    for hit in hits:
+        metadata = getattr(hit, "metadata", None)
+        if not isinstance(metadata, dict):
+            raise EvaluationError("evaluation search returned malformed Evidence payload")
+        evidence_id = metadata.get("evidence_id")
+        kind = metadata.get("evidence_kind")
+        if not isinstance(evidence_id, str) or not evidence_id or not isinstance(kind, str) or not kind:
+            raise EvaluationError("evaluation search returned malformed Evidence payload")
+        if evidence_id not in ranks:
+            ranks[evidence_id] = len(ranks) + 1
+            kinds[evidence_id] = kind
+    return ranks, kinds
+
+
+def _search_generation(
+    searcher,
+    embedding: list[float],
+    *,
+    rendered_query: str,
+    purpose: str,
+    workspace_id: str,
+    generation: str,
+    document_ids: list[str],
+    language: str,
+    retrieval_mode: str,
+):
+    return searcher.search(
+        ["evidence"],
+        embedding,
+        limit=10,
+        kinds=["evidence"],
+        query_text=rendered_query,
+        query_language=language,
+        retrieval_mode=retrieval_mode,
+        metadata_filter={
+            "workspace_id": workspace_id,
+            "vector_generation": generation,
+            "document_ids": document_ids,
+            "purpose": purpose,
+            "required_kinds": [],
+            "required_concepts": [],
+            "required_tables": [],
+            "required_columns": [],
+        },
+    )
+
+
+def evaluate_retrieval(
+    fixture: EvaluationFixture,
+    *,
+    workspace_revision: str,
+    document_generations: dict[str, str],
+    workspace_id: str,
+    language: str,
+    searcher,
+    embedder,
+    vector_generation: str | None = None,
+    expected_kinds: dict[str, str] | None = None,
+) -> EvaluationReport:
+    """Evaluate a generation with the runtime hybrid request plus branch diagnostics."""
+    if not document_generations:
+        raise EvaluationError("evaluation requires indexed Evidence documents")
+    by_generation: dict[str, list[str]] = {}
+    for document_id, generation in document_generations.items():
+        if not isinstance(document_id, str) or not isinstance(generation, str) or not generation:
+            raise EvaluationError("evaluation document generations are invalid")
+        by_generation.setdefault(generation, []).append(document_id)
+    for document_ids in by_generation.values():
+        document_ids.sort()
+
+    reports: list[EvaluationQueryReport] = []
+    expected_kinds = expected_kinds or {}
+    for query in fixture.queries:
+        rendered = render_evidence_query(query.query, EvidenceSearchContext())
+        embedding = embedder.embed_query(rendered)
+        branch_hits = {"dense": [], "bm25": [], "fused": []}
+        for generation, document_ids in sorted(by_generation.items()):
+            for mode, hits in branch_hits.items():
+                hits.extend(_search_generation(
+                    searcher,
+                    embedding,
+                    rendered_query=rendered,
+                    purpose=query.purpose,
+                    workspace_id=workspace_id,
+                    generation=generation,
+                    document_ids=document_ids,
+                    language=language,
+                    retrieval_mode=mode,
+                ))
+        ranks_by_branch: dict[str, dict[str, int]] = {}
+        kinds_by_branch: dict[str, dict[str, str]] = {}
+        for mode, hits in branch_hits.items():
+            ordered = sorted(hits, key=lambda hit: (-float(hit.similarity), str(hit.id)))
+            ranks_by_branch[mode], kinds_by_branch[mode] = _ranked_evidence(ordered)
+        expected = []
+        for evidence_id in query.expected:
+            kind = expected_kinds.get(evidence_id) or next((
+                kinds_by_branch[mode][evidence_id]
+                for mode in ("fused", "dense", "bm25")
+                if evidence_id in kinds_by_branch[mode]
+            ), None)
+            expected.append(ExpectedEvidenceReport(
+                evidence_id=evidence_id,
+                kind=kind,
+                dense_rank=ranks_by_branch["dense"].get(evidence_id),
+                bm25_rank=ranks_by_branch["bm25"].get(evidence_id),
+                fused_rank=ranks_by_branch["fused"].get(evidence_id),
+            ))
+        fused_ranks = ranks_by_branch["fused"]
+        missing = tuple(item.evidence_id for item in expected if item.fused_rank is None)
+        reports.append(EvaluationQueryReport(
+            query_id=query.query_id,
+            profile=query.profile,
+            purpose=query.purpose,
+            hit_at_5=any(item.fused_rank is not None and item.fused_rank <= 5 for item in expected),
+            hit_at_10=any(item.fused_rank is not None and item.fused_rank <= 10 for item in expected),
+            missing_expected=missing,
+            empty_result=not fused_ranks,
+            expected=tuple(expected),
+        ))
+    counts: dict[str, int] = {}
+    for query in reports:
+        for expected in query.expected:
+            if expected.kind is not None:
+                counts[expected.kind] = counts.get(expected.kind, 0) + 1
+    evaluated = vector_generation or (next(iter(by_generation)) if len(by_generation) == 1 else None)
+    return EvaluationReport(
+        workspace_revision=workspace_revision,
+        vector_generation=evaluated,
+        rrf={"algorithm": "rrf", "k": 60, "prefetch_limit_multiplier": 2},
+        passed=all(query.hit_at_10 for query in reports),
+        queries=tuple(reports),
+        counts_by_expected_kind=dict(sorted(counts.items())),
+    )
+
+
+__all__ = [
+    "EvaluationError",
+    "EvaluationFixture",
+    "EvaluationFixtureError",
+    "EvaluationQuery",
+    "EvaluationQueryReport",
+    "EvaluationReport",
+    "ExpectedEvidenceReport",
+    "evaluate_retrieval",
+    "load_evaluation_fixture",
+]

harness/tht/evidence/preprocessing.py

@@ -1,5 +1,6 @@
 """Explicit construction boundary for Evidence preprocessing."""
 
+from collections.abc import Callable
 from typing import Protocol
 
 from tht.evidence.contracts import EvidenceSource
@@ -26,6 +27,7 @@ def build_preprocessing_pipeline(
     retain_published_generations: int = 3,
     workspace_id: str | None = None,
     sparse_language: str = "italian",
+    candidate_evaluator: Callable[[object], object] | None = None,
 ) -> CorpusPipeline:
     """Construct preprocessing from the bounded infrastructure supplied by core."""
     return CorpusPipeline(
@@ -40,6 +42,7 @@ def build_preprocessing_pipeline(
         retain_published_generations=retain_published_generations,
         workspace_id=workspace_id,
         sparse_language=sparse_language,
+        candidate_evaluator=candidate_evaluator,
     )
 
 

harness/tht/ports/vector.py

@@ -79,6 +79,7 @@ class VectorStore(Protocol):
         metadata_filter: dict[str, object] | None = None,
         query_text: str | None = None,
         query_language: str | None = None,
+        retrieval_mode: str = "fused",
     ) -> list[VectorHit]: ...
 
     def existing_hashes(self, collection: str, kinds: list[str]) -> dict[str, str]: ...