feat(evidence): evaluate retrieval with a small fixture

harness/tht/cli/evidence_cmd.py

@@ -10,6 +10,7 @@ from typing import Annotated
 
 import typer
 
+from tht.cli.config_cmd import CONFIG_OPT
 from tht.evidence import (
     EvidencePreparationError,
     PiEvidenceRestructurer,
@@ -63,6 +64,48 @@ def _findings_payload(findings) -> list[dict[str, object]]:
     return [finding.__dict__ for finding in findings]
 
 
+def evaluate_from_config(
+    workspace_root: Path,
+    config: Path,
+    *,
+    generation: str | None = None,
+) -> dict[str, object]:
+    """Evaluate one stored Evidence generation without changing corpus or vectors."""
+    from tht.adapters.factory import build_vector_store
+    from tht.cli.schema_cmd import _load_config_or_exit
+    from tht.cli.vector_cmd import make_embedder
+    from tht.evidence.canonical import load_curated_tree
+    from tht.evidence.corpus.store import CorpusStore
+    from tht.evidence.evaluation import evaluate_retrieval, load_evaluation_fixture
+
+    cfg = _load_config_or_exit(config)
+    store = CorpusStore(cfg.paths.artifacts.parent / "corpus")
+    manifest = store.manifest(generation) if generation is not None else store.active_manifest()
+    if manifest is None:
+        raise RuntimeError("active Evidence generation is unavailable")
+    document_generations = manifest.metadata.get("document_generations")
+    if not isinstance(document_generations, dict):
+        raise TypeError("Evidence generation is invalid")
+    language = {"en": "english", "it": "italian"}.get(cfg.language)
+    if language is None:
+        raise RuntimeError("workspace language is unsupported for Qdrant BM25")
+    report = evaluate_retrieval(
+        load_evaluation_fixture(workspace_root / "evidence" / "evaluation.yaml"),
+        workspace_revision=cfg._workspace_revision,
+        vector_generation=manifest.vector_generation,
+        document_generations=document_generations,
+        workspace_id=cfg._workspace_id,
+        language=language,
+        searcher=build_vector_store(cfg),
+        embedder=make_embedder(cfg.embeddings),
+        expected_kinds={
+            evidence.id: evidence.kind
+            for evidence in load_curated_tree(workspace_root / "evidence" / "curated")
+        },
+    )
+    return report.model_dump()
+
+
 @evidence_app.command("prepare")
 def prepare_cmd(
     workspace_root: Path,
@@ -106,6 +149,36 @@ def validate_cmd(
     raise typer.Exit(code=1)
 
 
+@evidence_app.command("evaluate")
+def evaluate_cmd(
+    workspace_root: Path,
+    config: Path = CONFIG_OPT,
+    generation: str | None = typer.Option(None, "--generation"),
+    json_output: bool = typer.Option(False, "--json"),
+) -> None:
+    """Evaluate active or selected Evidence retrieval generation without publishing it."""
+    root = _canonical_worktree(workspace_root)
+    try:
+        payload = evaluate_from_config(root, config, generation=generation)
+    except Exception:  # noqa: BLE001 - CLI reports a safe operational failure.
+        _emit({
+            "schemaVersion": 1,
+            "operation": "evidence_evaluate",
+            "status": "failed",
+            "code": "evaluation_failed",
+        }, json_output)
+        raise typer.Exit(code=1) from None
+    payload = {
+        "schemaVersion": 1,
+        "operation": "evidence_evaluate",
+        "status": "passed" if payload["passed"] else "failed",
+        **payload,
+    }
+    _emit(payload, json_output)
+    if not payload["passed"]:
+        raise typer.Exit(code=1)
+
+
 @evidence_app.command("resolve")
 def resolve_cmd(
     workspace_root: Path,

harness/tht/cli/preprocess_cmd.py

@@ -28,6 +28,53 @@ def _evidence_json_context(config: Path):
     return _load_config_or_exit(config)
 
 
+def _evaluation_workspace_root(cfg) -> Path:
+    evidence = cfg.evidence
+    if evidence is None:
+        raise RuntimeError("Evidence evaluation fixture is unavailable")
+    if evidence.source_root is not None:
+        return evidence.source_root
+    filesystem_roots = [source.root for source in evidence.sources if source.type == "filesystem"]
+    if len(filesystem_roots) != 1:
+        raise RuntimeError("Evidence evaluation requires one filesystem workspace source")
+    root = filesystem_roots[0]
+    if root.name == "curated":
+        root = root.parent
+    if root.name == "evidence":
+        return root.parent
+    return root
+
+
+def _candidate_evaluator(cfg, *, vector_store, embedder):
+    """Bind candidate publication to the same read-only retrieval evaluator as the CLI."""
+    from tht.evidence.canonical import load_curated_tree
+    from tht.evidence.evaluation import evaluate_retrieval, load_evaluation_fixture
+
+    workspace_root = _evaluation_workspace_root(cfg)
+    language = _bm25_language(cfg.language)
+
+    def evaluate(manifest):
+        document_generations = manifest.metadata.get("document_generations")
+        if not isinstance(document_generations, dict):
+            raise TypeError("candidate Evidence generation is invalid")
+        return evaluate_retrieval(
+            load_evaluation_fixture(workspace_root / "evidence" / "evaluation.yaml"),
+            workspace_revision=cfg._workspace_revision,
+            vector_generation=manifest.vector_generation,
+            document_generations=document_generations,
+            workspace_id=cfg._workspace_id,
+            language=language,
+            searcher=vector_store,
+            embedder=embedder,
+            expected_kinds={
+                evidence.id: evidence.kind
+                for evidence in load_curated_tree(workspace_root / "evidence" / "curated")
+            },
+        )
+
+    return evaluate
+
+
 def _evidence_json_payload(cfg, payload: dict, *, code: str, error: str | None = None) -> dict:
     value = {
         **payload,
@@ -112,15 +159,18 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None =
     if cfg.embeddings is None:
         raise RuntimeError("embeddings are not configured")
     corpus_root = cfg.paths.artifacts.parent / "corpus"
+    vector_store = build_vector_store(cfg, require_write=True)
+    embedder = make_embedder(cfg.embeddings)
     pipeline = build_preprocessing_pipeline(
         store=CorpusStore(corpus_root), sources=build_sources(cfg.evidence),
-        embedder=make_embedder(cfg.embeddings),
-        vector_store=build_vector_store(cfg, require_write=True),
+        embedder=embedder,
+        vector_store=vector_store,
         embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim,
         chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
         pipeline_version="evidence-v1",
         retain_published_generations=cfg.vector.retain_published_generations,
         sparse_language=_bm25_language(cfg.language),
+        candidate_evaluator=_candidate_evaluator(cfg, vector_store=vector_store, embedder=embedder),
     )
     def fingerprint(value: str) -> str:
         return "sha256:" + hashlib.sha256(value.encode()).hexdigest()