feat(evidence): evaluate retrieval with a small fixture

harness/tests/fixtures/approved_cli_surface.json

@@ -10,6 +10,7 @@
     "decision add",
     "decision add-batch",
     "decision add-join-set",
+    "evidence evaluate",
     "evidence prepare",
     "evidence resolve",
     "evidence validate",

harness/tests/test_cli_surface.py

@@ -32,7 +32,7 @@ def test_typer_tree_matches_the_approved_command_surface():
     approved = _approved_surface()
     expected = set(approved["maintained"]) | set(approved["enhanced"])
 
-    assert len(approved["maintained"]) == 59
+    assert len(approved["maintained"]) == 60
     assert len(approved["enhanced"]) == 8
     assert len(approved["erased"]) == 14
     assert not (expected & set(approved["erased"]))

harness/tests/test_corpus_pipeline.py

@@ -106,7 +106,7 @@ def item(name, fingerprint):
 
 
 def pipeline(tmp_path, source, *, embedder=None, vectors=None, model="model-a", policy=None,
-             retain=3):
+             retain=3, candidate_evaluator=None):
     return CorpusPipeline(
         store=CorpusStore(tmp_path / "corpus"), sources=[source],
         embedder=embedder or Embedder(), vector_store=vectors or Vectors(),
@@ -114,6 +114,7 @@ def pipeline(tmp_path, source, *, embedder=None, vectors=None, model="model-a",
         chunk_policy=policy or ChunkPolicy(version="chunk-v1", max_chars=100),
         pipeline_version="evidence-v1",
         retain_published_generations=retain,
+        candidate_evaluator=candidate_evaluator,
     )
 
 
@@ -853,6 +854,55 @@ def test_dimension_mismatch_fails_before_vector_write_and_publish(tmp_path):
     assert candidate.store.active_generation() is None
 
 
+def test_failed_candidate_evaluation_never_switches_the_active_generation(tmp_path):
+    from types import SimpleNamespace
+
+    vectors = Vectors()
+    active = pipeline(tmp_path, Source([(item("one", "a"), "old")]), vectors=vectors).run().generation
+    candidate = pipeline(
+        tmp_path,
+        Source([(item("one", "b"), "new")]),
+        vectors=vectors,
+        candidate_evaluator=lambda manifest: SimpleNamespace(passed=False),
+    )
+
+    with pytest.raises(PipelineError, match="candidate retrieval evaluation failed"):
+        candidate.run()
+
+    assert candidate.store.active_generation() == active
+    assert {record.record.metadata["vector_generation"] for record in vectors.records} == {active}
+
+
+def test_job_failed_candidate_evaluation_never_switches_the_active_generation(tmp_path):
+    from types import SimpleNamespace
+
+    vectors = Vectors()
+    active = pipeline(tmp_path, Source([(item("one", "a"), "old")]), vectors=vectors).run_as_job(
+        workspace_id="demo",
+        workspace_root=tmp_path,
+        config_fingerprint="sha256:" + "1" * 64,
+        input_fingerprint="sha256:" + "2" * 64,
+    ).generation
+    candidate = pipeline(
+        tmp_path,
+        Source([(item("one", "b"), "new")]),
+        vectors=vectors,
+        candidate_evaluator=lambda manifest: SimpleNamespace(passed=False),
+    )
+
+    result = candidate.run_as_job(
+        workspace_id="demo",
+        workspace_root=tmp_path,
+        config_fingerprint="sha256:" + "1" * 64,
+        input_fingerprint="sha256:" + "3" * 64,
+    )
+
+    assert result.status == "failed"
+    assert result.published is False
+    assert candidate.store.active_generation() == active
+    assert {record.record.metadata["vector_generation"] for record in vectors.records} == {active}
+
+
 def test_pipeline_marks_each_evidence_fragment_for_server_side_italian_bm25(tmp_path):
     vectors = Vectors()
 

harness/tests/test_evidence_cli.py

@@ -115,3 +115,35 @@ def test_evidence_validate_json_is_pristine_and_reports_review_required(monkeypa
         "schemaVersion": 1,
         "status": "review_required",
     }
+
+
+def test_evidence_evaluate_json_reports_the_read_only_generation(monkeypatch, tmp_path):
+    from tht.cli import evidence_cmd
+
+    monkeypatch.setattr(evidence_cmd, "_canonical_worktree", lambda root: root)
+    monkeypatch.setattr(evidence_cmd, "evaluate_from_config", lambda *args, **kwargs: {
+        "workspaceRevision": "a" * 40,
+        "vectorGeneration": "gen:" + "1" * 32,
+        "rrf": {"algorithm": "rrf", "k": 60, "prefetch_limit_multiplier": 2},
+        "passed": True,
+        "countsByExpectedKind": {"formula": 1},
+        "queries": [],
+    })
+
+    result = CliRunner().invoke(app, [
+        "evidence", "evaluate", str(tmp_path), "--json", "-c", str(tmp_path / "runtime.yaml"),
+    ])
+
+    assert result.exit_code == 0
+    assert result.stderr == ""
+    assert json.loads(result.stdout) == {
+        "countsByExpectedKind": {"formula": 1},
+        "operation": "evidence_evaluate",
+        "passed": True,
+        "queries": [],
+        "rrf": {"algorithm": "rrf", "k": 60, "prefetch_limit_multiplier": 2},
+        "schemaVersion": 1,
+        "status": "passed",
+        "vectorGeneration": "gen:" + "1" * 32,
+        "workspaceRevision": "a" * 40,
+    }

harness/tests/test_evidence_evaluation.py

@@ -0,0 +1,185 @@
+import pytest
+
+
+def _fixture(path):
+    path.write_text(
+        """
+schema_version: 1
+queries:
+  - id: lexical-code
+    query: Qual è il codice ICD-10?
+    profile: lexical
+    purpose: schema_linking
+    expected: [evidence:icd]
+  - id: semantic-age
+    query: Come distinguo i pazienti pediatrici?
+    profile: semantic
+    purpose: sql_generation
+    expected: [evidence:fascia-pediatrica]
+  - id: mixed-formula
+    query: Formula per patient.birth_date?
+    profile: mixed
+    purpose: sql_generation
+    expected: [evidence:fascia-pediatrica]
+""".strip(),
+        encoding="utf-8",
+    )
+
+
+class _Embedder:
+    def embed_query(self, query):
+        return [float(len(query))]
+
+
+class _Hit:
+    def __init__(self, evidence_id, kind, score):
+        self.id = evidence_id + ":fragment"
+        self.similarity = score
+        self.metadata = {"evidence_id": evidence_id, "evidence_kind": kind}
+
+
+class _Searcher:
+    def __init__(self):
+        self.calls = []
+
+    def search(self, collections, embedding, **kwargs):
+        self.calls.append((collections, embedding, kwargs))
+        mode = kwargs["retrieval_mode"]
+        if mode == "dense":
+            return [_Hit("evidence:fascia-pediatrica", "formula", 0.9)]
+        if mode == "bm25":
+            return [_Hit("evidence:icd", "enum", 0.8)]
+        return [
+            _Hit("evidence:fascia-pediatrica", "formula", 0.95),
+            _Hit("evidence:icd", "enum", 0.8),
+        ]
+
+
+def test_evaluation_reports_branch_and_fused_ranks_without_turning_diagnostics_into_gates(tmp_path):
+    from tht.evidence.evaluation import evaluate_retrieval, load_evaluation_fixture
+
+    fixture_path = tmp_path / "evaluation.yaml"
+    _fixture(fixture_path)
+    searcher = _Searcher()
+
+    report = evaluate_retrieval(
+        load_evaluation_fixture(fixture_path),
+        workspace_revision="a" * 40,
+        document_generations={"doc:one": "gen:" + "1" * 32},
+        workspace_id="psd-clinical",
+        language="italian",
+        searcher=searcher,
+        embedder=_Embedder(),
+    )
+
+    assert report.passed is True
+    assert report.workspace_revision == "a" * 40
+    assert report.vector_generation == "gen:" + "1" * 32
+    assert report.rrf == {"algorithm": "rrf", "k": 60, "prefetch_limit_multiplier": 2}
+    assert report.queries[0].hit_at_5 is True
+    assert report.queries[0].hit_at_10 is True
+    assert report.queries[0].missing_expected == ()
+    assert report.queries[0].expected[0].dense_rank is None
+    assert report.queries[0].expected[0].bm25_rank == 1
+    assert report.queries[0].expected[0].fused_rank == 2
+    assert report.counts_by_expected_kind == {"enum": 1, "formula": 2}
+    assert len(searcher.calls) == 9
+    assert {call[2]["retrieval_mode"] for call in searcher.calls} == {"dense", "bm25", "fused"}
+    assert all(call[2]["metadata_filter"] == {
+        "workspace_id": "psd-clinical",
+        "vector_generation": "gen:" + "1" * 32,
+        "document_ids": ["doc:one"],
+        "purpose": call[2]["metadata_filter"]["purpose"],
+        "required_kinds": [],
+        "required_concepts": [],
+        "required_tables": [],
+        "required_columns": [],
+    } for call in searcher.calls)
+
+
+def test_evaluation_fails_only_when_a_query_has_no_expected_fused_hit_in_top_ten(tmp_path):
+    from tht.evidence.evaluation import evaluate_retrieval, load_evaluation_fixture
+
+    fixture_path = tmp_path / "evaluation.yaml"
+    _fixture(fixture_path)
+
+    class MissingExpectedSearcher(_Searcher):
+        def search(self, collections, embedding, **kwargs):
+            self.calls.append((collections, embedding, kwargs))
+            return [_Hit("evidence:other", "domain", 1.0)]
+
+    report = evaluate_retrieval(
+        load_evaluation_fixture(fixture_path),
+        workspace_revision="a" * 40,
+        document_generations={"doc:one": "gen:" + "1" * 32},
+        workspace_id="psd-clinical",
+        language="italian",
+        searcher=MissingExpectedSearcher(),
+        embedder=_Embedder(),
+        expected_kinds={"evidence:icd": "enum", "evidence:fascia-pediatrica": "formula"},
+    )
+
+    assert report.passed is False
+    assert all(query.hit_at_5 is False and query.hit_at_10 is False for query in report.queries)
+    assert all(query.empty_result is False for query in report.queries)
+    assert report.queries[0].missing_expected == ("evidence:icd",)
+    assert report.counts_by_expected_kind == {"enum": 1, "formula": 2}
+
+
+def test_evaluation_fixture_requires_all_retrieval_profiles(tmp_path):
+    from tht.evidence.evaluation import EvaluationFixtureError, load_evaluation_fixture
+
+    path = tmp_path / "evaluation.yaml"
+    path.write_text(
+        """
+schema_version: 1
+queries:
+  - id: lexical-code
+    query: Qual è il codice ICD-10?
+    profile: lexical
+    purpose: schema_linking
+    expected: [evidence:icd]
+  - id: semantic-age
+    query: Come distinguo i pazienti pediatrici?
+    profile: semantic
+    purpose: sql_generation
+    expected: [evidence:fascia-pediatrica]
+""".strip(),
+        encoding="utf-8",
+    )
+
+    with pytest.raises(EvaluationFixtureError, match="mixed"):
+        load_evaluation_fixture(path)
+
+
+def test_evaluation_fixture_rejects_duplicate_ids_empty_expectations_and_private_purposes(tmp_path):
+    from tht.evidence.evaluation import EvaluationFixtureError, load_evaluation_fixture
+
+    path = tmp_path / "evaluation.yaml"
+    path.write_text(
+        """
+schema_version: 1
+queries:
+  - id: duplicate
+    query: a
+    profile: lexical
+    purpose: private
+    expected: []
+  - id: duplicate
+    query: b
+    profile: semantic
+    purpose: sql_generation
+    expected: [evidence:b]
+  - id: mixed
+    query: c
+    profile: mixed
+    purpose: rewriting
+    expected: [evidence:c]
+""".strip(),
+        encoding="utf-8",
+    )
+
+    with pytest.raises(EvaluationFixtureError) as failure:
+        load_evaluation_fixture(path)
+
+    assert {"duplicate", "expected", "purpose"} <= set(str(failure.value).split())

harness/tests/test_evidence_facade_contract.py

@@ -156,6 +156,7 @@ def test_preprocessing_factory_forwards_only_evidence_pipeline_dependencies(monk
         "retain_published_generations": 2,
         "workspace_id": None,
         "sparse_language": "italian",
+        "candidate_evaluator": None,
     }
 
     pipeline = build_preprocessing_pipeline(**dependencies)

harness/tests/test_preprocess_cli.py

@@ -237,12 +237,14 @@ def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_pat
             pipeline_version,
             retain_published_generations,
             sparse_language,
+            candidate_evaluator,
         ):
             calls["init"] = {
                 "embedding_model": embedding_model,
                 "embedding_dimensions": embedding_dimensions,
                 "pipeline_version": pipeline_version,
                 "sparse_language": sparse_language,
+                "candidate_evaluator": candidate_evaluator,
             }
 
         def run_as_job(self, **kwargs):
@@ -260,6 +262,7 @@ def test_run_from_config_uses_runtime_identity_workspace_id(monkeypatch, tmp_pat
     command.run_from_config(config)
 
     assert calls["init"]["sparse_language"] == "english"
+    assert callable(calls["init"]["candidate_evaluator"])
     assert calls["run_as_job"]["workspace_id"] == "psd-clinical"
     assert calls["run_as_job"]["input_fingerprint"] != calls["run_as_job"]["config_fingerprint"]
 

harness/tests/test_qdrant_vector_store.py

@@ -475,6 +475,35 @@ def test_evidence_search_uses_filtered_dense_and_bm25_prefetches_with_default_rr
     ]
 
 
+@pytest.mark.parametrize("retrieval_mode, expected_query", [
+    ("dense", None),
+    ("bm25", {"text": "cardiomiopatia", "model": "qdrant/bm25", "options": {"language": "italian"}}),
+])
+def test_evidence_diagnostic_branch_searches_use_the_runtime_filter(retrieval_mode, expected_query):
+    fake = FakeQdrantHttp()
+    _ready_collection_with_bm25(fake)
+    store = _store(fake)
+    generation = "gen:" + "1" * 32
+
+    store.search(
+        ["evidence"], [0.2] * 1024, limit=10, kinds=["evidence"],
+        query_text="cardiomiopatia", query_language="italian", retrieval_mode=retrieval_mode,
+        metadata_filter={"workspace_id": "demo", "vector_generation": generation, "document_ids": ["doc:abc"]},
+    )
+
+    query = next(call[2] for call in reversed(fake.calls) if call[1].endswith("/points/query"))
+    assert query["limit"] == 10
+    assert query["filter"]["must"][-2:] == [
+        {"key": "vector_generation", "match": {"value": generation}},
+        {"key": "document_id", "match": {"any": ["doc:abc"]}},
+    ]
+    if retrieval_mode == "dense":
+        assert query["vector"] == [0.2] * 1024
+    else:
+        assert query["query"] == expected_query
+        assert query["using"] == "bm25"
+
+
 def test_search_filters_by_workspace_and_allowed_record_kinds():
     fake = FakeQdrantHttp()
     store = _store(fake)