feat(evidence): add safe retention and materialized reads

harness/tests/l0/test_vector_adapter_parity.py

@@ -8,7 +8,7 @@ from tht.adapters.vector.pgvector import PgVectorStore
 from tht.adapters.vector.thoth_http import ThothHttpVectorStore
 from tht.config import DatabaseConfig, RestConfig
 from tht.ports.vector import VectorRecord, VectorStoreError, VectorWriteRecord
-from tht.vectorstore.rest_client import VectorRestClient
+from tht.vectorstore.rest_client import VectorRestClient, VectorRestError
 
 
 def _write(record_id, kind, embedding, content_hash):
@@ -226,3 +226,27 @@ def test_http_adapter_legacy_fallback_preserves_kind_semantics(monkeypatch):
     )
     assert [hit.id for hit in hits] == ["right"]
     assert "kinds" in calls[0] and "kinds" not in calls[1]
+
+
+def test_http_delete_generation_uses_exact_allowlisted_rpc_payload(monkeypatch):
+    calls = []
+    monkeypatch.setattr(
+        "tht.vectorstore.rest_client.requests.post",
+        lambda url, json, **kwargs: calls.append((url, json)) or Response({"deleted": 2}),
+    )
+    client = VectorRestClient(RestConfig(base_url="https://vectors.test", api_key="writer"))
+    assert client.delete_generation("evidence", "gen:" + "a" * 32) == 2
+    assert calls == [("https://vectors.test/rpc/delete_vector_generation", {
+        "table_name": "evidence", "kind": "evidence", "generation": "gen:" + "a" * 32,
+    })]
+
+
+def test_http_delete_generation_legacy_404_fails_closed_without_body_leak(monkeypatch):
+    monkeypatch.setattr(
+        "tht.vectorstore.rest_client.requests.post",
+        lambda *args, **kwargs: Response({"message": "secret legacy endpoint detail"}, status=404),
+    )
+    client = VectorRestClient(RestConfig(base_url="https://vectors.test", api_key="writer"))
+    with pytest.raises(VectorRestError, match="delete_vector_generation RPC is unavailable") as error:
+        client.delete_generation("evidence", "gen:" + "a" * 32)
+    assert "secret" not in str(error.value)

harness/tests/test_corpus_pipeline.py

@@ -84,16 +84,64 @@ def item(name, fingerprint):
     )
 
 
-def pipeline(tmp_path, source, *, embedder=None, vectors=None, model="model-a", policy=None):
+def pipeline(tmp_path, source, *, embedder=None, vectors=None, model="model-a", policy=None,
+             retain=3):
     return CorpusPipeline(
         store=CorpusStore(tmp_path / "corpus"), sources=[source],
         embedder=embedder or Embedder(), vector_store=vectors or Vectors(),
         embedding_model=model, embedding_dimensions=3,
         chunk_policy=policy or ChunkPolicy(version="chunk-v1", max_chars=100),
         pipeline_version="evidence-v1",
+        retain_published_generations=retain,
     )
 
 
+def test_retention_bounds_generations_and_purges_vectors_after_publish(tmp_path):
+    vectors = Vectors()
+    generations = []
+    for index in range(4):
+        result = pipeline(
+            tmp_path, Source([(item("one", str(index)), f"version {index}")]),
+            vectors=vectors, retain=2,
+        ).run_as_job(
+            workspace_id="demo", workspace_root=tmp_path,
+            config_fingerprint="sha256:" + "1" * 64,
+            input_fingerprint="sha256:" + str(index) * 64,
+        )
+        generations.append(result.generation)
+    store = CorpusStore(tmp_path / "corpus")
+    assert store.list_generations() == generations[-2:]
+    assert {r.record.metadata["vector_generation"] for r in vectors.records} == set(generations[-2:])
+    assert store.active_generation() == generations[-1]
+
+
+def test_retention_keeps_filesystem_when_vector_purge_fails_then_retries(tmp_path):
+    class FailingDelete(Vectors):
+        def __init__(self):
+            super().__init__()
+            self.fail_delete = True
+
+        def delete_generation(self, collection, generation):
+            if self.fail_delete:
+                raise RuntimeError("credential secret")
+            return super().delete_generation(collection, generation)
+
+    vectors = FailingDelete()
+    for index in range(2):
+        pipeline(tmp_path, Source([(item("one", str(index)), str(index))]), vectors=vectors,
+                 retain=1).run_as_job(
+            workspace_id="demo", workspace_root=tmp_path,
+            config_fingerprint="sha256:" + "1" * 64,
+            input_fingerprint="sha256:" + str(index) * 64,
+        )
+    assert len(CorpusStore(tmp_path / "corpus").list_generations()) == 2
+    vectors.fail_delete = False
+    report = pipeline(tmp_path, Source([(item("one", "1"), "1")]), vectors=vectors,
+                      retain=1).gc(workspace_root=tmp_path)
+    assert report["status"] == "succeeded"
+    assert len(CorpusStore(tmp_path / "corpus").list_generations()) == 1
+
+
 def test_unchanged_documents_skip_acquire_normalize_chunk_and_embed(tmp_path):
     one = item("one", "a")
     first_source = Source([(one, "hello")])

harness/tests/test_corpus_publish.py

@@ -1,4 +1,5 @@
 import pytest
+import os
 
 from tht.corpus.models import CorpusManifest
 from tht.corpus.store import CorpusStore, UnsafeCorpusPath
@@ -56,3 +57,43 @@ def test_publish_restores_previous_active_when_directory_fsync_fails_after_repla
     with pytest.raises(OSError, match="post replace"):
         store.publish(second)
     assert store.active_generation() == first
+
+
+def test_read_document_rejects_symlink_hardlink_and_hash_mismatch(tmp_path):
+    from tht.corpus.models import CanonicalDocument
+
+    content = "trusted"
+    digest = "sha256:" + __import__("hashlib").sha256(content.encode()).hexdigest()
+    document = CanonicalDocument(
+        document_id="doc:" + "a" * 64, source_id="fs:one", source_uri="file:///one",
+        source_fingerprint="sha256:" + "b" * 64, content_hash=digest, content=content,
+        pipeline_version="evidence-v1",
+    )
+    store = CorpusStore(tmp_path / "corpus")
+    generation = store.stage(CorpusManifest(documents=(document,)), {document.document_id: content})
+    path = store.resolve_document(document.document_id, generation)
+    assert store.read_document(document.document_id, generation) == content
+
+    path.unlink()
+    path.symlink_to(tmp_path / "outside")
+    (tmp_path / "outside").write_text(content)
+    with pytest.raises(UnsafeCorpusPath):
+        store.read_document(document.document_id, generation)
+
+    path.unlink()
+    os.link(tmp_path / "outside", path)
+    with pytest.raises(UnsafeCorpusPath):
+        store.read_document(document.document_id, generation)
+
+    path.unlink()
+    path.write_text("tampered")
+    with pytest.raises(UnsafeCorpusPath):
+        store.read_document(document.document_id, generation)
+
+
+def test_generation_inventory_is_validated_and_sorted(tmp_path):
+    store = CorpusStore(tmp_path / "corpus")
+    first = store.stage(CorpusManifest(), {}, generation="gen:" + "1" * 32)
+    second = store.stage(CorpusManifest(), {}, generation="gen:" + "2" * 32)
+    (store.root / "unrelated").mkdir()
+    assert store.list_generations() == [first, second]

harness/tests/test_preprocess_cli.py

@@ -54,3 +54,17 @@ def test_preprocess_resume_rejects_generation_id_before_configuration(monkeypatc
         "status": "failed", "error": "resume requires a preprocessing run id"
     }
     assert called is False
+
+
+def test_preprocess_evidence_gc_json_is_pristine(monkeypatch, tmp_path):
+    import tht.cli.preprocess_cmd as command
+
+    monkeypatch.setattr(command, "gc_from_config", lambda *a, **k: {
+        "status": "succeeded", "dry_run": True, "evicted": [], "failures": [],
+    })
+    response = CliRunner().invoke(
+        app, ["preprocess", "evidence", "gc", "--dry-run", "--json", "-c",
+              str(tmp_path / "workspace.yaml")]
+    )
+    assert response.exit_code == 0, response.output
+    assert json.loads(response.output)["dry_run"] is True