feat(evidence): add safe retention and materialized reads

harness/tht/corpus/pipeline.py

@@ -61,7 +61,7 @@ class CorpusPipeline:
     def __init__(
         self, *, store: CorpusStore, sources: list[EvidenceSource], embedder,
         vector_store: VectorStore, embedding_model: str, embedding_dimensions: int,
-        chunk_policy: ChunkPolicy, pipeline_version: str,
+        chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3,
     ) -> None:
         self.store = store
         self.sources = sources
@@ -71,6 +71,50 @@ class CorpusPipeline:
         self.embedding_dimensions = embedding_dimensions
         self.chunk_policy = chunk_policy
         self.pipeline_version = pipeline_version
+        if isinstance(retain_published_generations, bool) or retain_published_generations < 1:
+            raise ValueError("retain_published_generations must be at least 1")
+        self.retain_published_generations = retain_published_generations
+
+    def _protected_generations(self, workspace_root: Path) -> set[str]:
+        protected = {value for value in (self.store.active_generation(),) if value}
+        runs = workspace_root / ".tht-jobs" / "evidence" / "runs"
+        for checkpoint in runs.glob("*/checkpoint.json") if runs.exists() else ():
+            try:
+                state = json.loads(checkpoint.read_text(encoding="utf-8"))
+                if state.get("status") not in {"running", "failed"}:
+                    continue
+                plan = checkpoint.parent / "artifacts" / "plan.json"
+                generation = json.loads(plan.read_text(encoding="utf-8")).get("generation")
+                if isinstance(generation, str):
+                    protected.add(generation)
+            except (OSError, ValueError):
+                continue
+        return protected
+
+    def gc(self, *, workspace_root: Path, dry_run: bool = False) -> dict:
+        generations = self.store.list_generations()
+        protected = self._protected_generations(workspace_root)
+        keep = set(generations[-self.retain_published_generations:]) | protected
+        evicted, failures = [], []
+        for generation in generations:
+            if generation in keep:
+                continue
+            if dry_run:
+                evicted.append(generation)
+                continue
+            try:
+                self.vector_store.delete_generation("evidence", generation)
+            except Exception:
+                failures.append({"generation": generation, "error": "vector cleanup failed"})
+                continue
+            try:
+                self.store.discard(generation)
+                evicted.append(generation)
+            except Exception:
+                failures.append({"generation": generation, "error": "filesystem cleanup failed"})
+        return {"status": "partial" if failures else "succeeded", "dry_run": dry_run,
+                "active_generation": self.store.active_generation(), "evicted": evicted,
+                "protected": sorted(protected), "failures": failures}
 
     def _discover(self) -> list[tuple[EvidenceSource, SourceObject]]:
         discovered = []
@@ -343,8 +387,8 @@ class CorpusPipeline:
             return StageArtifacts(("plan.json", "manifest.json", "vector-intent.json"))
 
         def retention_stage(context: JobContext) -> None:
-            # Retention policy is intentionally a stable no-op until configured.
-            return
+            if not context.dry_run:
+                self.gc(workspace_root=workspace_root)
 
         report = run_job(spec, [
             discover_stage, acquire_stage, embed_stage, vector_stage,
@@ -459,6 +503,7 @@ class CorpusPipeline:
                     generation=generation,
                 )
             self.store.publish(staged)
+            self.gc(workspace_root=self.store.root.parent)
         except PipelineError:
             self._compensate(generation, vector_written)
             raise

harness/tht/corpus/store.py

@@ -9,6 +9,7 @@ import re
 import stat
 import shutil
 import uuid
+import hashlib
 from pathlib import Path
 from contextlib import contextmanager
 
@@ -170,6 +171,14 @@ class CorpusStore:
         generation = self.active_generation()
         return self.manifest(generation) if generation else None
 
+    def list_generations(self) -> list[str]:
+        values = []
+        for entry in self.root.iterdir():
+            match = re.fullmatch(r"gen-([0-9a-f]{32})", entry.name)
+            if match and not entry.is_symlink() and stat.S_ISDIR(entry.lstat().st_mode):
+                values.append(f"gen:{match.group(1)}")
+        return sorted(values, key=lambda value: self.generation_path(value).stat().st_mtime_ns)
+
     def resolve_document(self, document_id: str, generation: str | None = None) -> Path | None:
         generation = generation or self.active_generation()
         if generation is None:
@@ -178,8 +187,37 @@ class CorpusStore:
         relative = manifest.metadata.get("files", {}).get(document_id)
         if not isinstance(relative, str):
             return None
-        base = self.generation_path(generation).resolve()
-        path = (base / relative).resolve()
-        if not path.is_relative_to(base) or path.is_symlink():
-            raise UnsafeCorpusPath("materialized document escapes its generation")
-        return path
+        parts = Path(relative).parts
+        if Path(relative).is_absolute() or parts[:1] != ("documents",) or len(parts) != 2:
+            raise UnsafeCorpusPath("materialized document path is unsafe")
+        return self.generation_path(generation) / relative
+
+    def read_document(self, document_id: str, generation: str | None = None) -> str | None:
+        generation = generation or self.active_generation()
+        if generation is None:
+            return None
+        manifest = self.manifest(generation)
+        path = self.resolve_document(document_id, generation)
+        document = next((item for item in manifest.documents if item.document_id == document_id), None)
+        if path is None or document is None:
+            return None
+        generation_fd = os.open(self.generation_path(generation), os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
+        documents_fd = fd = None
+        try:
+            documents_fd = os.open("documents", os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=generation_fd)
+            fd = os.open(path.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=documents_fd)
+            info = os.fstat(fd)
+            if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_nlink != 1:
+                raise UnsafeCorpusPath("materialized document is unsafe")
+            payload = os.read(fd, info.st_size + 1)
+            if len(payload) != info.st_size or "sha256:" + hashlib.sha256(payload).hexdigest() != document.content_hash:
+                raise UnsafeCorpusPath("materialized document hash mismatch")
+            return payload.decode("utf-8")
+        except (OSError, UnicodeError) as error:
+            raise UnsafeCorpusPath("materialized document read failed") from error
+        finally:
+            if fd is not None:
+                os.close(fd)
+            if documents_fd is not None:
+                os.close(documents_fd)
+            os.close(generation_fd)