fix(evidence): reconcile generations safely

harness/tht/corpus/pipeline.py

@@ -92,9 +92,16 @@ class CorpusPipeline:
         return protected
 
     def gc(self, *, workspace_root: Path, dry_run: bool = False) -> dict:
-        generations = self.store.list_generations()
+        published = self.store.published_generations()
+        list_vectors = getattr(self.vector_store, "list_evidence_generations", None)
+        vector_generations = set(list_vectors("evidence")) if list_vectors else set()
+        generations = sorted(set(published) | vector_generations)
         protected = self._protected_generations(workspace_root)
-        keep = set(generations[-self.retain_published_generations:]) | protected
+        active = self.store.active_generation()
+        rollback_count = self.retain_published_generations - 1
+        rollback = [generation for generation in published if generation != active]
+        keep = ({active} if active else set()) | set(rollback[-rollback_count:] if rollback_count else ())
+        keep |= protected
         evicted, failures = [], []
         for generation in generations:
             if generation in keep:
@@ -108,7 +115,8 @@ class CorpusPipeline:
                 failures.append({"generation": generation, "error": "vector cleanup failed"})
                 continue
             try:
-                self.store.discard(generation)
+                if generation in self.store.list_generations():
+                    self.store.discard(generation)
                 evicted.append(generation)
             except Exception:
                 failures.append({"generation": generation, "error": "filesystem cleanup failed"})
@@ -131,7 +139,11 @@ class CorpusPipeline:
         with self.store.writer_lock():
             return self._run(dry_run=dry_run, resume=resume)
 
-    def run_as_job(
+    def run_as_job(self, **kwargs) -> PipelineResult:
+        with self.store.writer_lock():
+            return self._run_as_job(**kwargs)
+
+    def _run_as_job(
         self,
         *,
         workspace_id: str,

harness/tht/corpus/store.py

@@ -10,6 +10,7 @@ import stat
 import shutil
 import uuid
 import hashlib
+from datetime import UTC, datetime
 from pathlib import Path
 from contextlib import contextmanager
 
@@ -115,6 +116,7 @@ class CorpusStore:
         if self.active_generation() == generation:
             return generation
         previous = self.active_generation()
+        published_marker = self.generation_path(generation) / "PUBLISHED"
         temporary = self.active_path.with_name(f".ACTIVE.{uuid.uuid4().hex}.tmp")
         replaced = False
         try:
@@ -122,6 +124,10 @@ class CorpusStore:
             self._replace(temporary, self.active_path)
             replaced = True
             self._fsync_directory()
+            _atomic_write(
+                published_marker,
+                (datetime.now(UTC).isoformat().replace("+00:00", "Z") + "\n").encode("ascii"),
+            )
         except BaseException:
             temporary.unlink(missing_ok=True)
             if replaced:
@@ -179,6 +185,25 @@ class CorpusStore:
                 values.append(f"gen:{match.group(1)}")
         return sorted(values, key=lambda value: self.generation_path(value).stat().st_mtime_ns)
 
+    def published_generations(self) -> list[str]:
+        active = self.active_generation()
+        published = []
+        for generation in self.list_generations():
+            path = self.generation_path(generation)
+            marker = path / "PUBLISHED"
+            if generation != active and not marker.is_file():
+                continue
+            try:
+                manifest = self.manifest(generation)
+                if manifest.manifest_id != generation:
+                    continue
+                timestamp = marker.read_text(encoding="ascii").strip() if marker.is_file() else ""
+                key = (timestamp or manifest.created_at.isoformat(), generation)
+                published.append((key, generation))
+            except (OSError, ValueError):
+                continue
+        return [generation for _, generation in sorted(published)]
+
     def resolve_document(self, document_id: str, generation: str | None = None) -> Path | None:
         generation = generation or self.active_generation()
         if generation is None:
@@ -221,3 +246,15 @@ class CorpusStore:
             if documents_fd is not None:
                 os.close(documents_fd)
             os.close(generation_fd)
+
+    def materialize_document(
+        self, document_id: str, destination: Path, generation: str | None = None,
+    ) -> Path | None:
+        content = self.read_document(document_id, generation)
+        if content is None:
+            return None
+        destination = Path(destination)
+        destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+        _atomic_write(destination, content.encode("utf-8"))
+        destination.chmod(0o400)
+        return destination