fix(preprocess): harden crash recovery integrity

harness/tht/corpus/pipeline.py

@@ -16,7 +16,7 @@ from tht.ports.evidence import EvidenceSource, SourceObject
 from tht.ports.vector import VectorStore, VectorWriteRecord
 from tht.vectorstore.records import VectorRecord
 from tht.jobs.models import JobSpec
-from tht.jobs.runner import JobContext, run_job
+from tht.jobs.runner import JobContext, StageArtifacts, run_job, seal_stage_artifacts
 
 
 EVIDENCE_STAGE_IDS = (
@@ -96,6 +96,7 @@ class CorpusPipeline:
         input_fingerprint: str,
         dry_run: bool = False,
         resume_run_id: str | None = None,
+        after_stage_return=None,
     ) -> PipelineResult:
         """Execute preprocessing through the durable shared job envelope."""
         discovered = self._discover()
@@ -159,10 +160,11 @@ class CorpusPipeline:
                 "removed": removed,
                 "previous": previous.model_dump(mode="json") if previous else None,
             })
+            return StageArtifacts(("plan.json",))
 
         def acquire_stage(context: JobContext) -> None:
             if context.dry_run:
-                return
+                return StageArtifacts()
             plan = read(context, "plan.json")
             previous = CorpusManifest.model_validate(plan["previous"]) if plan["previous"] else None
             prior = {doc.source_id: doc for doc in previous.documents} if previous else {}
@@ -194,10 +196,11 @@ class CorpusPipeline:
                 },
             )
             write(context, "manifest.json", manifest.model_dump(mode="json"))
+            return StageArtifacts(("manifest.json",))
 
         def embed_stage(context: JobContext) -> None:
             if context.dry_run:
-                return
+                return StageArtifacts()
             plan = read(context, "plan.json")
             manifest = CorpusManifest.model_validate(read(context, "manifest.json"))
             changed_docs = {doc.document_id for doc in manifest.documents if doc.source_id in plan["changed"]}
@@ -208,6 +211,7 @@ class CorpusPipeline:
             ):
                 raise PipelineError("embedding output is incompatible")
             write(context, "embeddings.json", embeddings)
+            return StageArtifacts(("embeddings.json",))
 
         def records(context: JobContext):
             plan = read(context, "plan.json")
@@ -220,7 +224,8 @@ class CorpusPipeline:
 
         def compensate(context: JobContext) -> None:
             generation = read(context, "plan.json")["generation"]
-            self.store.discard(generation)
+            if self.store.active_generation() != generation:
+                self.store.discard(generation)
             try:
                 self.vector_store.delete_generation("evidence", generation)
             except Exception:
@@ -241,62 +246,101 @@ class CorpusPipeline:
             for document in manifest.documents:
                 if document.source_id in changed and generations.get(document.document_id) == old:
                     generations[document.document_id] = plan["generation"]
-            metadata = dict(manifest.metadata)
-            metadata["document_generations"] = generations
-            manifest = manifest.model_copy(update={
-                "vector_generation": plan["generation"], "metadata": metadata,
-            })
+            manifest_payload = manifest.model_dump(mode="json")
+            manifest_payload["metadata"]["document_generations"] = generations
+            manifest_payload["vector_generation"] = plan["generation"]
+            manifest = CorpusManifest.model_validate(manifest_payload)
             write(context, "manifest.json", manifest.model_dump(mode="json"))
             marker.unlink()
 
         def vector_stage(context: JobContext) -> None:
             if context.dry_run:
-                return
+                return StageArtifacts()
             rotate_compensated_generation(context)
             values = records(context)
+            write(context, "vector-intent.json", {
+                "generation": read(context, "plan.json")["generation"],
+                "records": {value.record.id: value.content_hash for value in values},
+            })
+            seal_stage_artifacts(
+                context, "vector_upsert",
+                ("plan.json", "manifest.json", "vector-intent.json"), spec,
+            )
             try:
-                if values and self.vector_store.upsert("evidence", values) != len(values):
+                existing = self.vector_store.existing_hashes("evidence", ["evidence"])
+                missing = [
+                    value for value in values
+                    if existing.get(value.record.id) != value.content_hash
+                ]
+                if missing and self.vector_store.upsert("evidence", missing) != len(missing):
                     raise PipelineError("vector write count mismatch")
             except Exception:
                 compensate(context)
                 raise
+            return StageArtifacts(("plan.json", "manifest.json", "vector-intent.json"))
 
         def stage_stage(context: JobContext) -> None:
             if context.dry_run:
-                return
+                return StageArtifacts()
             plan = read(context, "plan.json")
             manifest = CorpusManifest.model_validate(read(context, "manifest.json"))
+            recovered = False
             try:
-                self.store.stage(
-                    manifest, {doc.document_id: doc.content for doc in manifest.documents},
-                    generation=plan["generation"],
-                )
+                if artifact(context, "compensated.json").exists():
+                    recovered = True
+                    rotate_compensated_generation(context)
+                    values = records(context)
+                    existing = self.vector_store.existing_hashes("evidence", ["evidence"])
+                    missing = [value for value in values if existing.get(value.record.id) != value.content_hash]
+                    if missing and self.vector_store.upsert("evidence", missing) != len(missing):
+                        raise PipelineError("vector write count mismatch")
+                    plan = read(context, "plan.json")
+                    manifest = CorpusManifest.model_validate(read(context, "manifest.json"))
+                if not self.store.generation_path(plan["generation"]).exists():
+                    self.store.stage(
+                        manifest, {doc.document_id: doc.content for doc in manifest.documents},
+                        generation=plan["generation"],
+                    )
                 self.store.manifest(plan["generation"])
             except Exception:
                 compensate(context)
                 raise
+            return StageArtifacts(
+                ("plan.json", "manifest.json", "vector-intent.json") if recovered else ()
+            )
 
         def publish_stage(context: JobContext) -> None:
             if context.dry_run:
-                return
+                return StageArtifacts()
             if artifact(context, "compensated.json").exists():
                 rotate_compensated_generation(context)
                 values = records(context)
-                if values and self.vector_store.upsert("evidence", values) != len(values):
+                try:
+                    existing = self.vector_store.existing_hashes("evidence", ["evidence"])
+                    missing = [value for value in values if existing.get(value.record.id) != value.content_hash]
+                    if missing and self.vector_store.upsert("evidence", missing) != len(missing):
+                        raise PipelineError("vector write count mismatch")
+                except Exception:
                     compensate(context)
-                    raise PipelineError("vector write count mismatch")
+                    raise
                 manifest = CorpusManifest.model_validate(read(context, "manifest.json"))
                 generation = read(context, "plan.json")["generation"]
-                self.store.stage(
-                    manifest, {doc.document_id: doc.content for doc in manifest.documents},
-                    generation=generation,
-                )
+                try:
+                    if not self.store.generation_path(generation).exists():
+                        self.store.stage(
+                            manifest, {doc.document_id: doc.content for doc in manifest.documents},
+                            generation=generation,
+                        )
+                except Exception:
+                    compensate(context)
+                    raise
             generation = read(context, "plan.json")["generation"]
             try:
                 self.store.publish(generation)
             except Exception:
                 compensate(context)
                 raise
+            return StageArtifacts(("plan.json", "manifest.json", "vector-intent.json"))
 
         def retention_stage(context: JobContext) -> None:
             # Retention policy is intentionally a stable no-op until configured.
@@ -305,7 +349,7 @@ class CorpusPipeline:
         report = run_job(spec, [
             discover_stage, acquire_stage, embed_stage, vector_stage,
             stage_stage, publish_stage, retention_stage,
-        ])
+        ], after_stage_return=after_stage_return)
         run_dir = workspace_root / ".tht-jobs" / "evidence" / "runs" / report.run_id
         plan = json.loads((run_dir / "artifacts" / "plan.json").read_text())
         if dry_run:

harness/tht/corpus/store.py

@@ -46,6 +46,7 @@ class CorpusStore:
         self.root = Path(root)
         self.active_path = self.root / "ACTIVE"
         self._replace = os.replace
+        self._fsync_directory = self._sync_root
         self._ensure_root()
 
     def _ensure_root(self) -> None:
@@ -110,15 +111,35 @@ class CorpusStore:
         manifest = self.manifest(generation)
         if manifest.manifest_id != generation:
             raise UnsafeCorpusPath("manifest generation mismatch")
+        if self.active_generation() == generation:
+            return generation
+        previous = self.active_generation()
         temporary = self.active_path.with_name(f".ACTIVE.{uuid.uuid4().hex}.tmp")
-        _atomic_write(temporary, (generation + "\n").encode())
-        self._replace(temporary, self.active_path)
+        replaced = False
+        try:
+            _atomic_write(temporary, (generation + "\n").encode())
+            self._replace(temporary, self.active_path)
+            replaced = True
+            self._fsync_directory()
+        except BaseException:
+            temporary.unlink(missing_ok=True)
+            if replaced:
+                if previous is None:
+                    self.active_path.unlink(missing_ok=True)
+                else:
+                    rollback = self.active_path.with_name(f".ACTIVE.rollback.{uuid.uuid4().hex}.tmp")
+                    _atomic_write(rollback, (previous + "\n").encode())
+                    self._replace(rollback, self.active_path)
+                self._sync_root()
+            raise
+        return generation
+
+    def _sync_root(self) -> None:
         directory = os.open(self.root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
         try:
             os.fsync(directory)
         finally:
             os.close(directory)
-        return generation
 
     def active_generation(self) -> str | None:
         try:

harness/tht/jobs/runner.py

@@ -23,6 +23,11 @@ class CorruptCheckpointError(RuntimeError):
     pass
 
 
+@dataclass(frozen=True)
+class StageArtifacts:
+    required: tuple[str, ...] = ()
+
+
 @dataclass(frozen=True)
 class JobContext:
     run_id: str
@@ -35,6 +40,84 @@ class JobContext:
 Stage = Callable[[JobContext], Any]
 
 
+def _artifact_digest(path: Path) -> dict[str, Any]:
+    payload = path.read_bytes()
+    return {"sha256": hashlib.sha256(payload).hexdigest(), "size": len(payload)}
+
+
+def _seal_artifacts(context: JobContext, stage: str, result: Any, spec: JobSpec) -> None:
+    required = result.required if isinstance(result, StageArtifacts) else ()
+    root = context.run_dir / "artifacts"
+    root.mkdir(exist_ok=True)
+    manifest_path = root / "artifact-manifest.json"
+    manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {
+        "schema_version": 1,
+        "spec_fingerprint": _compatibility_fingerprint(spec, list(spec.stage_ids)),
+        "stages": {},
+    }
+    files = {}
+    for relative in required:
+        candidate = root / relative
+        if Path(relative).is_absolute() or ".." in Path(relative).parts or candidate.is_symlink():
+            raise CorruptCheckpointError("artifact path is unsafe")
+        if not candidate.is_file():
+            raise CorruptCheckpointError("required stage artifact is missing")
+        files[relative] = _artifact_digest(candidate)
+        for prior in manifest["stages"].values():
+            if relative in prior.get("required", []):
+                prior["required"].remove(relative)
+                prior["files"].pop(relative, None)
+    manifest["stages"][stage] = {"required": list(required), "files": files}
+    _atomic_write(manifest_path, json.dumps(manifest, sort_keys=True, separators=(",", ":")) + "\n")
+
+
+def seal_stage_artifacts(
+    context: JobContext, stage: str, required: tuple[str, ...], spec: JobSpec,
+) -> None:
+    """Durably record external-effect intent before a stage performs that effect."""
+    _seal_artifacts(context, stage, StageArtifacts(required), spec)
+
+
+def _validate_artifacts(run_dir: Path, spec: JobSpec, source: JobRun) -> set[str]:
+    root = run_dir / "artifacts"
+    manifest_path = root / "artifact-manifest.json"
+    successful = {stage.name for stage in source.stages if stage.status == "succeeded"}
+    if not successful and not manifest_path.exists():
+        return set()
+    try:
+        manifest = json.loads(manifest_path.read_text())
+        if manifest["spec_fingerprint"] != _compatibility_fingerprint(spec, list(spec.stage_ids)):
+            raise ValueError
+        sealed = set(manifest["stages"])
+        allowed = {"artifact-manifest.json"}
+        for stage, record in manifest["stages"].items():
+            for relative in record["required"]:
+                candidate = root / relative
+                if Path(relative).is_absolute() or ".." in Path(relative).parts or candidate.is_symlink():
+                    raise ValueError
+                if not candidate.is_file() or _artifact_digest(candidate) != record["files"][relative]:
+                    raise ValueError
+                allowed.add(relative)
+        if not successful.issubset(sealed):
+            raise ValueError
+        entries = list(root.iterdir())
+        if any(path.is_symlink() or not path.is_file() for path in entries):
+            raise ValueError
+        actual = {path.name for path in entries}
+        incomplete = next((stage for stage in source.stages if stage.status != "succeeded"), None)
+        marker = root / "compensated.json"
+        if incomplete is not None and incomplete.status in {"failed", "running"} and marker.is_file():
+            payload = json.loads(marker.read_text())
+            if not isinstance(payload.get("generation"), str):
+                raise ValueError
+            allowed.add("compensated.json")
+        if actual != allowed:
+            raise ValueError
+        return sealed
+    except (OSError, KeyError, TypeError, ValueError, json.JSONDecodeError) as error:
+        raise CorruptCheckpointError("resume artifact manifest is invalid") from error
+
+
 def _atomic_write(path: Path, payload: str) -> None:
     temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
     fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
@@ -91,7 +174,10 @@ def _new_run(spec: JobSpec, run_id: str, stages: Sequence[Stage]) -> JobRun:
     )
 
 
-def _resume_run(spec: JobSpec, run_id: str, stages: Sequence[Stage], source: JobRun) -> JobRun:
+def _resume_run(
+    spec: JobSpec, run_id: str, stages: Sequence[Stage], source: JobRun,
+    effect_completed: set[str] | None = None,
+) -> JobRun:
     requested_names = list(spec.stage_ids)
     if len(requested_names) != len(stages) or len(requested_names) != len(set(requested_names)):
         raise CorruptCheckpointError("resume checkpoint is incompatible with requested stages")
@@ -107,11 +193,15 @@ def _resume_run(spec: JobSpec, run_id: str, stages: Sequence[Stage], source: Job
     resumed_stages = []
     for name in requested_names:
         previous = source_by_name.get(name)
-        resumed_stages.append(
-            previous
-            if previous is not None and previous.status == "succeeded"
-            else StageRun(name=name)
-        )
+        if previous is not None and previous.status == "succeeded":
+            resumed_stages.append(previous)
+        elif previous is not None and previous.status == "running" and name in (effect_completed or set()):
+            resumed_stages.append(StageRun(
+                name=name, status="succeeded", started_at=previous.started_at or utc_now(),
+                finished_at=utc_now(),
+            ))
+        else:
+            resumed_stages.append(StageRun(name=name))
     return JobRun(
         run_id=run_id,
         compatibility_fingerprint=source.compatibility_fingerprint,
@@ -129,7 +219,10 @@ def _resume_run(spec: JobSpec, run_id: str, stages: Sequence[Stage], source: Job
     )
 
 
-def run_job(spec: JobSpec, stages: Sequence[Stage]) -> JobReport:
+def run_job(
+    spec: JobSpec, stages: Sequence[Stage], *,
+    after_stage_return: Callable[[JobContext, str], Any] | None = None,
+) -> JobReport:
     """Run stages once, returning a terminal report instead of leaking stage exceptions."""
     with WorkspaceJobLock(spec.workspace_root, spec.workspace_id, spec.job_type):
         jobs_root = spec.workspace_root / ".tht-jobs" / spec.job_type / "runs"
@@ -147,6 +240,7 @@ def run_job(spec: JobSpec, stages: Sequence[Stage]) -> JobReport:
                     source_path = matches[0]
             source = _load_checkpoint(source_path)
             _validate_resume_source(spec, stages, source)
+            effect_completed = _validate_artifacts(source_path.parent, spec, source)
 
         run_id = uuid.uuid4().hex
         run_dir = jobs_root / run_id
@@ -155,7 +249,7 @@ def run_job(spec: JobSpec, stages: Sequence[Stage]) -> JobReport:
         if source is None:
             run = _new_run(spec, run_id, stages)
         else:
-            run = _resume_run(spec, run_id, stages, source)
+            run = _resume_run(spec, run_id, stages, source, effect_completed)
             source_artifacts = jobs_root / source.run_id / "artifacts"
             if source_artifacts.exists():
                 shutil.copytree(source_artifacts, run_dir / "artifacts")
@@ -173,7 +267,7 @@ def run_job(spec: JobSpec, stages: Sequence[Stage]) -> JobReport:
             )
             _persist(checkpoint_path, run)
             try:
-                stage_callable(context)
+                stage_result = stage_callable(context)
             except Exception:
                 failed = stage.model_copy(
                     update={
@@ -191,6 +285,9 @@ def run_job(spec: JobSpec, stages: Sequence[Stage]) -> JobReport:
                 )
                 _persist(checkpoint_path, run)
                 break
+            _seal_artifacts(context, stage.name, stage_result, spec)
+            if after_stage_return is not None:
+                after_stage_return(context, stage.name)
             succeeded = stage.model_copy(update={"status": "succeeded", "finished_at": utc_now()})
             run = run.model_copy(
                 update={"stages": run.stages[:index] + (succeeded,) + run.stages[index + 1 :]}