fix(preprocess): harden crash recovery integrity

harness/tests/test_corpus_pipeline.py

@@ -49,6 +49,11 @@ class Vectors:
             raise RuntimeError("partial write")
         return len(records)
 
+    def existing_hashes(self, collection, kinds):
+        return {
+            value.record.id: value.content_hash for value in self.records
+        }
+
     def delete_generation(self, collection, generation):
         self.records = [
             value for value in self.records
@@ -173,3 +178,134 @@ def test_job_pipeline_dry_run_only_discovers_and_reports_changes(tmp_path):
     assert embedder.calls == []
     assert vectors.records == []
     assert result.generation is None and result.published is False
+
+
+@pytest.mark.parametrize("crash_stage", [
+    "discover", "acquire_normalize_chunk", "embed", "vector_upsert",
+    "stage_validate", "publish", "retention_cleanup",
+])
+def test_job_pipeline_crash_after_each_stage_resumes_without_duplicate_effects(tmp_path, crash_stage):
+    one = item("one", "a")
+    source = Source([(one, "hello")])
+    embedder = Embedder()
+    vectors = Vectors()
+    candidate = pipeline(tmp_path, source, embedder=embedder, vectors=vectors)
+
+    class Crash(BaseException):
+        pass
+
+    def fault(_context, stage):
+        if stage == crash_stage:
+            raise Crash()
+
+    with pytest.raises(Crash):
+        candidate.run_as_job(
+            workspace_id="demo", workspace_root=tmp_path,
+            config_fingerprint="sha256:" + "1" * 64,
+            input_fingerprint="sha256:" + "2" * 64,
+            after_stage_return=fault,
+        )
+    runs = tmp_path / ".tht-jobs" / "evidence" / "runs"
+    crashed = next(runs.iterdir()).name
+    result = candidate.run_as_job(
+        workspace_id="demo", workspace_root=tmp_path,
+        config_fingerprint="sha256:" + "1" * 64,
+        input_fingerprint="sha256:" + "2" * 64,
+        resume_run_id=crashed,
+    )
+    assert result.status == "succeeded"
+    assert source.acquire_calls == ["fs:one"]
+    assert len(embedder.calls) == 1
+    assert len(vectors.records) == 1
+
+
+def test_job_pipeline_raw_upsert_failure_compensates_and_resumes_with_new_generation(tmp_path):
+    one = item("one", "a")
+    vectors = Vectors(fail=True)
+    candidate = pipeline(tmp_path, Source([(one, "hello")]), vectors=vectors)
+    first = candidate.run_as_job(
+        workspace_id="demo", workspace_root=tmp_path,
+        config_fingerprint="sha256:" + "1" * 64,
+        input_fingerprint="sha256:" + "2" * 64,
+    )
+    assert first.status == "failed"
+    assert vectors.records == []
+    old_generation = first.generation
+    vectors.fail = False
+    resumed = candidate.run_as_job(
+        workspace_id="demo", workspace_root=tmp_path,
+        config_fingerprint="sha256:" + "1" * 64,
+        input_fingerprint="sha256:" + "2" * 64,
+        resume_run_id=first.run_id,
+    )
+    assert resumed.status == "succeeded", resumed
+    assert resumed.generation != old_generation
+    assert candidate.store.active_generation() == resumed.generation
+
+
+def test_job_pipeline_raw_stage_failure_compensates_vectors_and_resumes(tmp_path, monkeypatch):
+    one = item("one", "a")
+    vectors = Vectors()
+    candidate = pipeline(tmp_path, Source([(one, "hello")]), vectors=vectors)
+    real_stage = candidate.store.stage
+    calls = 0
+
+    def fail_once(*args, **kwargs):
+        nonlocal calls
+        calls += 1
+        if calls == 1:
+            raise OSError("raw stage failure")
+        return real_stage(*args, **kwargs)
+
+    monkeypatch.setattr(candidate.store, "stage", fail_once)
+    first = candidate.run_as_job(
+        workspace_id="demo", workspace_root=tmp_path,
+        config_fingerprint="sha256:" + "1" * 64,
+        input_fingerprint="sha256:" + "2" * 64,
+    )
+    assert first.status == "failed" and vectors.records == []
+    resumed = candidate.run_as_job(
+        workspace_id="demo", workspace_root=tmp_path,
+        config_fingerprint="sha256:" + "1" * 64,
+        input_fingerprint="sha256:" + "2" * 64,
+        resume_run_id=first.run_id,
+    )
+    assert resumed.status == "succeeded", resumed
+
+
+@pytest.mark.parametrize("stage,filename", [
+    ("discover", "plan.json"),
+    ("acquire_normalize_chunk", "manifest.json"),
+    ("embed", "embeddings.json"),
+])
+@pytest.mark.parametrize("mutation", ["missing", "tampered"])
+def test_job_pipeline_rejects_corrupt_required_artifacts_before_resume(
+    tmp_path, stage, filename, mutation,
+):
+    one = item("one", "a")
+    candidate = pipeline(tmp_path, Source([(one, "hello")]))
+
+    class Crash(BaseException):
+        pass
+
+    with pytest.raises(Crash):
+        candidate.run_as_job(
+            workspace_id="demo", workspace_root=tmp_path,
+            config_fingerprint="sha256:" + "1" * 64,
+            input_fingerprint="sha256:" + "2" * 64,
+            after_stage_return=lambda _context, name: (
+                (_ for _ in ()).throw(Crash()) if name == stage else None
+            ),
+        )
+    runs = tmp_path / ".tht-jobs" / "evidence" / "runs"
+    crashed = next(runs.iterdir())
+    target = crashed / "artifacts" / filename
+    target.unlink() if mutation == "missing" else target.write_text("tampered")
+    from tht.jobs.runner import CorruptCheckpointError
+    with pytest.raises(CorruptCheckpointError, match="artifact"):
+        candidate.run_as_job(
+            workspace_id="demo", workspace_root=tmp_path,
+            config_fingerprint="sha256:" + "1" * 64,
+            input_fingerprint="sha256:" + "2" * 64,
+            resume_run_id=crashed.name,
+        )