fix(preprocess): publish DWH artifacts atomically

harness/tests/test_dwh_preprocess_job.py

@@ -1,5 +1,6 @@
 import hashlib
 import json
+from pathlib import Path
 
 from typer.testing import CliRunner
 
@@ -22,8 +23,8 @@ def test_selected_dwh_stages_run_in_declared_order(tmp_path):
         workspace_root=tmp_path,
         config_fingerprint=FP,
         input_fingerprint=FP,
-        introspect=lambda: calls.append("introspect"),
-        build_lsh=lambda: calls.append("lsh"),
+        introspect=lambda output: (calls.append("introspect"), output.write_text("catalog")),
+        build_lsh=lambda physical, output: _write_lsh(calls, physical, output),
     )
 
     report = pipeline.run(("introspect", "lsh"))
@@ -31,6 +32,19 @@ def test_selected_dwh_stages_run_in_declared_order(tmp_path):
     assert report.status == "succeeded"
     assert calls == ["introspect", "lsh"]
     assert [stage.name for stage in report.stages] == ["introspect", "lsh"]
+    active = (tmp_path / ".tht-dwh" / "ACTIVE").read_text().strip()
+    published = tmp_path / ".tht-dwh" / "generations" / active
+    assert (published / "physical.yaml").read_text() == "catalog"
+    assert sorted(path.name for path in published.iterdir()) == [
+        "demo_lsh.pkl", "demo_meta.json", "demo_minhashes.pkl", "physical.yaml"
+    ]
+
+
+def _write_lsh(calls, physical: Path, output: Path):
+    calls.append("lsh")
+    assert physical.read_text() == "catalog"
+    for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json"):
+        (output / name).write_text(name)
 
 
 def test_preprocess_dwh_json_is_pristine(monkeypatch, tmp_path):
@@ -81,3 +95,47 @@ def test_preprocess_dwh_rejects_unknown_or_duplicate_steps(monkeypatch, tmp_path
         assert response.exit_code == 2
         assert json.loads(response.output)["status"] == "failed"
     assert called is False
+
+
+def test_failed_multi_file_build_never_replaces_active_generation(tmp_path):
+    def catalog(output):
+        output.write_text("old-catalog")
+
+    first = DwhPreprocessPipeline(
+        workspace_id="demo", workspace_root=tmp_path,
+        config_fingerprint=FP, input_fingerprint=FP,
+        introspect=catalog,
+        build_lsh=lambda physical, output: [
+            (output / name).write_text(name)
+            for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json")
+        ],
+    ).run(("introspect", "lsh"))
+    assert first.status == "succeeded"
+    old_active = (tmp_path / ".tht-dwh" / "ACTIVE").read_text()
+
+    def partial_lsh(physical, output):
+        (output / "demo_lsh.pkl").write_text("new-but-partial")
+        raise RuntimeError("crash between LSH files")
+
+    failed = DwhPreprocessPipeline(
+        workspace_id="demo", workspace_root=tmp_path,
+        config_fingerprint=FP, input_fingerprint=FP,
+        introspect=lambda output: output.write_text("new-catalog"),
+        build_lsh=partial_lsh,
+    ).run(("introspect", "lsh"))
+
+    assert failed.status == "failed"
+    assert (tmp_path / ".tht-dwh" / "ACTIVE").read_text() == old_active
+
+    resumed = DwhPreprocessPipeline(
+        workspace_id="demo", workspace_root=tmp_path,
+        config_fingerprint=FP, input_fingerprint=FP,
+        introspect=lambda output: (_ for _ in ()).throw(
+            AssertionError("completed introspection must not repeat")
+        ),
+        build_lsh=lambda physical, output: [
+            (output / name).write_text("recovered")
+            for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json")
+        ],
+    ).run(("introspect", "lsh"), resume_run_id=failed.run_id)
+    assert resumed.status == "succeeded"

harness/tests/test_lsh_job_resume.py

@@ -1,5 +1,7 @@
 import hashlib
 
+import pytest
+
 from tht.jobs.dwh_pipeline import DwhPreprocessPipeline
 
 
@@ -9,10 +11,11 @@ FP = "sha256:" + hashlib.sha256(b"test").hexdigest()
 def test_lsh_failure_resumes_exact_run_without_repeating_introspection(tmp_path):
     calls = []
 
-    def introspect():
+    def introspect(output):
         calls.append("introspect")
+        output.write_text("catalog")
 
-    def fail_lsh():
+    def fail_lsh(physical, output):
         calls.append("lsh-failed")
         raise RuntimeError("database detail that must not leak")
 
@@ -32,7 +35,7 @@ def test_lsh_failure_resumes_exact_run_without_repeating_introspection(tmp_path)
         config_fingerprint=FP,
         input_fingerprint=FP,
         introspect=introspect,
-        build_lsh=lambda: calls.append("lsh-recovered"),
+        build_lsh=lambda physical, output: _recover_lsh(calls, output),
     ).run(("introspect", "lsh"), resume_run_id=failed.run_id)
 
     assert resumed.status == "succeeded"
@@ -40,14 +43,20 @@ def test_lsh_failure_resumes_exact_run_without_repeating_introspection(tmp_path)
     assert calls == ["introspect", "lsh-failed", "lsh-recovered"]
 
 
+def _recover_lsh(calls, output):
+    calls.append("lsh-recovered")
+    for name in ("demo_lsh.pkl", "demo_minhashes.pkl", "demo_meta.json"):
+        (output / name).write_text(name)
+
+
 def test_resume_rejects_a_different_stage_selection(tmp_path):
     failed = DwhPreprocessPipeline(
         workspace_id="demo",
         workspace_root=tmp_path,
         config_fingerprint=FP,
         input_fingerprint=FP,
-        introspect=lambda: None,
-        build_lsh=lambda: (_ for _ in ()).throw(RuntimeError()),
+        introspect=lambda output: output.write_text("catalog"),
+        build_lsh=lambda physical, output: (_ for _ in ()).throw(RuntimeError()),
     ).run(("introspect", "lsh"))
 
     pipeline = DwhPreprocessPipeline(
@@ -55,8 +64,8 @@ def test_resume_rejects_a_different_stage_selection(tmp_path):
         workspace_root=tmp_path,
         config_fingerprint=FP,
         input_fingerprint=FP,
-        introspect=lambda: None,
-        build_lsh=lambda: None,
+        introspect=lambda output: output.write_text("catalog"),
+        build_lsh=lambda physical, output: None,
     )
     try:
         pipeline.run(("lsh",), resume_run_id=failed.run_id)
@@ -64,3 +73,71 @@ def test_resume_rejects_a_different_stage_selection(tmp_path):
         assert "incompatible" in str(error)
     else:
         raise AssertionError("resume with different stages must fail")
+
+
+def test_resume_rejects_tampered_succeeded_stage_artifact(tmp_path):
+    failed = DwhPreprocessPipeline(
+        workspace_id="demo", workspace_root=tmp_path,
+        config_fingerprint=FP, input_fingerprint=FP,
+        introspect=lambda output: output.write_text("catalog"),
+        build_lsh=lambda physical, output: (_ for _ in ()).throw(RuntimeError()),
+    ).run(("introspect", "lsh"))
+    artifact = (
+        tmp_path / ".tht-jobs" / "dwh" / "runs" / failed.run_id
+        / "artifacts" / "physical.yaml"
+    )
+    artifact.write_text("tampered")
+
+    pipeline = DwhPreprocessPipeline(
+        workspace_id="demo", workspace_root=tmp_path,
+        config_fingerprint=FP, input_fingerprint=FP,
+        introspect=lambda output: output.write_text("catalog"),
+        build_lsh=lambda physical, output: _recover_lsh([], output),
+    )
+    with pytest.raises(Exception, match="artifact manifest is invalid"):
+        pipeline.run(("introspect", "lsh"), resume_run_id=failed.run_id)
+
+
+def test_post_publish_crash_reconciles_same_generation_on_resume(tmp_path):
+    crashed = False
+
+    def crash_once(_generation):
+        nonlocal crashed
+        if not crashed:
+            crashed = True
+            raise KeyboardInterrupt("simulated process death")
+
+    pipeline = DwhPreprocessPipeline(
+        workspace_id="demo", workspace_root=tmp_path,
+        config_fingerprint=FP, input_fingerprint=FP,
+        introspect=lambda output: output.write_text("catalog"),
+        build_lsh=lambda physical, output: _recover_lsh([], output),
+        after_publish=crash_once,
+    )
+    with pytest.raises(KeyboardInterrupt):
+        pipeline.run(("introspect", "lsh"))
+    active = (tmp_path / ".tht-dwh" / "ACTIVE").read_text().strip()
+    checkpoint = next((tmp_path / ".tht-jobs" / "dwh" / "runs").glob("*/checkpoint.json"))
+    source_run_id = checkpoint.parent.name
+    assert active == source_run_id
+
+    resumed = pipeline.run(("introspect", "lsh"), resume_run_id=source_run_id)
+    assert resumed.status == "succeeded"
+    assert (tmp_path / ".tht-dwh" / "ACTIVE").read_text().strip() == resumed.run_id
+
+
+def test_resume_of_succeeded_run_detects_tampered_published_file(tmp_path):
+    pipeline = DwhPreprocessPipeline(
+        workspace_id="demo", workspace_root=tmp_path,
+        config_fingerprint=FP, input_fingerprint=FP,
+        introspect=lambda output: output.write_text("catalog"),
+        build_lsh=lambda physical, output: _recover_lsh([], output),
+    )
+    succeeded = pipeline.run(("introspect", "lsh"))
+    published = (
+        tmp_path / ".tht-dwh" / "generations" / succeeded.run_id / "demo_meta.json"
+    )
+    published.write_text("tampered")
+
+    with pytest.raises(Exception, match="digest mismatch"):
+        pipeline.run(("introspect", "lsh"), resume_run_id=succeeded.run_id)