fix(preprocess): harden crash recovery integrity

This commit is contained in:
2026-07-12 04:49:15 +02:00
parent 1d5f8c76a7
commit c964920f16
7 changed files with 440 additions and 37 deletions
+61 -1
View File
@@ -4,7 +4,7 @@ import pytest
from pydantic import ValidationError
from tht.jobs.models import JobSpec
from tht.jobs.runner import CorruptCheckpointError, run_job
from tht.jobs.runner import CorruptCheckpointError, StageArtifacts, run_job
import tht.jobs.runner as runner_module
@@ -69,6 +69,7 @@ def test_resume_carries_successful_stage_artifacts_into_new_run(tmp_path):
artifacts = context.run_dir / "artifacts"
artifacts.mkdir()
(artifacts / "discovery.json").write_text('{"source":"one"}')
return StageArtifacts(("discovery.json",))
first = run_job(
_spec(tmp_path, stage_ids=("discover", "acquire")),
@@ -85,6 +86,65 @@ def test_resume_carries_successful_stage_artifacts_into_new_run(tmp_path):
assert resumed.status == "succeeded"
def test_crash_after_stage_effect_resumes_without_repeating_stage(tmp_path):
calls = []
def stage(context):
calls.append("stage")
artifacts = context.run_dir / "artifacts"
artifacts.mkdir()
(artifacts / "effect.json").write_text("ok")
return StageArtifacts(("effect.json",))
class Crash(BaseException):
pass
with pytest.raises(Crash):
run_job(
_spec(tmp_path), [stage],
after_stage_return=lambda *_: (_ for _ in ()).throw(Crash()),
)
runs = tmp_path / ".tht-jobs" / "evidence" / "runs"
crashed_run = next(runs.iterdir()).name
resumed = run_job(_spec(tmp_path).with_resume(crashed_run), [stage])
assert resumed.status == "succeeded"
assert calls == ["stage"]
def test_resume_rejects_tampered_successful_stage_artifact(tmp_path):
def stage(context):
artifacts = context.run_dir / "artifacts"
artifacts.mkdir()
(artifacts / "effect.json").write_text("ok")
return StageArtifacts(("effect.json",))
report = run_job(_spec(tmp_path), [stage])
path = tmp_path / ".tht-jobs" / "evidence" / "runs" / report.run_id / "artifacts" / "effect.json"
path.write_text("tampered")
with pytest.raises(CorruptCheckpointError, match="artifact"):
run_job(_spec(tmp_path).with_resume(report.run_id), [stage])
def test_resume_rejects_extra_symlink_before_any_stage(tmp_path):
report = run_job(_spec(tmp_path), [lambda _context: StageArtifacts()])
artifacts = tmp_path / ".tht-jobs" / "evidence" / "runs" / report.run_id / "artifacts"
(artifacts / "unsafe").symlink_to(tmp_path)
called = False
def forbidden(_context):
nonlocal called
called = True
with pytest.raises(CorruptCheckpointError, match="artifact"):
run_job(_spec(tmp_path).with_resume(report.run_id), [forbidden])
assert called is False
def test_nonexistent_well_formed_resume_run_id_is_rejected(tmp_path):
with pytest.raises(CorruptCheckpointError, match="checkpoint"):
run_job(_spec(tmp_path).with_resume("a" * 32), [lambda _context: None])
def test_successful_job_is_idempotently_resumable(tmp_path):
calls = []