fix(jobs): validate resume checkpoints before allocation

This commit is contained in:
2026-07-12 04:17:34 +02:00
parent 9f069cdd5b
commit 16a8bd9df6
4 changed files with 157 additions and 7 deletions
+56
View File
@@ -233,3 +233,59 @@ def test_run_directories_are_private_and_fsynced_before_atomic_replace(tmp_path,
first_replace = events.index("replace")
assert "fsync" in events[:first_replace]
assert "fsync" in events[first_replace + 1 :]
def _tamper_checkpoint(tmp_path, report, transform):
path = tmp_path / ".tht-jobs" / report.job_type / "runs" / report.run_id / "checkpoint.json"
payload = json.loads(path.read_text())
transform(payload)
path.write_text(json.dumps(payload))
@pytest.mark.parametrize(
"transform",
[
lambda payload: payload["stages"].pop(),
lambda payload: payload["stages"].reverse(),
lambda payload: payload["stages"].append(dict(payload["stages"][0])),
lambda payload: payload["stages"][0].update(name="substitute"),
lambda payload: payload.update(input_fingerprint="sha256:" + "f" * 64),
lambda payload: payload["stages"][0].update(status="pending"),
],
)
def test_semantically_tampered_checkpoint_fails_without_orphan_run(tmp_path, transform):
def one(_context):
pass
def two(_context):
pass
spec = _spec(tmp_path, stage_ids=("one", "two"))
first = run_job(spec, [one, two])
runs = tmp_path / ".tht-jobs" / "evidence" / "runs"
before = {path.name for path in runs.iterdir()}
_tamper_checkpoint(tmp_path, first, transform)
called = False
def forbidden(_context):
nonlocal called
called = True
with pytest.raises(CorruptCheckpointError, match="invalid|incompatible"):
run_job(spec.with_resume(first.run_id), [forbidden, forbidden])
assert called is False
assert {path.name for path in runs.iterdir()} == before
def test_stored_compatibility_fingerprint_tamper_fails_without_orphan(tmp_path):
first = run_job(_spec(tmp_path), [lambda _context: None])
runs = tmp_path / ".tht-jobs" / "evidence" / "runs"
before = {path.name for path in runs.iterdir()}
_tamper_checkpoint(
tmp_path,
first,
lambda payload: payload.update(compatibility_fingerprint="sha256:" + "0" * 64),
)
with pytest.raises(CorruptCheckpointError, match="invalid"):
run_job(_spec(tmp_path, resume_run_id=first.run_id), [lambda _context: None])
assert {path.name for path in runs.iterdir()} == before