fix(jobs): validate resume checkpoints before allocation
This commit is contained in:
@@ -233,3 +233,59 @@ def test_run_directories_are_private_and_fsynced_before_atomic_replace(tmp_path,
|
||||
first_replace = events.index("replace")
|
||||
assert "fsync" in events[:first_replace]
|
||||
assert "fsync" in events[first_replace + 1 :]
|
||||
|
||||
|
||||
def _tamper_checkpoint(tmp_path, report, transform):
|
||||
path = tmp_path / ".tht-jobs" / report.job_type / "runs" / report.run_id / "checkpoint.json"
|
||||
payload = json.loads(path.read_text())
|
||||
transform(payload)
|
||||
path.write_text(json.dumps(payload))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"transform",
|
||||
[
|
||||
lambda payload: payload["stages"].pop(),
|
||||
lambda payload: payload["stages"].reverse(),
|
||||
lambda payload: payload["stages"].append(dict(payload["stages"][0])),
|
||||
lambda payload: payload["stages"][0].update(name="substitute"),
|
||||
lambda payload: payload.update(input_fingerprint="sha256:" + "f" * 64),
|
||||
lambda payload: payload["stages"][0].update(status="pending"),
|
||||
],
|
||||
)
|
||||
def test_semantically_tampered_checkpoint_fails_without_orphan_run(tmp_path, transform):
|
||||
def one(_context):
|
||||
pass
|
||||
|
||||
def two(_context):
|
||||
pass
|
||||
|
||||
spec = _spec(tmp_path, stage_ids=("one", "two"))
|
||||
first = run_job(spec, [one, two])
|
||||
runs = tmp_path / ".tht-jobs" / "evidence" / "runs"
|
||||
before = {path.name for path in runs.iterdir()}
|
||||
_tamper_checkpoint(tmp_path, first, transform)
|
||||
called = False
|
||||
|
||||
def forbidden(_context):
|
||||
nonlocal called
|
||||
called = True
|
||||
|
||||
with pytest.raises(CorruptCheckpointError, match="invalid|incompatible"):
|
||||
run_job(spec.with_resume(first.run_id), [forbidden, forbidden])
|
||||
assert called is False
|
||||
assert {path.name for path in runs.iterdir()} == before
|
||||
|
||||
|
||||
def test_stored_compatibility_fingerprint_tamper_fails_without_orphan(tmp_path):
|
||||
first = run_job(_spec(tmp_path), [lambda _context: None])
|
||||
runs = tmp_path / ".tht-jobs" / "evidence" / "runs"
|
||||
before = {path.name for path in runs.iterdir()}
|
||||
_tamper_checkpoint(
|
||||
tmp_path,
|
||||
first,
|
||||
lambda payload: payload.update(compatibility_fingerprint="sha256:" + "0" * 64),
|
||||
)
|
||||
with pytest.raises(CorruptCheckpointError, match="invalid"):
|
||||
run_job(_spec(tmp_path, resume_run_id=first.run_id), [lambda _context: None])
|
||||
assert {path.name for path in runs.iterdir()} == before
|
||||
|
||||
Reference in New Issue
Block a user