fix(preprocess): harden crash recovery integrity
This commit is contained in:
@@ -4,7 +4,7 @@ import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from tht.jobs.models import JobSpec
|
||||
from tht.jobs.runner import CorruptCheckpointError, run_job
|
||||
from tht.jobs.runner import CorruptCheckpointError, StageArtifacts, run_job
|
||||
import tht.jobs.runner as runner_module
|
||||
|
||||
|
||||
@@ -69,6 +69,7 @@ def test_resume_carries_successful_stage_artifacts_into_new_run(tmp_path):
|
||||
artifacts = context.run_dir / "artifacts"
|
||||
artifacts.mkdir()
|
||||
(artifacts / "discovery.json").write_text('{"source":"one"}')
|
||||
return StageArtifacts(("discovery.json",))
|
||||
|
||||
first = run_job(
|
||||
_spec(tmp_path, stage_ids=("discover", "acquire")),
|
||||
@@ -85,6 +86,65 @@ def test_resume_carries_successful_stage_artifacts_into_new_run(tmp_path):
|
||||
assert resumed.status == "succeeded"
|
||||
|
||||
|
||||
def test_crash_after_stage_effect_resumes_without_repeating_stage(tmp_path):
|
||||
calls = []
|
||||
|
||||
def stage(context):
|
||||
calls.append("stage")
|
||||
artifacts = context.run_dir / "artifacts"
|
||||
artifacts.mkdir()
|
||||
(artifacts / "effect.json").write_text("ok")
|
||||
return StageArtifacts(("effect.json",))
|
||||
|
||||
class Crash(BaseException):
|
||||
pass
|
||||
|
||||
with pytest.raises(Crash):
|
||||
run_job(
|
||||
_spec(tmp_path), [stage],
|
||||
after_stage_return=lambda *_: (_ for _ in ()).throw(Crash()),
|
||||
)
|
||||
runs = tmp_path / ".tht-jobs" / "evidence" / "runs"
|
||||
crashed_run = next(runs.iterdir()).name
|
||||
resumed = run_job(_spec(tmp_path).with_resume(crashed_run), [stage])
|
||||
assert resumed.status == "succeeded"
|
||||
assert calls == ["stage"]
|
||||
|
||||
|
||||
def test_resume_rejects_tampered_successful_stage_artifact(tmp_path):
|
||||
def stage(context):
|
||||
artifacts = context.run_dir / "artifacts"
|
||||
artifacts.mkdir()
|
||||
(artifacts / "effect.json").write_text("ok")
|
||||
return StageArtifacts(("effect.json",))
|
||||
|
||||
report = run_job(_spec(tmp_path), [stage])
|
||||
path = tmp_path / ".tht-jobs" / "evidence" / "runs" / report.run_id / "artifacts" / "effect.json"
|
||||
path.write_text("tampered")
|
||||
with pytest.raises(CorruptCheckpointError, match="artifact"):
|
||||
run_job(_spec(tmp_path).with_resume(report.run_id), [stage])
|
||||
|
||||
|
||||
def test_resume_rejects_extra_symlink_before_any_stage(tmp_path):
|
||||
report = run_job(_spec(tmp_path), [lambda _context: StageArtifacts()])
|
||||
artifacts = tmp_path / ".tht-jobs" / "evidence" / "runs" / report.run_id / "artifacts"
|
||||
(artifacts / "unsafe").symlink_to(tmp_path)
|
||||
called = False
|
||||
|
||||
def forbidden(_context):
|
||||
nonlocal called
|
||||
called = True
|
||||
|
||||
with pytest.raises(CorruptCheckpointError, match="artifact"):
|
||||
run_job(_spec(tmp_path).with_resume(report.run_id), [forbidden])
|
||||
assert called is False
|
||||
|
||||
|
||||
def test_nonexistent_well_formed_resume_run_id_is_rejected(tmp_path):
|
||||
with pytest.raises(CorruptCheckpointError, match="checkpoint"):
|
||||
run_job(_spec(tmp_path).with_resume("a" * 32), [lambda _context: None])
|
||||
|
||||
|
||||
def test_successful_job_is_idempotently_resumable(tmp_path):
|
||||
calls = []
|
||||
|
||||
|
||||
Reference in New Issue
Block a user