fix(jobs): bind completed effects to checkpoints
This commit is contained in:
@@ -62,6 +62,22 @@ class Vectors:
|
||||
return 0
|
||||
|
||||
|
||||
class InterruptingVectors(Vectors):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.batches = []
|
||||
self.interrupt = True
|
||||
|
||||
def upsert(self, collection, records):
|
||||
self.batches.append([value.record.id for value in records])
|
||||
if self.interrupt:
|
||||
self.interrupt = False
|
||||
self.records.append(records[0])
|
||||
raise KeyboardInterrupt("process interruption after partial write")
|
||||
self.records.extend(records)
|
||||
return len(records)
|
||||
|
||||
|
||||
def item(name, fingerprint):
|
||||
return SourceObject(
|
||||
source_id=f"fs:{name}", uri=f"file:///safe/{name}.md", fingerprint=f"sha256:{fingerprint}"
|
||||
@@ -309,3 +325,35 @@ def test_job_pipeline_rejects_corrupt_required_artifacts_before_resume(
|
||||
input_fingerprint="sha256:" + "2" * 64,
|
||||
resume_run_id=crashed.name,
|
||||
)
|
||||
|
||||
|
||||
def test_vector_intent_is_reconciled_after_process_interruption_without_duplicate_upsert(tmp_path):
|
||||
one = item("one", "a")
|
||||
vectors = InterruptingVectors()
|
||||
candidate = pipeline(
|
||||
tmp_path, Source([(one, "a" * 250)]), vectors=vectors,
|
||||
policy=ChunkPolicy(version="chunk-v1", max_chars=100),
|
||||
)
|
||||
with pytest.raises(KeyboardInterrupt):
|
||||
candidate.run_as_job(
|
||||
workspace_id="demo", workspace_root=tmp_path,
|
||||
config_fingerprint="sha256:" + "1" * 64,
|
||||
input_fingerprint="sha256:" + "2" * 64,
|
||||
)
|
||||
runs = tmp_path / ".tht-jobs" / "evidence" / "runs"
|
||||
interrupted = next(runs.iterdir())
|
||||
checkpoint = __import__("json").loads((interrupted / "checkpoint.json").read_text())
|
||||
vector_stage = checkpoint["stages"][3]
|
||||
assert vector_stage["status"] == "running"
|
||||
assert vector_stage["effect_state"] == "intent"
|
||||
first_written = vectors.batches[0][0]
|
||||
|
||||
result = candidate.run_as_job(
|
||||
workspace_id="demo", workspace_root=tmp_path,
|
||||
config_fingerprint="sha256:" + "1" * 64,
|
||||
input_fingerprint="sha256:" + "2" * 64,
|
||||
resume_run_id=interrupted.name,
|
||||
)
|
||||
assert result.status == "succeeded" and result.published is True
|
||||
assert first_written not in vectors.batches[1]
|
||||
assert len(vectors.records) == 3
|
||||
|
||||
@@ -145,6 +145,34 @@ def test_nonexistent_well_formed_resume_run_id_is_rejected(tmp_path):
|
||||
run_job(_spec(tmp_path).with_resume("a" * 32), [lambda _context: None])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tamper", ["artifact_and_manifest", "spec", "producer"])
|
||||
def test_resume_rejects_manifest_root_or_binding_tamper(tmp_path, tamper):
|
||||
def stage(context):
|
||||
artifacts = context.run_dir / "artifacts"
|
||||
artifacts.mkdir()
|
||||
(artifacts / "effect.json").write_text("ok")
|
||||
return StageArtifacts(("effect.json",))
|
||||
|
||||
report = run_job(_spec(tmp_path), [stage])
|
||||
artifacts = tmp_path / ".tht-jobs" / "evidence" / "runs" / report.run_id / "artifacts"
|
||||
manifest_path = artifacts / "artifact-manifest.json"
|
||||
manifest = json.loads(manifest_path.read_text())
|
||||
if tamper == "artifact_and_manifest":
|
||||
(artifacts / "effect.json").write_text("evil")
|
||||
digest = __import__("hashlib").sha256(b"evil").hexdigest()
|
||||
manifest["stages"]["stage"]["files"]["effect.json"] = {
|
||||
"sha256": digest, "size": 4,
|
||||
}
|
||||
elif tamper == "spec":
|
||||
manifest["spec_fingerprint"] = "sha256:" + "0" * 64
|
||||
else:
|
||||
manifest["stages"]["other"] = manifest["stages"].pop("stage")
|
||||
manifest_path.write_text(json.dumps(manifest, sort_keys=True, separators=(",", ":")) + "\n")
|
||||
|
||||
with pytest.raises(CorruptCheckpointError, match="artifact"):
|
||||
run_job(_spec(tmp_path).with_resume(report.run_id), [stage])
|
||||
|
||||
|
||||
def test_successful_job_is_idempotently_resumable(tmp_path):
|
||||
calls = []
|
||||
|
||||
|
||||
Reference in New Issue
Block a user