fix(jobs): bind completed effects to checkpoints

This commit is contained in:
2026-07-12 04:54:00 +02:00
parent c964920f16
commit b6a52995ae
5 changed files with 174 additions and 7 deletions
+48
View File
@@ -62,6 +62,22 @@ class Vectors:
return 0
class InterruptingVectors(Vectors):
def __init__(self):
super().__init__()
self.batches = []
self.interrupt = True
def upsert(self, collection, records):
self.batches.append([value.record.id for value in records])
if self.interrupt:
self.interrupt = False
self.records.append(records[0])
raise KeyboardInterrupt("process interruption after partial write")
self.records.extend(records)
return len(records)
def item(name, fingerprint):
return SourceObject(
source_id=f"fs:{name}", uri=f"file:///safe/{name}.md", fingerprint=f"sha256:{fingerprint}"
@@ -309,3 +325,35 @@ def test_job_pipeline_rejects_corrupt_required_artifacts_before_resume(
input_fingerprint="sha256:" + "2" * 64,
resume_run_id=crashed.name,
)
def test_vector_intent_is_reconciled_after_process_interruption_without_duplicate_upsert(tmp_path):
one = item("one", "a")
vectors = InterruptingVectors()
candidate = pipeline(
tmp_path, Source([(one, "a" * 250)]), vectors=vectors,
policy=ChunkPolicy(version="chunk-v1", max_chars=100),
)
with pytest.raises(KeyboardInterrupt):
candidate.run_as_job(
workspace_id="demo", workspace_root=tmp_path,
config_fingerprint="sha256:" + "1" * 64,
input_fingerprint="sha256:" + "2" * 64,
)
runs = tmp_path / ".tht-jobs" / "evidence" / "runs"
interrupted = next(runs.iterdir())
checkpoint = __import__("json").loads((interrupted / "checkpoint.json").read_text())
vector_stage = checkpoint["stages"][3]
assert vector_stage["status"] == "running"
assert vector_stage["effect_state"] == "intent"
first_written = vectors.batches[0][0]
result = candidate.run_as_job(
workspace_id="demo", workspace_root=tmp_path,
config_fingerprint="sha256:" + "1" * 64,
input_fingerprint="sha256:" + "2" * 64,
resume_run_id=interrupted.name,
)
assert result.status == "succeeded" and result.published is True
assert first_written not in vectors.batches[1]
assert len(vectors.records) == 3
+28
View File
@@ -145,6 +145,34 @@ def test_nonexistent_well_formed_resume_run_id_is_rejected(tmp_path):
run_job(_spec(tmp_path).with_resume("a" * 32), [lambda _context: None])
@pytest.mark.parametrize("tamper", ["artifact_and_manifest", "spec", "producer"])
def test_resume_rejects_manifest_root_or_binding_tamper(tmp_path, tamper):
def stage(context):
artifacts = context.run_dir / "artifacts"
artifacts.mkdir()
(artifacts / "effect.json").write_text("ok")
return StageArtifacts(("effect.json",))
report = run_job(_spec(tmp_path), [stage])
artifacts = tmp_path / ".tht-jobs" / "evidence" / "runs" / report.run_id / "artifacts"
manifest_path = artifacts / "artifact-manifest.json"
manifest = json.loads(manifest_path.read_text())
if tamper == "artifact_and_manifest":
(artifacts / "effect.json").write_text("evil")
digest = __import__("hashlib").sha256(b"evil").hexdigest()
manifest["stages"]["stage"]["files"]["effect.json"] = {
"sha256": digest, "size": 4,
}
elif tamper == "spec":
manifest["spec_fingerprint"] = "sha256:" + "0" * 64
else:
manifest["stages"]["other"] = manifest["stages"].pop("stage")
manifest_path.write_text(json.dumps(manifest, sort_keys=True, separators=(",", ":")) + "\n")
with pytest.raises(CorruptCheckpointError, match="artifact"):
run_job(_spec(tmp_path).with_resume(report.run_id), [stage])
def test_successful_job_is_idempotently_resumable(tmp_path):
calls = []