fix(jobs): validate resume checkpoints before allocation

harness/tht/jobs/runner.py

@@ -94,8 +94,11 @@ def _resume_run(spec: JobSpec, run_id: str, stages: Sequence[Stage], source: Job
     requested_names = list(spec.stage_ids)
     if len(requested_names) != len(stages) or len(requested_names) != len(set(requested_names)):
         raise CorruptCheckpointError("resume checkpoint is incompatible with requested stages")
+    source_fingerprint = _source_compatibility_fingerprint(source)
+    if source.compatibility_fingerprint != source_fingerprint:
+        raise CorruptCheckpointError("resume checkpoint compatibility fingerprint is invalid")
     expected = _compatibility_fingerprint(spec, requested_names)
-    if source.compatibility_fingerprint != expected:
+    if source_fingerprint != expected or [stage.name for stage in source.stages] != requested_names:
         raise CorruptCheckpointError(
             "resume checkpoint is incompatible; start an intentional new run without resume"
         )
@@ -129,12 +132,8 @@ def run_job(spec: JobSpec, stages: Sequence[Stage]) -> JobReport:
     """Run stages once, returning a terminal report instead of leaking stage exceptions."""
     with WorkspaceJobLock(spec.workspace_root, spec.workspace_id, spec.job_type):
         jobs_root = spec.workspace_root / ".tht-jobs" / spec.job_type / "runs"
-        run_id = uuid.uuid4().hex
-        run_dir = jobs_root / run_id
-        _prepare_run_directory(spec.workspace_root, spec.job_type, run_id)
-        checkpoint_path = run_dir / "checkpoint.json"
         if spec.resume_run_id is None:
-            run = _new_run(spec, run_id, stages)
+            source = None
         else:
             source_path = jobs_root / spec.resume_run_id / "checkpoint.json"
             if not source_path.exists():
@@ -146,6 +145,15 @@ def run_job(spec: JobSpec, stages: Sequence[Stage]) -> JobReport:
                 if len(matches) == 1:
                     source_path = matches[0]
             source = _load_checkpoint(source_path)
+            _validate_resume_source(spec, stages, source)
+
+        run_id = uuid.uuid4().hex
+        run_dir = jobs_root / run_id
+        _prepare_run_directory(spec.workspace_root, spec.job_type, run_id)
+        checkpoint_path = run_dir / "checkpoint.json"
+        if source is None:
+            run = _new_run(spec, run_id, stages)
+        else:
             run = _resume_run(spec, run_id, stages, source)
         _persist(checkpoint_path, run)
         context = JobContext(run_id, spec.job_type, spec.dry_run, spec.workspace_root, run_dir)
@@ -213,6 +221,26 @@ def _compatibility_fingerprint(spec: JobSpec, stage_ids: list[str]) -> str:
     return _value_fingerprint(canonical)
 
 
+def _source_compatibility_fingerprint(source: JobRun) -> str:
+    payload = {
+        "schema_version": source.schema_version,
+        "workspace": source.workspace_fingerprint,
+        "job_type": source.job_type,
+        "dry_run": source.dry_run,
+        "spec_version": source.spec_version,
+        "pipeline_version": source.pipeline_version,
+        "config_fingerprint": source.config_fingerprint,
+        "input_fingerprint": source.input_fingerprint,
+        "stage_ids": [stage.name for stage in source.stages],
+    }
+    canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"))
+    return _value_fingerprint(canonical)
+
+
+def _validate_resume_source(spec: JobSpec, stages: Sequence[Stage], source: JobRun) -> None:
+    _resume_run(spec, "0" * 32, stages, source)
+
+
 def _prepare_run_directory(workspace_root: Path, job_type: str, run_id: str) -> None:
     parent_fd = os.open(workspace_root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
     try: