fix(jobs): validate resume checkpoints before allocation

harness/tht/jobs/models.py

@@ -7,7 +7,7 @@ from datetime import UTC, datetime
 from pathlib import Path
 from typing import Literal, Self
 
-from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator
+from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator, model_validator
 
 
 _JOB_KEY = re.compile(r"^[a-z][a-z0-9_-]{0,63}$")
@@ -113,6 +113,26 @@ class StageRun(_FrozenModel):
 
     _name_key = field_validator("name")(_validate_job_key)
 
+    @model_validator(mode="after")
+    def state_shape(self) -> "StageRun":
+        if self.status == "pending" and any(
+            value is not None for value in (self.started_at, self.finished_at, self.error)
+        ):
+            raise ValueError("pending stage cannot contain timestamps or error")
+        if self.status == "running" and (
+            self.started_at is None or self.finished_at is not None or self.error is not None
+        ):
+            raise ValueError("running stage requires only started_at")
+        if self.status == "succeeded" and (
+            self.started_at is None or self.finished_at is None or self.error is not None
+        ):
+            raise ValueError("succeeded stage requires timestamps and no error")
+        if self.status == "failed" and (
+            self.started_at is None or self.finished_at is None or self.error is None
+        ):
+            raise ValueError("failed stage requires timestamps and safe error")
+        return self
+
 
 class JobRun(_FrozenModel):
     """Durable checkpoint, persisted after every state transition."""
@@ -144,6 +164,32 @@ class JobRun(_FrozenModel):
     _persisted_versions = field_validator("spec_version", "pipeline_version")(_validate_job_key)
     _resumed_from = field_validator("resumed_from")(_validate_run_id)
 
+    @model_validator(mode="after")
+    def ledger_shape(self) -> "JobRun":
+        names = [stage.name for stage in self.stages]
+        if len(names) != len(set(names)):
+            raise ValueError("stage identifiers must be unique")
+        statuses = [stage.status for stage in self.stages]
+        first_incomplete = next(
+            (index for index, status in enumerate(statuses) if status != "succeeded"),
+            len(statuses),
+        )
+        if any(status != "pending" for status in statuses[first_incomplete + 1 :]):
+            raise ValueError("stage ledger must be an ordered execution prefix")
+        if self.status == "succeeded" and (
+            self.finished_at is None or any(status != "succeeded" for status in statuses)
+        ):
+            raise ValueError("succeeded job requires a complete succeeded ledger")
+        if self.status == "failed" and (
+            self.finished_at is None
+            or first_incomplete == len(statuses)
+            or statuses[first_incomplete] != "failed"
+        ):
+            raise ValueError("failed job requires the first incomplete stage to be failed")
+        if self.status == "running" and self.finished_at is not None:
+            raise ValueError("running job cannot have finished_at")
+        return self
+
 
 class JobReport(JobRun):
     """Public machine-readable terminal report (contains no paths or stage outputs)."""