fix(jobs): harden resume locks and durability

harness/tht/jobs/models.py

@@ -12,6 +12,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_valid
 
 _JOB_KEY = re.compile(r"^[a-z][a-z0-9_-]{0,63}$")
 _RUN_ID = re.compile(r"^[0-9a-f]{32}$")
+_FINGERPRINT = re.compile(r"^sha256:[0-9a-f]{64}$")
 JobStatus = Literal["pending", "running", "succeeded", "failed"]
 StageStatus = Literal["pending", "running", "succeeded", "failed"]
 
@@ -54,18 +55,38 @@ class JobSpec(_FrozenModel):
     workspace_id: str
     job_type: str
     workspace_root: Path = Field(exclude=True)
+    spec_version: str = Field(min_length=1, max_length=64)
+    pipeline_version: str = Field(min_length=1, max_length=64)
+    config_fingerprint: str
+    input_fingerprint: str
+    stage_ids: tuple[str, ...]
     dry_run: bool = False
     resume_run_id: str | None = None
 
     _workspace_key = field_validator("workspace_id")(_validate_job_key)
     _job_type_key = field_validator("job_type")(_validate_job_key)
+    _version_keys = field_validator("spec_version", "pipeline_version")(_validate_job_key)
     _resume_id = field_validator("resume_run_id")(_validate_run_id)
+    _config_fingerprint = field_validator("config_fingerprint")(
+        lambda value: value if _FINGERPRINT.fullmatch(value) else _invalid_fingerprint()
+    )
+    _input_fingerprint = field_validator("input_fingerprint")(
+        lambda value: value if _FINGERPRINT.fullmatch(value) else _invalid_fingerprint()
+    )
+    _stage_ids = field_validator("stage_ids")(
+        lambda values: tuple(_validate_job_key(value) for value in values)
+    )
 
     def model_copy(self, *, update=None, deep: bool = False) -> Self:
         data = {
             "workspace_id": self.workspace_id,
             "job_type": self.job_type,
             "workspace_root": self.workspace_root,
+            "spec_version": self.spec_version,
+            "pipeline_version": self.pipeline_version,
+            "config_fingerprint": self.config_fingerprint,
+            "input_fingerprint": self.input_fingerprint,
+            "stage_ids": self.stage_ids,
             "dry_run": self.dry_run,
             "resume_run_id": self.resume_run_id,
         }
@@ -78,7 +99,8 @@ class JobSpec(_FrozenModel):
 
 
 class StageError(_FrozenModel):
-    category: str = Field(pattern=r"^[A-Za-z][A-Za-z0-9_]{0,127}$")
+    category: Literal["internal"] = "internal"
+    code: Literal["stage_exception"] = "stage_exception"
     message: Literal["stage execution failed"] = "stage execution failed"
 
 
@@ -97,7 +119,13 @@ class JobRun(_FrozenModel):
 
     schema_version: Literal[1] = 1
     run_id: str
+    compatibility_fingerprint: str
+    workspace_fingerprint: str
     job_type: str
+    spec_version: str
+    pipeline_version: str
+    config_fingerprint: str
+    input_fingerprint: str
     dry_run: bool
     status: JobStatus
     started_at: datetime
@@ -106,9 +134,20 @@ class JobRun(_FrozenModel):
     stages: tuple[StageRun, ...] = ()
 
     _run_id = field_validator("run_id")(_validate_run_id)
+    _compatibility = field_validator("compatibility_fingerprint", "workspace_fingerprint")(
+        lambda value: value if _FINGERPRINT.fullmatch(value) else _invalid_fingerprint()
+    )
+    _input_fingerprints = field_validator("config_fingerprint", "input_fingerprint")(
+        lambda value: value if _FINGERPRINT.fullmatch(value) else _invalid_fingerprint()
+    )
     _job_type = field_validator("job_type")(_validate_job_key)
+    _persisted_versions = field_validator("spec_version", "pipeline_version")(_validate_job_key)
     _resumed_from = field_validator("resumed_from")(_validate_run_id)
 
 
 class JobReport(JobRun):
     """Public machine-readable terminal report (contains no paths or stage outputs)."""
+
+
+def _invalid_fingerprint():
+    raise ValueError("fingerprint must be sha256 followed by 64 lowercase hexadecimal characters")