"""Immutable, secret-free records for preprocessing execution.""" from __future__ import annotations import re from datetime import UTC, datetime from pathlib import Path from typing import Literal, Self from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator, model_validator _JOB_KEY = re.compile(r"^[a-z][a-z0-9_-]{0,63}$") _RUN_ID = re.compile(r"^[0-9a-f]{32}$") _FINGERPRINT = re.compile(r"^sha256:[0-9a-f]{64}$") JobStatus = Literal["pending", "running", "succeeded", "failed"] StageStatus = Literal["pending", "running", "succeeded", "failed"] EffectState = Literal["intent", "completed"] def utc_now() -> datetime: return datetime.now(UTC) def _validate_job_key(value: str) -> str: if not _JOB_KEY.fullmatch(value): raise ValueError("job identifiers must be lowercase filesystem-safe keys") return value def _validate_run_id(value: str | None) -> str | None: if value is not None and not _RUN_ID.fullmatch(value): raise ValueError("run id must contain 32 lowercase hexadecimal characters") return value class _FrozenModel(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid", validate_default=True) def model_copy(self, *, update=None, deep: bool = False) -> Self: data = self.model_dump(round_trip=True) if update: data.update(update) return type(self).model_validate(data) @field_serializer("*", when_used="json", check_fields=False) def serialize_utc(self, value): if isinstance(value, datetime): return value.astimezone(UTC).isoformat().replace("+00:00", "Z") return value class JobSpec(_FrozenModel): """Execution input. The local root is deliberately excluded from serialization.""" workspace_id: str job_type: str workspace_root: Path = Field(exclude=True) spec_version: str = Field(min_length=1, max_length=64) pipeline_version: str = Field(min_length=1, max_length=64) config_fingerprint: str input_fingerprint: str stage_ids: tuple[str, ...] dry_run: bool = False resume_run_id: str | None = None _workspace_key = field_validator("workspace_id")(_validate_job_key) _job_type_key = field_validator("job_type")(_validate_job_key) _version_keys = field_validator("spec_version", "pipeline_version")(_validate_job_key) _resume_id = field_validator("resume_run_id")(_validate_run_id) _config_fingerprint = field_validator("config_fingerprint")( lambda value: value if _FINGERPRINT.fullmatch(value) else _invalid_fingerprint() ) _input_fingerprint = field_validator("input_fingerprint")( lambda value: value if _FINGERPRINT.fullmatch(value) else _invalid_fingerprint() ) _stage_ids = field_validator("stage_ids")( lambda values: tuple(_validate_job_key(value) for value in values) ) def model_copy(self, *, update=None, deep: bool = False) -> Self: data = { "workspace_id": self.workspace_id, "job_type": self.job_type, "workspace_root": self.workspace_root, "spec_version": self.spec_version, "pipeline_version": self.pipeline_version, "config_fingerprint": self.config_fingerprint, "input_fingerprint": self.input_fingerprint, "stage_ids": self.stage_ids, "dry_run": self.dry_run, "resume_run_id": self.resume_run_id, } if update: data.update(update) return type(self).model_validate(data) def with_resume(self, run_id: str) -> "JobSpec": return self.model_copy(update={"resume_run_id": run_id}) class StageError(_FrozenModel): category: Literal["internal"] = "internal" code: Literal["stage_exception"] = "stage_exception" message: Literal["stage execution failed"] = "stage execution failed" class StageRun(_FrozenModel): name: str status: StageStatus = "pending" started_at: datetime | None = None finished_at: datetime | None = None error: StageError | None = None effect_state: EffectState | None = None artifact_manifest_digest: str | None = None artifact_files: tuple[str, ...] = () _name_key = field_validator("name")(_validate_job_key) _artifact_digest = field_validator("artifact_manifest_digest")( lambda value: value if value is None or _FINGERPRINT.fullmatch(value) else _invalid_fingerprint() ) @model_validator(mode="after") def state_shape(self) -> "StageRun": if self.status == "pending" and any( value is not None for value in ( self.started_at, self.finished_at, self.error, self.effect_state, self.artifact_manifest_digest, ) ): raise ValueError("pending stage cannot contain timestamps or error") if self.status == "running" and ( self.started_at is None or self.finished_at is not None or self.error is not None ): raise ValueError("running stage requires only started_at") if self.status == "succeeded" and ( self.started_at is None or self.finished_at is None or self.error is not None ): raise ValueError("succeeded stage requires timestamps and no error") if self.status == "failed" and ( self.started_at is None or self.finished_at is None or self.error is None ): raise ValueError("failed stage requires timestamps and safe error") if (self.effect_state is None) != (self.artifact_manifest_digest is None): raise ValueError("effect state and artifact manifest digest must be persisted together") if self.artifact_files and self.effect_state is None: raise ValueError("artifact files require a persisted effect state") return self class JobRun(_FrozenModel): """Durable checkpoint, persisted after every state transition.""" schema_version: Literal[1] = 1 run_id: str compatibility_fingerprint: str workspace_fingerprint: str job_type: str spec_version: str pipeline_version: str config_fingerprint: str input_fingerprint: str dry_run: bool status: JobStatus started_at: datetime finished_at: datetime | None = None resumed_from: str | None = None stages: tuple[StageRun, ...] = () _run_id = field_validator("run_id")(_validate_run_id) _compatibility = field_validator("compatibility_fingerprint", "workspace_fingerprint")( lambda value: value if _FINGERPRINT.fullmatch(value) else _invalid_fingerprint() ) _input_fingerprints = field_validator("config_fingerprint", "input_fingerprint")( lambda value: value if _FINGERPRINT.fullmatch(value) else _invalid_fingerprint() ) _job_type = field_validator("job_type")(_validate_job_key) _persisted_versions = field_validator("spec_version", "pipeline_version")(_validate_job_key) _resumed_from = field_validator("resumed_from")(_validate_run_id) @model_validator(mode="after") def ledger_shape(self) -> "JobRun": names = [stage.name for stage in self.stages] if len(names) != len(set(names)): raise ValueError("stage identifiers must be unique") statuses = [stage.status for stage in self.stages] first_incomplete = next( (index for index, status in enumerate(statuses) if status != "succeeded"), len(statuses), ) if any(status != "pending" for status in statuses[first_incomplete + 1 :]): raise ValueError("stage ledger must be an ordered execution prefix") if self.status == "succeeded" and ( self.finished_at is None or any(status != "succeeded" for status in statuses) ): raise ValueError("succeeded job requires a complete succeeded ledger") if self.status == "failed" and ( self.finished_at is None or first_incomplete == len(statuses) or statuses[first_incomplete] != "failed" ): raise ValueError("failed job requires the first incomplete stage to be failed") if self.status == "running" and self.finished_at is not None: raise ValueError("running job cannot have finished_at") return self class JobReport(JobRun): """Public machine-readable terminal report (contains no paths or stage outputs).""" def _invalid_fingerprint(): raise ValueError("fingerprint must be sha256 followed by 64 lowercase hexadecimal characters")