feat(jobs): add resumable preprocessing envelope
This commit is contained in:
@@ -0,0 +1,15 @@
|
||||
"""Shared execution envelope for resumable preprocessing jobs."""
|
||||
|
||||
from tht.jobs.locking import JobAlreadyRunningError, WorkspaceJobLock
|
||||
from tht.jobs.models import JobReport, JobRun, JobSpec
|
||||
from tht.jobs.runner import JobContext, run_job
|
||||
|
||||
__all__ = [
|
||||
"JobAlreadyRunningError",
|
||||
"JobContext",
|
||||
"JobReport",
|
||||
"JobRun",
|
||||
"JobSpec",
|
||||
"WorkspaceJobLock",
|
||||
"run_job",
|
||||
]
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Crash-safe interprocess locking scoped by workspace and job type."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fcntl
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from types import TracebackType
|
||||
|
||||
|
||||
class JobAlreadyRunningError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
_JOB_KEY = re.compile(r"^[a-z][a-z0-9_-]{0,63}$")
|
||||
|
||||
|
||||
def _lock_name(workspace_id: str, job_type: str) -> str:
|
||||
if not _JOB_KEY.fullmatch(workspace_id) or not _JOB_KEY.fullmatch(job_type):
|
||||
raise ValueError("lock identifiers must be lowercase filesystem-safe keys")
|
||||
workspace_key = hashlib.sha256(workspace_id.encode("utf-8")).hexdigest()[:16]
|
||||
return f"{workspace_key}-{job_type}.lock"
|
||||
|
||||
|
||||
class WorkspaceJobLock:
|
||||
"""Advisory kernel lock; the inode remains stable and is never deleted by PID."""
|
||||
|
||||
def __init__(self, workspace_root: Path, workspace_id: str, job_type: str) -> None:
|
||||
self.path = workspace_root / ".tht-jobs" / ".locks" / _lock_name(
|
||||
workspace_id, job_type
|
||||
)
|
||||
self._fd: int | None = None
|
||||
|
||||
def acquire(self) -> "WorkspaceJobLock":
|
||||
if self._fd is not None:
|
||||
raise RuntimeError("job lock is already held by this object")
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd = os.open(self.path, os.O_RDWR | os.O_CREAT, 0o600)
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError as error:
|
||||
os.close(fd)
|
||||
raise JobAlreadyRunningError("this workspace job is already running") from error
|
||||
self._fd = fd
|
||||
return self
|
||||
|
||||
def release(self) -> None:
|
||||
if self._fd is None:
|
||||
return
|
||||
fd, self._fd = self._fd, None
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_UN)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
def __enter__(self) -> "WorkspaceJobLock":
|
||||
return self.acquire()
|
||||
|
||||
def __exit__(
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc: BaseException | None,
|
||||
traceback: TracebackType | None,
|
||||
) -> None:
|
||||
self.release()
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Immutable, secret-free records for preprocessing execution."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Literal, Self
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator
|
||||
|
||||
|
||||
_JOB_KEY = re.compile(r"^[a-z][a-z0-9_-]{0,63}$")
|
||||
_RUN_ID = re.compile(r"^[0-9a-f]{32}$")
|
||||
JobStatus = Literal["pending", "running", "succeeded", "failed"]
|
||||
StageStatus = Literal["pending", "running", "succeeded", "failed"]
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _validate_job_key(value: str) -> str:
|
||||
if not _JOB_KEY.fullmatch(value):
|
||||
raise ValueError("job identifiers must be lowercase filesystem-safe keys")
|
||||
return value
|
||||
|
||||
|
||||
def _validate_run_id(value: str | None) -> str | None:
|
||||
if value is not None and not _RUN_ID.fullmatch(value):
|
||||
raise ValueError("run id must contain 32 lowercase hexadecimal characters")
|
||||
return value
|
||||
|
||||
|
||||
class _FrozenModel(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid", validate_default=True)
|
||||
|
||||
def model_copy(self, *, update=None, deep: bool = False) -> Self:
|
||||
data = self.model_dump(round_trip=True)
|
||||
if update:
|
||||
data.update(update)
|
||||
return type(self).model_validate(data)
|
||||
|
||||
@field_serializer("*", when_used="json", check_fields=False)
|
||||
def serialize_utc(self, value):
|
||||
if isinstance(value, datetime):
|
||||
return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
|
||||
return value
|
||||
|
||||
|
||||
class JobSpec(_FrozenModel):
|
||||
"""Execution input. The local root is deliberately excluded from serialization."""
|
||||
|
||||
workspace_id: str
|
||||
job_type: str
|
||||
workspace_root: Path = Field(exclude=True)
|
||||
dry_run: bool = False
|
||||
resume_run_id: str | None = None
|
||||
|
||||
_workspace_key = field_validator("workspace_id")(_validate_job_key)
|
||||
_job_type_key = field_validator("job_type")(_validate_job_key)
|
||||
_resume_id = field_validator("resume_run_id")(_validate_run_id)
|
||||
|
||||
def model_copy(self, *, update=None, deep: bool = False) -> Self:
|
||||
data = {
|
||||
"workspace_id": self.workspace_id,
|
||||
"job_type": self.job_type,
|
||||
"workspace_root": self.workspace_root,
|
||||
"dry_run": self.dry_run,
|
||||
"resume_run_id": self.resume_run_id,
|
||||
}
|
||||
if update:
|
||||
data.update(update)
|
||||
return type(self).model_validate(data)
|
||||
|
||||
def with_resume(self, run_id: str) -> "JobSpec":
|
||||
return self.model_copy(update={"resume_run_id": run_id})
|
||||
|
||||
|
||||
class StageError(_FrozenModel):
|
||||
category: str = Field(pattern=r"^[A-Za-z][A-Za-z0-9_]{0,127}$")
|
||||
message: Literal["stage execution failed"] = "stage execution failed"
|
||||
|
||||
|
||||
class StageRun(_FrozenModel):
|
||||
name: str
|
||||
status: StageStatus = "pending"
|
||||
started_at: datetime | None = None
|
||||
finished_at: datetime | None = None
|
||||
error: StageError | None = None
|
||||
|
||||
_name_key = field_validator("name")(_validate_job_key)
|
||||
|
||||
|
||||
class JobRun(_FrozenModel):
|
||||
"""Durable checkpoint, persisted after every state transition."""
|
||||
|
||||
schema_version: Literal[1] = 1
|
||||
run_id: str
|
||||
job_type: str
|
||||
dry_run: bool
|
||||
status: JobStatus
|
||||
started_at: datetime
|
||||
finished_at: datetime | None = None
|
||||
resumed_from: str | None = None
|
||||
stages: tuple[StageRun, ...] = ()
|
||||
|
||||
_run_id = field_validator("run_id")(_validate_run_id)
|
||||
_job_type = field_validator("job_type")(_validate_job_key)
|
||||
_resumed_from = field_validator("resumed_from")(_validate_run_id)
|
||||
|
||||
|
||||
class JobReport(JobRun):
|
||||
"""Public machine-readable terminal report (contains no paths or stage outputs)."""
|
||||
@@ -0,0 +1,171 @@
|
||||
"""Resumable stage runner with durable atomic checkpoints and reports."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from tht.jobs.locking import WorkspaceJobLock
|
||||
from tht.jobs.models import JobReport, JobRun, JobSpec, StageError, StageRun, utc_now
|
||||
|
||||
|
||||
class CorruptCheckpointError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class JobContext:
|
||||
run_id: str
|
||||
job_type: str
|
||||
dry_run: bool
|
||||
workspace_root: Path
|
||||
run_dir: Path
|
||||
|
||||
|
||||
Stage = Callable[[JobContext], Any]
|
||||
|
||||
|
||||
def _atomic_write(path: Path, payload: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
|
||||
fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as stream:
|
||||
stream.write(payload)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temporary, path)
|
||||
directory_fd = os.open(path.parent, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
os.close(directory_fd)
|
||||
except BaseException:
|
||||
try:
|
||||
temporary.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def _persist(path: Path, run: JobRun) -> None:
|
||||
_atomic_write(path, run.model_dump_json(indent=2) + "\n")
|
||||
|
||||
|
||||
def _load_checkpoint(path: Path) -> JobRun:
|
||||
try:
|
||||
return JobRun.model_validate_json(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValidationError, ValueError, json.JSONDecodeError) as error:
|
||||
raise CorruptCheckpointError("checkpoint is invalid and cannot be resumed") from error
|
||||
|
||||
|
||||
def _stage_name(stage: Stage) -> str:
|
||||
name = getattr(stage, "__name__", "")
|
||||
if name == "<lambda>":
|
||||
name = "stage"
|
||||
return name.replace("_", "-")
|
||||
|
||||
|
||||
def _new_run(spec: JobSpec, run_id: str, stages: Sequence[Stage]) -> JobRun:
|
||||
names = [_stage_name(stage) for stage in stages]
|
||||
if len(names) != len(set(names)):
|
||||
raise ValueError("stage names must be unique")
|
||||
return JobRun(
|
||||
run_id=run_id,
|
||||
job_type=spec.job_type,
|
||||
dry_run=spec.dry_run,
|
||||
status="running",
|
||||
started_at=utc_now(),
|
||||
resumed_from=spec.resume_run_id,
|
||||
stages=tuple(StageRun(name=name) for name in names),
|
||||
)
|
||||
|
||||
|
||||
def _resume_run(spec: JobSpec, run_id: str, stages: Sequence[Stage], source: JobRun) -> JobRun:
|
||||
if source.job_type != spec.job_type:
|
||||
raise CorruptCheckpointError("checkpoint job type does not match resume request")
|
||||
requested_names = [_stage_name(stage) for stage in stages]
|
||||
source_by_name = {stage.name: stage for stage in source.stages}
|
||||
resumed_stages = []
|
||||
for name in requested_names:
|
||||
previous = source_by_name.get(name)
|
||||
resumed_stages.append(
|
||||
previous
|
||||
if previous is not None and previous.status == "succeeded"
|
||||
else StageRun(name=name)
|
||||
)
|
||||
return JobRun(
|
||||
run_id=run_id,
|
||||
job_type=spec.job_type,
|
||||
dry_run=spec.dry_run,
|
||||
status="running",
|
||||
started_at=utc_now(),
|
||||
resumed_from=source.run_id,
|
||||
stages=tuple(resumed_stages),
|
||||
)
|
||||
|
||||
|
||||
def run_job(spec: JobSpec, stages: Sequence[Stage]) -> JobReport:
|
||||
"""Run stages once, returning a terminal report instead of leaking stage exceptions."""
|
||||
with WorkspaceJobLock(spec.workspace_root, spec.workspace_id, spec.job_type):
|
||||
jobs_root = spec.workspace_root / ".tht-jobs" / spec.job_type / "runs"
|
||||
run_id = uuid.uuid4().hex
|
||||
run_dir = jobs_root / run_id
|
||||
checkpoint_path = run_dir / "checkpoint.json"
|
||||
if spec.resume_run_id is None:
|
||||
run = _new_run(spec, run_id, stages)
|
||||
else:
|
||||
source_path = jobs_root / spec.resume_run_id / "checkpoint.json"
|
||||
source = _load_checkpoint(source_path)
|
||||
run = _resume_run(spec, run_id, stages, source)
|
||||
_persist(checkpoint_path, run)
|
||||
context = JobContext(run_id, spec.job_type, spec.dry_run, spec.workspace_root, run_dir)
|
||||
|
||||
for index, stage_callable in enumerate(stages):
|
||||
if run.stages[index].status == "succeeded":
|
||||
continue
|
||||
stage = run.stages[index].model_copy(
|
||||
update={"status": "running", "started_at": utc_now()}
|
||||
)
|
||||
run = run.model_copy(
|
||||
update={"stages": run.stages[:index] + (stage,) + run.stages[index + 1 :]}
|
||||
)
|
||||
_persist(checkpoint_path, run)
|
||||
try:
|
||||
stage_callable(context)
|
||||
except Exception as error:
|
||||
failed = stage.model_copy(
|
||||
update={
|
||||
"status": "failed",
|
||||
"finished_at": utc_now(),
|
||||
"error": StageError(category=type(error).__name__),
|
||||
}
|
||||
)
|
||||
run = run.model_copy(
|
||||
update={
|
||||
"status": "failed",
|
||||
"finished_at": utc_now(),
|
||||
"stages": run.stages[:index] + (failed,) + run.stages[index + 1 :],
|
||||
}
|
||||
)
|
||||
_persist(checkpoint_path, run)
|
||||
break
|
||||
succeeded = stage.model_copy(update={"status": "succeeded", "finished_at": utc_now()})
|
||||
run = run.model_copy(
|
||||
update={"stages": run.stages[:index] + (succeeded,) + run.stages[index + 1 :]}
|
||||
)
|
||||
_persist(checkpoint_path, run)
|
||||
else:
|
||||
run = run.model_copy(update={"status": "succeeded", "finished_at": utc_now()})
|
||||
_persist(checkpoint_path, run)
|
||||
|
||||
report = JobReport.model_validate(run.model_dump())
|
||||
_atomic_write(run_dir / "report.json", report.model_dump_json(indent=2) + "\n")
|
||||
return report
|
||||
Reference in New Issue
Block a user