feat(jobs): add resumable preprocessing envelope

harness/tests/test_job_locking.py

@@ -0,0 +1,61 @@
+import multiprocessing
+from pathlib import Path
+
+import pytest
+
+from tht.jobs.locking import JobAlreadyRunningError, WorkspaceJobLock
+
+
+def _hold_lock(root: str, ready, release):
+    with WorkspaceJobLock(Path(root), "demo", "evidence"):
+        ready.set()
+        release.wait(10)
+
+
+def _crash_with_lock(root: str, ready):
+    lock = WorkspaceJobLock(Path(root), "demo", "evidence")
+    lock.acquire()
+    ready.set()
+    raise SystemExit(7)
+
+
+def test_same_workspace_and_job_are_exclusive_across_processes(tmp_path):
+    context = multiprocessing.get_context("spawn")
+    ready = context.Event()
+    release = context.Event()
+    process = context.Process(target=_hold_lock, args=(str(tmp_path), ready, release))
+    process.start()
+    assert ready.wait(10)
+    try:
+        with pytest.raises(JobAlreadyRunningError):
+            WorkspaceJobLock(tmp_path, "demo", "evidence").acquire()
+    finally:
+        release.set()
+        process.join(10)
+    assert process.exitcode == 0
+
+
+def test_evidence_and_dwh_jobs_have_distinct_locks(tmp_path):
+    with WorkspaceJobLock(tmp_path, "demo", "evidence"):
+        with WorkspaceJobLock(tmp_path, "demo", "dwh"):
+            pass
+
+
+def test_lock_keys_cannot_escape_lock_directory(tmp_path):
+    with pytest.raises(ValueError, match="filesystem-safe"):
+        WorkspaceJobLock(tmp_path, "demo", "../evidence")
+
+
+def test_lock_is_recoverable_after_process_crash_without_stale_deletion(tmp_path):
+    context = multiprocessing.get_context("spawn")
+    ready = context.Event()
+    process = context.Process(target=_crash_with_lock, args=(str(tmp_path), ready))
+    process.start()
+    assert ready.wait(10)
+    process.join(10)
+    assert process.exitcode == 7
+
+    lock_path = WorkspaceJobLock(tmp_path, "demo", "evidence").path
+    assert lock_path.exists()
+    with WorkspaceJobLock(tmp_path, "demo", "evidence"):
+        assert lock_path.exists()

harness/tests/test_job_runner.py

@@ -0,0 +1,127 @@
+import json
+import pytest
+from pydantic import ValidationError
+
+from tht.jobs.models import JobSpec
+from tht.jobs.runner import CorruptCheckpointError, run_job
+
+
+def _spec(tmp_path, **updates):
+    values = {
+        "workspace_id": "demo",
+        "job_type": "evidence",
+        "workspace_root": tmp_path,
+    }
+    values.update(updates)
+    return JobSpec(**values)
+
+
+def test_job_models_are_immutable(tmp_path):
+    spec = _spec(tmp_path)
+    with pytest.raises(ValidationError, match="Instance is frozen"):
+        spec.job_type = "dwh"
+
+    resumed = spec.with_resume("a" * 32)
+    assert resumed.workspace_root == tmp_path
+    assert resumed.resume_run_id == "a" * 32
+
+
+def test_failed_stage_is_resumable_and_skips_completed_stage(tmp_path):
+    calls = []
+
+    def discover(context):
+        calls.append(("discover", context.dry_run))
+
+    def acquire(_context):
+        calls.append(("acquire", False))
+        raise RuntimeError("source /customer/alice token=secret unavailable")
+
+    first = run_job(_spec(tmp_path), [discover, acquire])
+    assert first.status == "failed"
+    assert [stage.status for stage in first.stages] == ["succeeded", "failed"]
+    assert first.stages[1].error.model_dump() == {
+        "category": "RuntimeError",
+        "message": "stage execution failed",
+    }
+
+    def acquire(_context):
+        calls.append(("recovered", False))
+
+    second = run_job(
+        _spec(tmp_path, resume_run_id=first.run_id),
+        [discover, acquire],
+    )
+    assert second.resumed_from == first.run_id
+    assert second.status == "succeeded"
+    assert calls == [("discover", False), ("acquire", False), ("recovered", False)]
+
+
+def test_successful_job_is_idempotently_resumable(tmp_path):
+    calls = []
+
+    def normalize(_context):
+        calls.append("normalize")
+
+    first = run_job(_spec(tmp_path), [normalize])
+    second = run_job(_spec(tmp_path, resume_run_id=first.run_id), [normalize])
+    assert first.status == second.status == "succeeded"
+    assert second.resumed_from == first.run_id
+    assert calls == ["normalize"]
+
+
+def test_checkpoints_and_report_are_json_safe_and_do_not_disclose_workspace_path(tmp_path):
+    def publish(_context):
+        return {"ignored": "/customer/alice", "password": "secret"}
+
+    report = run_job(_spec(tmp_path), [publish])
+    run_dir = tmp_path / ".tht-jobs" / "evidence" / "runs" / report.run_id
+    checkpoint = json.loads((run_dir / "checkpoint.json").read_text())
+    payload = (run_dir / "report.json").read_text()
+    parsed = json.loads(payload)
+
+    assert checkpoint["status"] == "succeeded"
+    assert parsed["schema_version"] == 1
+    assert parsed["run_id"] == report.run_id
+    assert str(tmp_path) not in payload
+    assert "alice" not in payload
+    assert "secret" not in payload
+    assert parsed["started_at"].endswith("Z")
+    assert parsed["finished_at"].endswith("Z")
+
+
+def test_dry_run_is_exposed_to_stages_and_report(tmp_path):
+    observed = []
+
+    def plan(context):
+        observed.append(context.dry_run)
+
+    report = run_job(_spec(tmp_path, dry_run=True), [plan])
+    assert observed == [True]
+    assert report.dry_run is True
+    assert report.status == "succeeded"
+
+
+def test_corrupt_checkpoint_is_rejected_without_running_stages(tmp_path):
+    first = run_job(_spec(tmp_path), [lambda _context: None])
+    checkpoint = (
+        tmp_path / ".tht-jobs" / "evidence" / "runs" / first.run_id / "checkpoint.json"
+    )
+    checkpoint.write_text("{not-json")
+    called = False
+
+    def stage(_context):
+        nonlocal called
+        called = True
+
+    with pytest.raises(CorruptCheckpointError, match="checkpoint is invalid"):
+        run_job(_spec(tmp_path, resume_run_id=first.run_id), [stage])
+    assert called is False
+
+
+def test_stage_timestamps_are_aware_and_ordered(tmp_path):
+    report = run_job(_spec(tmp_path), [lambda _context: None])
+    stage = report.stages[0]
+    assert stage.started_at.tzinfo is not None
+    assert stage.finished_at.tzinfo is not None
+    assert stage.started_at <= stage.finished_at
+    assert report.started_at <= stage.started_at <= report.finished_at