feat(preprocess): resume evidence jobs by run id

This commit is contained in:
2026-07-12 04:39:32 +02:00
parent 981b149249
commit 1d5f8c76a7
7 changed files with 413 additions and 5 deletions
+43
View File
@@ -49,6 +49,13 @@ class Vectors:
raise RuntimeError("partial write")
return len(records)
def delete_generation(self, collection, generation):
self.records = [
value for value in self.records
if value.record.metadata["vector_generation"] != generation
]
return 0
def item(name, fingerprint):
return SourceObject(
@@ -130,3 +137,39 @@ def test_dry_run_and_failed_acquire_never_change_active(tmp_path):
with pytest.raises(PipelineError):
pipeline(tmp_path, Source([(changed, RuntimeError("boom"))])).run()
assert CorpusStore(tmp_path / "corpus").active_generation() == active
def test_job_pipeline_uses_ordered_plan_and_returns_run_id(tmp_path):
one = item("one", "a")
candidate = pipeline(tmp_path, Source([(one, "hello")]))
result = candidate.run_as_job(
workspace_id="demo", workspace_root=tmp_path,
config_fingerprint="sha256:" + "1" * 64,
input_fingerprint="sha256:" + "2" * 64,
)
assert result.status == "succeeded"
assert result.run_id and len(result.run_id) == 32
checkpoint = tmp_path / ".tht-jobs" / "evidence" / "runs" / result.run_id / "checkpoint.json"
payload = __import__("json").loads(checkpoint.read_text())
assert [stage["name"] for stage in payload["stages"]] == [
"discover", "acquire_normalize_chunk", "embed", "vector_upsert",
"stage_validate", "publish", "retention_cleanup",
]
def test_job_pipeline_dry_run_only_discovers_and_reports_changes(tmp_path):
one = item("one", "a")
source = Source([(one, "hello")])
embedder = Embedder()
vectors = Vectors()
result = pipeline(tmp_path, source, embedder=embedder, vectors=vectors).run_as_job(
workspace_id="demo", workspace_root=tmp_path,
config_fingerprint="sha256:" + "1" * 64,
input_fingerprint="sha256:" + "2" * 64,
dry_run=True,
)
assert result.changed == ("fs:one",)
assert source.acquire_calls == []
assert embedder.calls == []
assert vectors.records == []
assert result.generation is None and result.published is False
+21
View File
@@ -64,6 +64,27 @@ def test_failed_stage_is_resumable_and_skips_completed_stage(tmp_path):
assert calls == [("discover", False), ("acquire", False), ("recovered", False)]
def test_resume_carries_successful_stage_artifacts_into_new_run(tmp_path):
def discover(context):
artifacts = context.run_dir / "artifacts"
artifacts.mkdir()
(artifacts / "discovery.json").write_text('{"source":"one"}')
first = run_job(
_spec(tmp_path, stage_ids=("discover", "acquire")),
[discover, lambda _context: (_ for _ in ()).throw(RuntimeError("crash"))],
)
def acquire(context):
assert (context.run_dir / "artifacts" / "discovery.json").read_text() == '{"source":"one"}'
resumed = run_job(
_spec(tmp_path, resume_run_id=first.run_id, stage_ids=("discover", "acquire")),
[discover, acquire],
)
assert resumed.status == "succeeded"
def test_successful_job_is_idempotently_resumable(tmp_path):
calls = []
+24
View File
@@ -30,3 +30,27 @@ def test_preprocess_failure_is_structured_and_nonzero(monkeypatch, tmp_path):
assert response.exit_code != 0
assert json.loads(response.output) == {"status": "failed", "error": "preprocessing failed"}
assert "secret detail" not in response.output
def test_preprocess_resume_rejects_generation_id_before_configuration(monkeypatch, tmp_path):
import tht.cli.preprocess_cmd as command
called = False
def forbidden(*args, **kwargs):
nonlocal called
called = True
monkeypatch.setattr(command, "run_from_config", forbidden)
response = CliRunner().invoke(
app,
[
"preprocess", "evidence", "--resume", "gen:" + "a" * 32,
"--json", "-c", str(tmp_path / "workspace.yaml"),
],
)
assert response.exit_code != 0
assert json.loads(response.output) == {
"status": "failed", "error": "resume requires a preprocessing run id"
}
assert called is False