feat(preprocess): resume evidence jobs by run id

This commit is contained in:
2026-07-12 04:39:32 +02:00
parent 981b149249
commit 1d5f8c76a7
7 changed files with 413 additions and 5 deletions
+43
View File
@@ -49,6 +49,13 @@ class Vectors:
raise RuntimeError("partial write")
return len(records)
def delete_generation(self, collection, generation):
self.records = [
value for value in self.records
if value.record.metadata["vector_generation"] != generation
]
return 0
def item(name, fingerprint):
return SourceObject(
@@ -130,3 +137,39 @@ def test_dry_run_and_failed_acquire_never_change_active(tmp_path):
with pytest.raises(PipelineError):
pipeline(tmp_path, Source([(changed, RuntimeError("boom"))])).run()
assert CorpusStore(tmp_path / "corpus").active_generation() == active
def test_job_pipeline_uses_ordered_plan_and_returns_run_id(tmp_path):
one = item("one", "a")
candidate = pipeline(tmp_path, Source([(one, "hello")]))
result = candidate.run_as_job(
workspace_id="demo", workspace_root=tmp_path,
config_fingerprint="sha256:" + "1" * 64,
input_fingerprint="sha256:" + "2" * 64,
)
assert result.status == "succeeded"
assert result.run_id and len(result.run_id) == 32
checkpoint = tmp_path / ".tht-jobs" / "evidence" / "runs" / result.run_id / "checkpoint.json"
payload = __import__("json").loads(checkpoint.read_text())
assert [stage["name"] for stage in payload["stages"]] == [
"discover", "acquire_normalize_chunk", "embed", "vector_upsert",
"stage_validate", "publish", "retention_cleanup",
]
def test_job_pipeline_dry_run_only_discovers_and_reports_changes(tmp_path):
one = item("one", "a")
source = Source([(one, "hello")])
embedder = Embedder()
vectors = Vectors()
result = pipeline(tmp_path, source, embedder=embedder, vectors=vectors).run_as_job(
workspace_id="demo", workspace_root=tmp_path,
config_fingerprint="sha256:" + "1" * 64,
input_fingerprint="sha256:" + "2" * 64,
dry_run=True,
)
assert result.changed == ("fs:one",)
assert source.acquire_calls == []
assert embedder.calls == []
assert vectors.records == []
assert result.generation is None and result.published is False