feat(preprocess): resume evidence jobs by run id
This commit is contained in:
@@ -49,6 +49,13 @@ class Vectors:
|
||||
raise RuntimeError("partial write")
|
||||
return len(records)
|
||||
|
||||
def delete_generation(self, collection, generation):
|
||||
self.records = [
|
||||
value for value in self.records
|
||||
if value.record.metadata["vector_generation"] != generation
|
||||
]
|
||||
return 0
|
||||
|
||||
|
||||
def item(name, fingerprint):
|
||||
return SourceObject(
|
||||
@@ -130,3 +137,39 @@ def test_dry_run_and_failed_acquire_never_change_active(tmp_path):
|
||||
with pytest.raises(PipelineError):
|
||||
pipeline(tmp_path, Source([(changed, RuntimeError("boom"))])).run()
|
||||
assert CorpusStore(tmp_path / "corpus").active_generation() == active
|
||||
|
||||
|
||||
def test_job_pipeline_uses_ordered_plan_and_returns_run_id(tmp_path):
|
||||
one = item("one", "a")
|
||||
candidate = pipeline(tmp_path, Source([(one, "hello")]))
|
||||
result = candidate.run_as_job(
|
||||
workspace_id="demo", workspace_root=tmp_path,
|
||||
config_fingerprint="sha256:" + "1" * 64,
|
||||
input_fingerprint="sha256:" + "2" * 64,
|
||||
)
|
||||
assert result.status == "succeeded"
|
||||
assert result.run_id and len(result.run_id) == 32
|
||||
checkpoint = tmp_path / ".tht-jobs" / "evidence" / "runs" / result.run_id / "checkpoint.json"
|
||||
payload = __import__("json").loads(checkpoint.read_text())
|
||||
assert [stage["name"] for stage in payload["stages"]] == [
|
||||
"discover", "acquire_normalize_chunk", "embed", "vector_upsert",
|
||||
"stage_validate", "publish", "retention_cleanup",
|
||||
]
|
||||
|
||||
|
||||
def test_job_pipeline_dry_run_only_discovers_and_reports_changes(tmp_path):
|
||||
one = item("one", "a")
|
||||
source = Source([(one, "hello")])
|
||||
embedder = Embedder()
|
||||
vectors = Vectors()
|
||||
result = pipeline(tmp_path, source, embedder=embedder, vectors=vectors).run_as_job(
|
||||
workspace_id="demo", workspace_root=tmp_path,
|
||||
config_fingerprint="sha256:" + "1" * 64,
|
||||
input_fingerprint="sha256:" + "2" * 64,
|
||||
dry_run=True,
|
||||
)
|
||||
assert result.changed == ("fs:one",)
|
||||
assert source.acquire_calls == []
|
||||
assert embedder.calls == []
|
||||
assert vectors.records == []
|
||||
assert result.generation is None and result.published is False
|
||||
|
||||
Reference in New Issue
Block a user