feat(preprocess): resume evidence jobs by run id
This commit is contained in:
@@ -49,6 +49,13 @@ class Vectors:
|
||||
raise RuntimeError("partial write")
|
||||
return len(records)
|
||||
|
||||
def delete_generation(self, collection, generation):
|
||||
self.records = [
|
||||
value for value in self.records
|
||||
if value.record.metadata["vector_generation"] != generation
|
||||
]
|
||||
return 0
|
||||
|
||||
|
||||
def item(name, fingerprint):
|
||||
return SourceObject(
|
||||
@@ -130,3 +137,39 @@ def test_dry_run_and_failed_acquire_never_change_active(tmp_path):
|
||||
with pytest.raises(PipelineError):
|
||||
pipeline(tmp_path, Source([(changed, RuntimeError("boom"))])).run()
|
||||
assert CorpusStore(tmp_path / "corpus").active_generation() == active
|
||||
|
||||
|
||||
def test_job_pipeline_uses_ordered_plan_and_returns_run_id(tmp_path):
|
||||
one = item("one", "a")
|
||||
candidate = pipeline(tmp_path, Source([(one, "hello")]))
|
||||
result = candidate.run_as_job(
|
||||
workspace_id="demo", workspace_root=tmp_path,
|
||||
config_fingerprint="sha256:" + "1" * 64,
|
||||
input_fingerprint="sha256:" + "2" * 64,
|
||||
)
|
||||
assert result.status == "succeeded"
|
||||
assert result.run_id and len(result.run_id) == 32
|
||||
checkpoint = tmp_path / ".tht-jobs" / "evidence" / "runs" / result.run_id / "checkpoint.json"
|
||||
payload = __import__("json").loads(checkpoint.read_text())
|
||||
assert [stage["name"] for stage in payload["stages"]] == [
|
||||
"discover", "acquire_normalize_chunk", "embed", "vector_upsert",
|
||||
"stage_validate", "publish", "retention_cleanup",
|
||||
]
|
||||
|
||||
|
||||
def test_job_pipeline_dry_run_only_discovers_and_reports_changes(tmp_path):
|
||||
one = item("one", "a")
|
||||
source = Source([(one, "hello")])
|
||||
embedder = Embedder()
|
||||
vectors = Vectors()
|
||||
result = pipeline(tmp_path, source, embedder=embedder, vectors=vectors).run_as_job(
|
||||
workspace_id="demo", workspace_root=tmp_path,
|
||||
config_fingerprint="sha256:" + "1" * 64,
|
||||
input_fingerprint="sha256:" + "2" * 64,
|
||||
dry_run=True,
|
||||
)
|
||||
assert result.changed == ("fs:one",)
|
||||
assert source.acquire_calls == []
|
||||
assert embedder.calls == []
|
||||
assert vectors.records == []
|
||||
assert result.generation is None and result.published is False
|
||||
|
||||
@@ -64,6 +64,27 @@ def test_failed_stage_is_resumable_and_skips_completed_stage(tmp_path):
|
||||
assert calls == [("discover", False), ("acquire", False), ("recovered", False)]
|
||||
|
||||
|
||||
def test_resume_carries_successful_stage_artifacts_into_new_run(tmp_path):
|
||||
def discover(context):
|
||||
artifacts = context.run_dir / "artifacts"
|
||||
artifacts.mkdir()
|
||||
(artifacts / "discovery.json").write_text('{"source":"one"}')
|
||||
|
||||
first = run_job(
|
||||
_spec(tmp_path, stage_ids=("discover", "acquire")),
|
||||
[discover, lambda _context: (_ for _ in ()).throw(RuntimeError("crash"))],
|
||||
)
|
||||
|
||||
def acquire(context):
|
||||
assert (context.run_dir / "artifacts" / "discovery.json").read_text() == '{"source":"one"}'
|
||||
|
||||
resumed = run_job(
|
||||
_spec(tmp_path, resume_run_id=first.run_id, stage_ids=("discover", "acquire")),
|
||||
[discover, acquire],
|
||||
)
|
||||
assert resumed.status == "succeeded"
|
||||
|
||||
|
||||
def test_successful_job_is_idempotently_resumable(tmp_path):
|
||||
calls = []
|
||||
|
||||
|
||||
@@ -30,3 +30,27 @@ def test_preprocess_failure_is_structured_and_nonzero(monkeypatch, tmp_path):
|
||||
assert response.exit_code != 0
|
||||
assert json.loads(response.output) == {"status": "failed", "error": "preprocessing failed"}
|
||||
assert "secret detail" not in response.output
|
||||
|
||||
|
||||
def test_preprocess_resume_rejects_generation_id_before_configuration(monkeypatch, tmp_path):
|
||||
import tht.cli.preprocess_cmd as command
|
||||
|
||||
called = False
|
||||
|
||||
def forbidden(*args, **kwargs):
|
||||
nonlocal called
|
||||
called = True
|
||||
|
||||
monkeypatch.setattr(command, "run_from_config", forbidden)
|
||||
response = CliRunner().invoke(
|
||||
app,
|
||||
[
|
||||
"preprocess", "evidence", "--resume", "gen:" + "a" * 32,
|
||||
"--json", "-c", str(tmp_path / "workspace.yaml"),
|
||||
],
|
||||
)
|
||||
assert response.exit_code != 0
|
||||
assert json.loads(response.output) == {
|
||||
"status": "failed", "error": "resume requires a preprocessing run id"
|
||||
}
|
||||
assert called is False
|
||||
|
||||
Reference in New Issue
Block a user