feat(preprocess): resume evidence jobs by run id
This commit is contained in:
@@ -3,6 +3,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
import typer
|
||||
@@ -25,9 +27,6 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None =
|
||||
if cfg.embeddings is None:
|
||||
raise RuntimeError("embeddings are not configured")
|
||||
corpus_root = cfg.paths.artifacts.parent / "corpus"
|
||||
generation = None
|
||||
if resume:
|
||||
generation = resume if resume.startswith("gen:") else f"gen:{resume}"
|
||||
pipeline = CorpusPipeline(
|
||||
store=CorpusStore(corpus_root), sources=build_evidence_sources(cfg),
|
||||
embedder=make_embedder(cfg.embeddings),
|
||||
@@ -36,7 +35,17 @@ def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None =
|
||||
chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
|
||||
pipeline_version="evidence-v1",
|
||||
)
|
||||
return pipeline.run(dry_run=dry_run, resume=generation)
|
||||
def fingerprint(value: str) -> str:
|
||||
return "sha256:" + hashlib.sha256(value.encode()).hexdigest()
|
||||
|
||||
return pipeline.run_as_job(
|
||||
workspace_id=config.stem.lower().replace(".", "-").replace("_", "-"),
|
||||
workspace_root=corpus_root.parent,
|
||||
config_fingerprint=fingerprint(cfg.model_dump_json()),
|
||||
input_fingerprint=fingerprint(config.resolve().as_posix()),
|
||||
dry_run=dry_run,
|
||||
resume_run_id=resume,
|
||||
)
|
||||
|
||||
|
||||
@preprocess_app.command("evidence")
|
||||
@@ -46,6 +55,13 @@ def evidence_cmd(
|
||||
resume: str | None = typer.Option(None, "--resume"),
|
||||
json_output: bool = typer.Option(False, "--json"),
|
||||
) -> None:
|
||||
if resume is not None and re.fullmatch(r"[0-9a-f]{32}", resume) is None:
|
||||
payload = {"status": "failed", "error": "resume requires a preprocessing run id"}
|
||||
if json_output:
|
||||
typer.echo(json.dumps(payload, sort_keys=True))
|
||||
else:
|
||||
typer.secho("ERRORE: resume requires a preprocessing run id", fg=typer.colors.RED, err=True)
|
||||
raise typer.Exit(code=2)
|
||||
try:
|
||||
result = run_from_config(config, dry_run=dry_run, resume=resume)
|
||||
except Exception:
|
||||
@@ -60,6 +76,6 @@ def evidence_cmd(
|
||||
typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True))
|
||||
else:
|
||||
typer.echo(
|
||||
f"OK: generation={payload['generation']} changed={len(payload['changed'])} "
|
||||
f"OK: run={payload['run_id']} generation={payload['generation']} changed={len(payload['changed'])} "
|
||||
f"unchanged={len(payload['unchanged'])} removed={len(payload['removed'])}"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user