feat(preprocess): publish incremental Evidence corpus

2026-07-12 04:26:24 +02:00
parent 16a8bd9df6
commit b96d4f13b9
10 changed files with 708 additions and 0 deletions
@@ -0,0 +1,65 @@
+"""One-shot preprocessing commands."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import typer
+
+from tht.cli.config_cmd import CONFIG_OPT
+
+
+preprocess_app = typer.Typer(help="Materialize versioned preprocessing artifacts")
+
+
+def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None = None):
+    from tht.adapters.factory import build_evidence_sources, build_vector_store
+    from tht.cli.schema_cmd import _load_config_or_exit
+    from tht.cli.vector_cmd import make_embedder
+    from tht.corpus.chunk import ChunkPolicy
+    from tht.corpus.pipeline import CorpusPipeline
+    from tht.corpus.store import CorpusStore
+
+    cfg = _load_config_or_exit(config)
+    if cfg.embeddings is None:
+        raise RuntimeError("embeddings are not configured")
+    corpus_root = cfg.paths.artifacts.parent / "corpus"
+    generation = None
+    if resume:
+        generation = resume if resume.startswith("gen:") else f"gen:{resume}"
+    pipeline = CorpusPipeline(
+        store=CorpusStore(corpus_root), sources=build_evidence_sources(cfg),
+        embedder=make_embedder(cfg.embeddings),
+        vector_store=build_vector_store(cfg, require_write=True),
+        embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim,
+        chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars),
+        pipeline_version="evidence-v1",
+    )
+    return pipeline.run(dry_run=dry_run, resume=generation)
+
+
+@preprocess_app.command("evidence")
+def evidence_cmd(
+    config: Path = CONFIG_OPT,
+    dry_run: bool = typer.Option(False, "--dry-run"),
+    resume: str | None = typer.Option(None, "--resume"),
+    json_output: bool = typer.Option(False, "--json"),
+) -> None:
+    try:
+        result = run_from_config(config, dry_run=dry_run, resume=resume)
+    except Exception:
+        payload = {"status": "failed", "error": "preprocessing failed"}
+        if json_output:
+            typer.echo(json.dumps(payload, sort_keys=True))
+        else:
+            typer.secho("ERRORE: preprocessing failed", fg=typer.colors.RED, err=True)
+        raise typer.Exit(code=1) from None
+    payload = result.model_dump(mode="json")
+    if json_output:
+        typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True))
+    else:
+        typer.echo(
+            f"OK: generation={payload['generation']} changed={len(payload['changed'])} "
+            f"unchanged={len(payload['unchanged'])} removed={len(payload['removed'])}"
+        )