"""One-shot preprocessing commands.""" from __future__ import annotations import json from pathlib import Path import typer from tht.cli.config_cmd import CONFIG_OPT preprocess_app = typer.Typer(help="Materialize versioned preprocessing artifacts") def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None = None): from tht.adapters.factory import build_evidence_sources, build_vector_store from tht.cli.schema_cmd import _load_config_or_exit from tht.cli.vector_cmd import make_embedder from tht.corpus.chunk import ChunkPolicy from tht.corpus.pipeline import CorpusPipeline from tht.corpus.store import CorpusStore cfg = _load_config_or_exit(config) if cfg.embeddings is None: raise RuntimeError("embeddings are not configured") corpus_root = cfg.paths.artifacts.parent / "corpus" generation = None if resume: generation = resume if resume.startswith("gen:") else f"gen:{resume}" pipeline = CorpusPipeline( store=CorpusStore(corpus_root), sources=build_evidence_sources(cfg), embedder=make_embedder(cfg.embeddings), vector_store=build_vector_store(cfg, require_write=True), embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim, chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars), pipeline_version="evidence-v1", ) return pipeline.run(dry_run=dry_run, resume=generation) @preprocess_app.command("evidence") def evidence_cmd( config: Path = CONFIG_OPT, dry_run: bool = typer.Option(False, "--dry-run"), resume: str | None = typer.Option(None, "--resume"), json_output: bool = typer.Option(False, "--json"), ) -> None: try: result = run_from_config(config, dry_run=dry_run, resume=resume) except Exception: payload = {"status": "failed", "error": "preprocessing failed"} if json_output: typer.echo(json.dumps(payload, sort_keys=True)) else: typer.secho("ERRORE: preprocessing failed", fg=typer.colors.RED, err=True) raise typer.Exit(code=1) from None payload = result.model_dump(mode="json") if json_output: typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True)) else: typer.echo( f"OK: generation={payload['generation']} changed={len(payload['changed'])} " f"unchanged={len(payload['unchanged'])} removed={len(payload['removed'])}" )