"""One-shot preprocessing commands.""" from __future__ import annotations import json import re import hashlib from pathlib import Path import typer from tht.cli.config_cmd import CONFIG_OPT preprocess_app = typer.Typer(help="Materialize versioned preprocessing artifacts") def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None = None): from tht.adapters.factory import build_evidence_sources, build_vector_store from tht.cli.schema_cmd import _load_config_or_exit from tht.cli.vector_cmd import make_embedder from tht.corpus.chunk import ChunkPolicy from tht.corpus.pipeline import CorpusPipeline from tht.corpus.store import CorpusStore cfg = _load_config_or_exit(config) if cfg.embeddings is None: raise RuntimeError("embeddings are not configured") corpus_root = cfg.paths.artifacts.parent / "corpus" pipeline = CorpusPipeline( store=CorpusStore(corpus_root), sources=build_evidence_sources(cfg), embedder=make_embedder(cfg.embeddings), vector_store=build_vector_store(cfg, require_write=True), embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim, chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars), pipeline_version="evidence-v1", retain_published_generations=cfg.vector.retain_published_generations, ) def fingerprint(value: str) -> str: return "sha256:" + hashlib.sha256(value.encode()).hexdigest() return pipeline.run_as_job( workspace_id=config.stem.lower().replace(".", "-").replace("_", "-"), workspace_root=corpus_root.parent, config_fingerprint=fingerprint(cfg.model_dump_json()), input_fingerprint=fingerprint(config.resolve().as_posix()), dry_run=dry_run, resume_run_id=resume, ) def gc_from_config(config: Path, *, dry_run: bool = False): from tht.adapters.factory import build_evidence_sources, build_vector_store from tht.cli.schema_cmd import _load_config_or_exit from tht.cli.vector_cmd import make_embedder from tht.corpus.chunk import ChunkPolicy from tht.corpus.pipeline import CorpusPipeline from tht.corpus.store import CorpusStore cfg = _load_config_or_exit(config) if cfg.embeddings is None: raise RuntimeError("embeddings are not configured") corpus_root = cfg.paths.artifacts.parent / "corpus" pipeline = CorpusPipeline( store=CorpusStore(corpus_root), sources=build_evidence_sources(cfg), embedder=make_embedder(cfg.embeddings), vector_store=build_vector_store(cfg, require_write=True), embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim, chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars), pipeline_version="evidence-v1", retain_published_generations=cfg.vector.retain_published_generations, ) with pipeline.store.writer_lock(): return pipeline.gc(workspace_root=corpus_root.parent, dry_run=dry_run) @preprocess_app.command("evidence") def evidence_cmd( action: str | None = typer.Argument(None), config: Path = CONFIG_OPT, dry_run: bool = typer.Option(False, "--dry-run"), resume: str | None = typer.Option(None, "--resume"), json_output: bool = typer.Option(False, "--json"), ) -> None: if action is not None and action != "gc": raise typer.BadParameter("only the optional 'gc' action is supported") if action == "gc": try: payload = gc_from_config(config, dry_run=dry_run) except Exception: payload = {"status": "failed", "error": "evidence cleanup failed"} if json_output: typer.echo(json.dumps(payload, sort_keys=True)) else: typer.secho("ERRORE: evidence cleanup failed", fg=typer.colors.RED, err=True) raise typer.Exit(code=1) from None if json_output: typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True)) else: typer.echo(f"OK: evicted={len(payload['evicted'])} failures={len(payload['failures'])}") return if resume is not None and re.fullmatch(r"[0-9a-f]{32}", resume) is None: payload = {"status": "failed", "error": "resume requires a preprocessing run id"} if json_output: typer.echo(json.dumps(payload, sort_keys=True)) else: typer.secho("ERRORE: resume requires a preprocessing run id", fg=typer.colors.RED, err=True) raise typer.Exit(code=2) try: result = run_from_config(config, dry_run=dry_run, resume=resume) except Exception: payload = {"status": "failed", "error": "preprocessing failed"} if json_output: typer.echo(json.dumps(payload, sort_keys=True)) else: typer.secho("ERRORE: preprocessing failed", fg=typer.colors.RED, err=True) raise typer.Exit(code=1) from None payload = result.model_dump(mode="json") if json_output: typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True)) else: typer.echo( f"OK: run={payload['run_id']} generation={payload['generation']} changed={len(payload['changed'])} " f"unchanged={len(payload['unchanged'])} removed={len(payload['removed'])}" )