"""One-shot preprocessing commands.""" from __future__ import annotations import hashlib import json import re from pathlib import Path import typer from tht.cli.config_cmd import CONFIG_OPT preprocess_app = typer.Typer(help="Materialize versioned preprocessing artifacts") def _bm25_language(workspace_language: str) -> str: languages = {"en": "english", "it": "italian"} try: return languages[workspace_language] except KeyError as exc: raise ValueError("workspace language is unsupported for Qdrant BM25") from exc def _evidence_json_context(config: Path): from tht.cli.schema_cmd import _load_config_or_exit return _load_config_or_exit(config) def _evaluation_workspace_root(cfg) -> Path: evidence = cfg.evidence if evidence is None: raise RuntimeError("Evidence evaluation fixture is unavailable") if evidence.source_root is not None: return evidence.source_root filesystem_roots = [source.root for source in evidence.sources if source.type == "filesystem"] if len(filesystem_roots) != 1: raise RuntimeError("Evidence evaluation requires one filesystem workspace source") root = filesystem_roots[0] if root.name == "curated": root = root.parent if root.name == "evidence": return root.parent return root def _candidate_evaluator(cfg, *, vector_store, embedder): """Bind candidate publication to the same read-only retrieval evaluator as the CLI.""" from tht.evidence.canonical import load_curated_tree from tht.evidence.evaluation import evaluate_retrieval, load_evaluation_fixture workspace_root = _evaluation_workspace_root(cfg) language = _bm25_language(cfg.language) def evaluate(manifest): document_generations = manifest.metadata.get("document_generations") if not isinstance(document_generations, dict): raise TypeError("candidate Evidence generation is invalid") return evaluate_retrieval( load_evaluation_fixture(workspace_root / "evidence" / "evaluation.yaml"), workspace_revision=cfg._workspace_revision, vector_generation=manifest.vector_generation, document_generations=document_generations, workspace_id=cfg._workspace_id, language=language, searcher=vector_store, embedder=embedder, expected_kinds={ evidence.id: evidence.kind for evidence in load_curated_tree(workspace_root / "evidence" / "curated") }, ) return evaluate def _validate_materialized_curated_corpus(cfg) -> None: """Fail closed on a v2 pinned filesystem corpus before any vector write is possible.""" if cfg.evidence is None or cfg.evidence.schema_version != 2: return if not any(source.type == "filesystem" for source in cfg.evidence.sources): return from tht.evidence import validate_workspace_evidence if not validate_workspace_evidence(_evaluation_workspace_root(cfg)).publishable: raise RuntimeError("curated Evidence corpus is invalid") def _evidence_json_payload(cfg, payload: dict, *, code: str, error: str | None = None) -> dict: value = { **payload, "schemaVersion": 1, "status": payload.get("status", "failed"), "code": code, "operation": "preprocess_evidence", "workspaceId": cfg._workspace_id, "workspaceRevision": cfg._workspace_revision, } if error is not None: value["error"] = error return value def _simple_json_payload(*, code: str, error: str) -> dict: return { "schemaVersion": 1, "status": "failed", "code": code, "operation": "preprocess_evidence", "error": error, } def run_dwh_from_config( config: Path, *, steps: tuple[str, ...], resume: str | None = None, ): from tht.cli.lsh_cmd import build_lsh_artifacts from tht.cli.schema_cmd import _load_config_or_exit, refresh_catalog from tht.jobs.dwh_pipeline import ( DwhPreprocessPipeline, config_dwh_binding, ) cfg = _load_config_or_exit(config) binding = config_dwh_binding(cfg) workspace_root = cfg.paths.artifacts.parent lsh_names = ( f"{cfg.database.db_schema}_lsh.pkl", f"{cfg.database.db_schema}_minhashes.pkl", f"{cfg.database.db_schema}_meta.json", ) pipeline = DwhPreprocessPipeline( workspace_id=binding["workspace_id"], workspace_root=workspace_root, config_fingerprint=binding["config_fingerprint"], input_fingerprint=binding["input_fingerprint"], introspect=lambda output: refresh_catalog(cfg, output_path=output), build_lsh=lambda physical, output: build_lsh_artifacts( cfg, physical_file=physical, output_dir=output ), lsh_filenames=lsh_names, current_physical=cfg.paths.artifacts / "mschema" / "physical.yaml", current_lsh_dir=cfg.paths.indexes / "lsh", ) return pipeline.run(steps, resume_run_id=resume) def _parse_dwh_steps(value: str) -> tuple[str, ...]: allowed = ("introspect", "lsh") steps = tuple(part.strip() for part in value.split(",") if part.strip()) if ( not steps or len(steps) != len(set(steps)) or any(step not in allowed for step in steps) or tuple(sorted(steps, key=allowed.index)) != steps ): raise ValueError("steps must be a unique ordered subset of introspect,lsh") return steps def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None = None): from tht.adapters.factory import build_vector_store from tht.cli.schema_cmd import _load_config_or_exit from tht.cli.vector_cmd import make_embedder from tht.evidence import build_preprocessing_pipeline, build_sources from tht.evidence.corpus.chunk import ChunkPolicy from tht.evidence.corpus.store import CorpusStore cfg = _load_config_or_exit(config) if cfg.embeddings is None: raise RuntimeError("embeddings are not configured") _validate_materialized_curated_corpus(cfg) corpus_root = cfg.paths.artifacts.parent / "corpus" vector_store = build_vector_store(cfg, require_write=True) embedder = make_embedder(cfg.embeddings) pipeline = build_preprocessing_pipeline( store=CorpusStore(corpus_root), sources=build_sources(cfg.evidence), embedder=embedder, vector_store=vector_store, embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim, chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars), pipeline_version="evidence-v1", retain_published_generations=cfg.vector.retain_published_generations, sparse_language=_bm25_language(cfg.language), candidate_evaluator=_candidate_evaluator(cfg, vector_store=vector_store, embedder=embedder), ) def fingerprint(value: str) -> str: return "sha256:" + hashlib.sha256(value.encode()).hexdigest() return pipeline.run_as_job( workspace_id=cfg._workspace_id, workspace_root=corpus_root.parent, config_fingerprint=fingerprint(cfg.model_dump_json()), input_fingerprint=fingerprint(config.resolve().as_posix()), dry_run=dry_run, resume_run_id=resume, ) def gc_from_config(config: Path, *, dry_run: bool = False): from tht.adapters.factory import build_vector_store from tht.cli.schema_cmd import _load_config_or_exit from tht.cli.vector_cmd import make_embedder from tht.evidence import build_preprocessing_pipeline, build_sources from tht.evidence.corpus.chunk import ChunkPolicy from tht.evidence.corpus.store import CorpusStore cfg = _load_config_or_exit(config) if cfg.embeddings is None: raise RuntimeError("embeddings are not configured") corpus_root = cfg.paths.artifacts.parent / "corpus" pipeline = build_preprocessing_pipeline( store=CorpusStore(corpus_root), sources=build_sources(cfg.evidence), embedder=make_embedder(cfg.embeddings), vector_store=build_vector_store(cfg, require_write=True), embedding_model=cfg.embeddings.model, embedding_dimensions=cfg.embeddings.dim, chunk_policy=ChunkPolicy(version="chunk-v1", max_chars=cfg.vector.max_chunk_chars), pipeline_version="evidence-v1", retain_published_generations=cfg.vector.retain_published_generations, sparse_language=_bm25_language(cfg.language), ) pipeline.workspace_id = cfg._workspace_id return pipeline.gc(workspace_root=corpus_root.parent, dry_run=dry_run) @preprocess_app.command("evidence") def evidence_cmd( action: str | None = typer.Argument(None), config: Path = CONFIG_OPT, dry_run: bool = typer.Option(False, "--dry-run"), resume: str | None = typer.Option(None, "--resume"), json_output: bool = typer.Option(False, "--json"), ) -> None: if action is not None and action != "gc": raise typer.BadParameter("only the optional 'gc' action is supported") if action == "gc": try: payload = gc_from_config(config, dry_run=dry_run) except Exception: # noqa: BLE001 payload = {"status": "failed", "error": "evidence cleanup failed"} if json_output: typer.echo(json.dumps(payload, sort_keys=True)) else: typer.secho("ERRORE: evidence cleanup failed", fg=typer.colors.RED, err=True) raise typer.Exit(code=1) from None if json_output: typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True)) else: typer.echo(f"OK: evicted={len(payload['evicted'])} failures={len(payload['failures'])}") return cfg = _evidence_json_context(config) if json_output else None if resume is not None and re.fullmatch(r"[0-9a-f]{32}", resume) is None: payload = _simple_json_payload( code="invalid_resume", error="resume requires a preprocessing run id", ) if json_output: typer.echo(json.dumps(payload, sort_keys=True)) else: typer.secho("ERRORE: resume requires a preprocessing run id", fg=typer.colors.RED, err=True) raise typer.Exit(code=2) try: result = run_from_config(config, dry_run=dry_run, resume=resume) except Exception: # noqa: BLE001 payload = {"status": "failed"} if json_output: typer.echo(json.dumps( _evidence_json_payload( cfg, payload, code="preprocessing_failed", error="preprocessing failed", ), sort_keys=True, )) else: typer.secho("ERRORE: preprocessing failed", fg=typer.colors.RED, err=True) raise typer.Exit(code=1) from None payload = result.model_dump(mode="json") if payload.get("status") != "succeeded": if json_output: typer.echo(json.dumps( _evidence_json_payload( cfg, payload, code="preprocessing_failed", error="preprocessing job failed", ), ensure_ascii=False, sort_keys=True, )) else: typer.secho("ERRORE: preprocessing job failed", fg=typer.colors.RED, err=True) raise typer.Exit(code=1) if json_output: typer.echo(json.dumps( _evidence_json_payload(cfg, payload, code="ok"), ensure_ascii=False, sort_keys=True, )) else: counts = payload["counts"] typer.echo( f"OK: run={payload['run_id']} generation={payload['generation']} " f"changed={counts['changed']} unchanged={counts['unchanged']} " f"removed={counts['removed']}" ) @preprocess_app.command("dwh") def dwh_cmd( config: Path = CONFIG_OPT, steps: str = typer.Option("introspect,lsh", "--steps"), resume: str | None = typer.Option(None, "--resume"), json_output: bool = typer.Option(False, "--json"), ) -> None: try: selected = _parse_dwh_steps(steps) except ValueError: payload = {"status": "failed", "error": "invalid DWH preprocessing steps"} if json_output: typer.echo(json.dumps(payload, sort_keys=True)) else: typer.secho("ERRORE: invalid DWH preprocessing steps", fg=typer.colors.RED, err=True) raise typer.Exit(code=2) from None if resume is not None and re.fullmatch(r"[0-9a-f]{32}", resume) is None: payload = {"status": "failed", "error": "resume requires a preprocessing run id"} if json_output: typer.echo(json.dumps(payload, sort_keys=True)) else: typer.secho("ERRORE: resume requires a preprocessing run id", fg=typer.colors.RED, err=True) raise typer.Exit(code=2) try: result = run_dwh_from_config(config, steps=selected, resume=resume) except Exception: # noqa: BLE001 payload = {"status": "failed", "error": "DWH preprocessing failed"} if json_output: typer.echo(json.dumps(payload, sort_keys=True)) else: typer.secho("ERRORE: DWH preprocessing failed", fg=typer.colors.RED, err=True) raise typer.Exit(code=1) from None payload = result.model_dump(mode="json") if json_output: typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True)) elif result.status == "succeeded": typer.echo(f"OK: run={result.run_id} stages={','.join(selected)}") else: typer.secho(f"ERRORE: run={result.run_id} DWH preprocessing failed", fg=typer.colors.RED, err=True) if result.status != "succeeded": raise typer.Exit(code=1)