feat: complete catalog-driven preprocessing

harness/tht/cli/preprocess_cmd.py

@@ -4,7 +4,10 @@ from __future__ import annotations
 
 import hashlib
 import json
+import os
 import re
+import shutil
+import stat
 from pathlib import Path
 
 import typer
@@ -144,6 +147,8 @@ def run_dwh_from_config(
         workspace_root=workspace_root,
         config_fingerprint=binding["config_fingerprint"],
         input_fingerprint=binding["input_fingerprint"],
+        catalog_database_id=binding.get("catalog_database_id"),
+        metadata_content_revision=binding.get("metadata_content_revision"),
         introspect=lambda output: refresh_catalog(cfg, output_path=output),
         build_lsh=lambda physical, output: build_lsh_artifacts(
             cfg, physical_file=physical, output_dir=output
@@ -155,6 +160,57 @@ def run_dwh_from_config(
     return pipeline.run(steps, resume_run_id=resume)
 
 
+def run_catalog_dwh_from_config(cfg):
+    """Publish Catalog-derived physical schema and LSH as one bound generation."""
+    from tht.cli.lsh_cmd import build_lsh_artifacts
+    from tht.jobs.dwh_pipeline import DwhPreprocessPipeline, config_dwh_binding
+    from tht.mschema.catalog_snapshot import load_catalog_metadata_snapshot
+
+    snapshot = load_catalog_metadata_snapshot(
+        cfg.paths.catalog_metadata_snapshot, cfg._workspace_id
+    )
+    binding = config_dwh_binding(cfg)
+    workspace_root = cfg.paths.artifacts.parent
+    # Catalog preprocessing is a replace-in-place operation. Its DWH/LSH output is wholly
+    # derived, there is no supported concurrent runtime, and a failed Clear may have left an
+    # older binding behind. Start from an empty owned generation root so a retry can always
+    # rebuild the current Catalog revision instead of deadlocking on the stale OWNER marker.
+    _remove_owned_derived_path(workspace_root, workspace_root / ".tht-dwh")
+    _remove_owned_derived_path(workspace_root, cfg.paths.artifacts / "mschema" / "physical.yaml")
+    _remove_owned_derived_path(workspace_root, cfg.paths.indexes / "lsh")
+    observed: dict[str, object] = {}
+
+    def materialize_physical(output: Path):
+        physical, _annotations, _relationships = snapshot.to_schema_inputs()
+        physical.to_yaml(output)
+        return physical
+
+    def materialize_lsh(physical: Path, output: Path):
+        result = build_lsh_artifacts(cfg, physical_file=physical, output_dir=output)
+        observed["result"] = result
+        return result
+
+    report = DwhPreprocessPipeline(
+        workspace_id=str(binding["workspace_id"]),
+        workspace_root=workspace_root,
+        config_fingerprint=str(binding["config_fingerprint"]),
+        input_fingerprint=str(binding["input_fingerprint"]),
+        catalog_database_id=str(binding["catalog_database_id"]),
+        metadata_content_revision=int(binding["metadata_content_revision"]),
+        introspect=materialize_physical,
+        build_lsh=materialize_lsh,
+        lsh_filenames=(
+            f"{cfg.database.db_schema}_lsh.pkl",
+            f"{cfg.database.db_schema}_minhashes.pkl",
+            f"{cfg.database.db_schema}_meta.json",
+        ),
+    ).run(("introspect", "lsh"))
+    result = observed.get("result")
+    if not isinstance(result, tuple) or len(result) != 4:
+        raise RuntimeError("Catalog LSH publication did not complete")
+    return report, result
+
+
 def _parse_dwh_steps(value: str) -> tuple[str, ...]:
     allowed = ("introspect", "lsh")
     steps = tuple(part.strip() for part in value.split(",") if part.strip())
@@ -234,6 +290,85 @@ def gc_from_config(config: Path, *, dry_run: bool = False):
     return pipeline.gc(workspace_root=corpus_root.parent, dry_run=dry_run)
 
 
+def _remove_owned_derived_path(workspace_root: Path, target: Path) -> bool:
+    """Remove one generated path without following links or escaping the workspace."""
+    root = workspace_root.resolve()
+    resolved = target.resolve(strict=False)
+    if not resolved.is_relative_to(root):
+        raise RuntimeError("derived cleanup target escapes the workspace")
+    try:
+        info = target.lstat()
+    except FileNotFoundError:
+        return False
+    if stat.S_ISLNK(info.st_mode) or info.st_uid != os.getuid():
+        raise RuntimeError("derived cleanup target is unsafe")
+    if stat.S_ISDIR(info.st_mode):
+        shutil.rmtree(target)
+    elif stat.S_ISREG(info.st_mode):
+        target.unlink()
+    else:
+        raise RuntimeError("derived cleanup target is unsafe")
+    return True
+
+
+def clear_from_config(config: Path) -> dict[str, int]:
+    """Clear workspace reference vectors and local preprocessing derivatives."""
+    from tht.adapters.factory import build_vector_store
+    from tht.cli._guards import require_vector_write_allowed
+    from tht.cli.schema_cmd import _load_config_or_exit
+
+    cfg = _load_config_or_exit(config)
+    require_vector_write_allowed(cfg, "preprocess clear")
+    workspace_root = cfg.paths.artifacts.parent
+    vector_store = build_vector_store(cfg, require_write=True)
+    reference_deleted = int(vector_store.clear_reference())
+    paths = [
+        workspace_root / ".tht-dwh",
+        workspace_root / ".tht-jobs",
+        workspace_root / "corpus",
+        cfg.paths.artifacts / "mschema" / "physical.yaml",
+        cfg.paths.indexes / "lsh",
+    ]
+    if cfg.paths.catalog_metadata_snapshot is not None:
+        paths.append(cfg.paths.catalog_metadata_snapshot)
+    removed = sum(int(_remove_owned_derived_path(workspace_root, path)) for path in paths)
+    return {"referenceCollections": reference_deleted, "derivedPaths": removed}
+
+
+@preprocess_app.command("clear", hidden=True)
+def clear_cmd(
+    config: Path = CONFIG_OPT,
+    json_output: bool = typer.Option(False, "--json"),
+) -> None:
+    """Clear replaceable preprocessing output while preserving workspace memory."""
+    try:
+        counts = clear_from_config(config)
+    except Exception:  # noqa: BLE001 - do not disclose paths, endpoints, or credentials
+        payload = {
+            "schemaVersion": 1,
+            "status": "failed",
+            "code": "preprocessing_clear_failed",
+            "operation": "preprocess_clear",
+            "error": "Preprocessing clear failed",
+        }
+        if json_output:
+            typer.echo(json.dumps(payload, sort_keys=True))
+        else:
+            typer.secho("ERRORE: preprocessing clear failed", fg=typer.colors.RED, err=True)
+        raise typer.Exit(code=1) from None
+    payload = {
+        "schemaVersion": 1,
+        "status": "succeeded",
+        "code": "ok",
+        "operation": "preprocess_clear",
+        "counts": counts,
+    }
+    if json_output:
+        typer.echo(json.dumps(payload, sort_keys=True))
+    else:
+        typer.echo("OK: reference vectors and LSH cleared; memory preserved")
+
+
 @preprocess_app.command("evidence")
 def evidence_cmd(
     action: str | None = typer.Argument(None),
@@ -359,3 +494,71 @@ def dwh_cmd(
         typer.secho(f"ERRORE: run={result.run_id} DWH preprocessing failed", fg=typer.colors.RED, err=True)
     if result.status != "succeeded":
         raise typer.Exit(code=1)
+
+
+@preprocess_app.command("catalog", hidden=True)
+def catalog_cmd(
+    catalog_metadata: Path = typer.Option(..., "--catalog-metadata"),
+    config: Path = CONFIG_OPT,
+    json_output: bool = typer.Option(False, "--json"),
+) -> None:
+    """Build current LSH and schema vectors from one immutable Catalog snapshot."""
+    from tht.cli._guards import require_vector_write_allowed
+    from tht.cli.schema_cmd import _load_config_or_exit
+    from tht.cli.vector_cmd import index_catalog_schema, require_vector_cfg
+
+    cfg = _load_config_or_exit(config)
+    cfg = cfg.model_copy(
+        update={
+            "paths": cfg.paths.model_copy(
+                update={"catalog_metadata_snapshot": catalog_metadata}
+            )
+        }
+    )
+    require_vector_write_allowed(cfg, "preprocess catalog")
+    require_vector_cfg(cfg)
+    try:
+        report, (minhashes, skipped, truncated, _values) = run_catalog_dwh_from_config(cfg)
+        if report.status != "succeeded":
+            raise RuntimeError("Catalog DWH preprocessing failed")
+        stats, counts, snapshot_path = index_catalog_schema(cfg)
+    except Exception:  # noqa: BLE001 - public output must never disclose endpoints or SQL
+        payload = {
+            "schemaVersion": 1,
+            "status": "failed",
+            "code": "catalog_preprocessing_failed",
+            "operation": "preprocess_catalog",
+            "error": "Catalog preprocessing failed",
+        }
+        if json_output:
+            typer.echo(json.dumps(payload, sort_keys=True))
+        else:
+            typer.secho("ERRORE: Catalog preprocessing failed", fg=typer.colors.RED, err=True)
+        raise typer.Exit(code=1) from None
+
+    payload = {
+        "schemaVersion": 1,
+        "status": "succeeded",
+        "code": "ok",
+        "operation": "preprocess_catalog",
+        "artifactIdentities": [
+            {
+                "kind": "catalog_metadata_snapshot",
+                "digest": "sha256:" + hashlib.sha256(snapshot_path.read_bytes()).hexdigest(),
+            }
+        ],
+        "counts": {
+            **counts,
+            "lshEntries": len(minhashes),
+            "lshSkippedColumns": len(skipped),
+            "lshTruncatedColumns": len(truncated),
+            "added": stats.added,
+            "deleted": stats.deleted,
+            "unchanged": stats.unchanged,
+            "updated": stats.updated,
+        },
+    }
+    if json_output:
+        typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True))
+    else:
+        typer.echo("OK: Catalog metadata, LSH and schema vectors rebuilt")