feat(preprocess): add resumable DWH jobs

harness/tht/cli/lsh_cmd.py

@@ -36,12 +36,30 @@ def _extract_lsh_values(dwh, physical, annotations, limit):
     return values, skipped, truncated
 
 
-@lsh_app.command("build")
-def build_cmd(config: Path = CONFIG_OPT) -> None:
-    """Costruisce l'indice LSH dai valori del database e lo salva su pickle."""
+def build_lsh_artifacts(cfg, *, dwh=None, verbose: bool = False):
+    """Run the existing LSH extraction/build algorithm and persist its outputs."""
+    from tht.adapters.factory import build_dwh
+    from tht.cli.schema_cmd import annotations_path
     from tht.lshindex import build_index, save_index
     from tht.mschema.models import Annotations, PhysicalSchema
 
+    phys_file = physical_path(cfg)
+    if not phys_file.exists():
+        raise FileNotFoundError("physical catalog is missing; run schema introspect first")
+    physical = PhysicalSchema.from_yaml(phys_file)
+    annotations = Annotations.from_yaml(annotations_path(cfg))
+    target = dwh if dwh is not None else build_dwh(cfg)
+    values, skipped, truncated = _extract_lsh_values(
+        target, physical, annotations, cfg.lsh.max_values_per_column
+    )
+    lsh, minhashes = build_index(values, cfg.lsh, verbose=verbose)
+    save_index(lsh, minhashes, cfg.lsh, _lsh_dir(cfg), name=cfg.database.db_schema)
+    return minhashes, skipped, truncated, values
+
+
+@lsh_app.command("build")
+def build_cmd(config: Path = CONFIG_OPT) -> None:
+    """Costruisce l'indice LSH dai valori del database e lo salva su pickle."""
     cfg = _load_config_or_exit(config)
     phys_file = physical_path(cfg)
     if not phys_file.exists():
@@ -50,17 +68,8 @@ def build_cmd(config: Path = CONFIG_OPT) -> None:
             fg=typer.colors.RED, err=True,
         )
         raise typer.Exit(code=1)
-    physical = PhysicalSchema.from_yaml(phys_file)
-    from tht.cli.schema_cmd import annotations_path
-
-    annotations = Annotations.from_yaml(annotations_path(cfg))
-
     typer.echo("Estrazione valori (i più frequenti) dalle colonne testuali eligible...")
-    from tht.adapters.factory import build_dwh
-    dwh = build_dwh(cfg)
-    values, skipped, truncated = _extract_lsh_values(
-        dwh, physical, annotations, cfg.lsh.max_values_per_column
-    )
+    minhashes, skipped, truncated, values = build_lsh_artifacts(cfg, verbose=True)
     n_values = sum(len(v) for t in values.values() for v in t.values())
     typer.echo(f"  {n_values} valori da {sum(len(t) for t in values.values())} colonne")
     for s in skipped:
@@ -72,8 +81,6 @@ def build_cmd(config: Path = CONFIG_OPT) -> None:
             fg=typer.colors.YELLOW,
         )
 
-    lsh, minhashes = build_index(values, cfg.lsh, verbose=True)
-    save_index(lsh, minhashes, cfg.lsh, _lsh_dir(cfg), name=cfg.database.db_schema)
     typer.secho(
         f"OK: indice LSH ({len(minhashes)} entry) -> {_lsh_dir(cfg)}", fg=typer.colors.GREEN
     )

harness/tht/cli/preprocess_cmd.py

@@ -15,6 +15,39 @@ from tht.cli.config_cmd import CONFIG_OPT
 preprocess_app = typer.Typer(help="Materialize versioned preprocessing artifacts")
 
 
+def run_dwh_from_config(
+    config: Path, *, steps: tuple[str, ...], resume: str | None = None,
+):
+    from tht.cli.lsh_cmd import build_lsh_artifacts
+    from tht.cli.schema_cmd import _load_config_or_exit, refresh_catalog
+    from tht.jobs.dwh_pipeline import DwhPreprocessPipeline, fingerprint
+
+    cfg = _load_config_or_exit(config)
+    workspace_root = cfg.paths.artifacts.parent
+    pipeline = DwhPreprocessPipeline(
+        workspace_id=config.stem.lower().replace(".", "-").replace("_", "-"),
+        workspace_root=workspace_root,
+        config_fingerprint=fingerprint(cfg.model_dump_json()),
+        input_fingerprint=fingerprint(config.resolve().as_posix()),
+        introspect=lambda: refresh_catalog(cfg),
+        build_lsh=lambda: build_lsh_artifacts(cfg),
+    )
+    return pipeline.run(steps, resume_run_id=resume)
+
+
+def _parse_dwh_steps(value: str) -> tuple[str, ...]:
+    allowed = ("introspect", "lsh")
+    steps = tuple(part.strip() for part in value.split(",") if part.strip())
+    if (
+        not steps
+        or len(steps) != len(set(steps))
+        or any(step not in allowed for step in steps)
+        or tuple(sorted(steps, key=allowed.index)) != steps
+    ):
+        raise ValueError("steps must be a unique ordered subset of introspect,lsh")
+    return steps
+
+
 def run_from_config(config: Path, *, dry_run: bool = False, resume: str | None = None):
     from tht.adapters.factory import build_evidence_sources, build_vector_store
     from tht.cli.schema_cmd import _load_config_or_exit
@@ -122,3 +155,46 @@ def evidence_cmd(
             f"OK: run={payload['run_id']} generation={payload['generation']} changed={len(payload['changed'])} "
             f"unchanged={len(payload['unchanged'])} removed={len(payload['removed'])}"
         )
+
+
+@preprocess_app.command("dwh")
+def dwh_cmd(
+    config: Path = CONFIG_OPT,
+    steps: str = typer.Option("introspect,lsh", "--steps"),
+    resume: str | None = typer.Option(None, "--resume"),
+    json_output: bool = typer.Option(False, "--json"),
+) -> None:
+    try:
+        selected = _parse_dwh_steps(steps)
+    except ValueError:
+        payload = {"status": "failed", "error": "invalid DWH preprocessing steps"}
+        if json_output:
+            typer.echo(json.dumps(payload, sort_keys=True))
+        else:
+            typer.secho("ERRORE: invalid DWH preprocessing steps", fg=typer.colors.RED, err=True)
+        raise typer.Exit(code=2) from None
+    if resume is not None and re.fullmatch(r"[0-9a-f]{32}", resume) is None:
+        payload = {"status": "failed", "error": "resume requires a preprocessing run id"}
+        if json_output:
+            typer.echo(json.dumps(payload, sort_keys=True))
+        else:
+            typer.secho("ERRORE: resume requires a preprocessing run id", fg=typer.colors.RED, err=True)
+        raise typer.Exit(code=2)
+    try:
+        result = run_dwh_from_config(config, steps=selected, resume=resume)
+    except Exception:
+        payload = {"status": "failed", "error": "DWH preprocessing failed"}
+        if json_output:
+            typer.echo(json.dumps(payload, sort_keys=True))
+        else:
+            typer.secho("ERRORE: DWH preprocessing failed", fg=typer.colors.RED, err=True)
+        raise typer.Exit(code=1) from None
+    payload = result.model_dump(mode="json")
+    if json_output:
+        typer.echo(json.dumps(payload, ensure_ascii=False, sort_keys=True))
+    elif result.status == "succeeded":
+        typer.echo(f"OK: run={result.run_id} stages={','.join(selected)}")
+    else:
+        typer.secho(f"ERRORE: run={result.run_id} DWH preprocessing failed", fg=typer.colors.RED, err=True)
+    if result.status != "succeeded":
+        raise typer.Exit(code=1)

harness/tht/cli/schema_cmd.py

@@ -44,6 +44,16 @@ def annotations_path(cfg) -> Path:
     return cfg.paths.artifacts / "mschema" / "annotations.yaml"
 
 
+def refresh_catalog(cfg, *, dwh=None):
+    """Run the existing catalog algorithm and persist its canonical output."""
+    target = dwh if dwh is not None else build_dwh(cfg)
+    physical = target.introspect()
+    _add_examples(target, physical, cfg.examples)
+    classify_all(physical, cfg.eligibility)
+    physical.to_yaml(physical_path(cfg))
+    return physical
+
+
 @schema_app.command("introspect")
 def introspect_cmd(
     config: Path = CONFIG_OPT,
@@ -80,14 +90,10 @@ def introspect_cmd(
             )
             return
     try:
-        dwh = build_dwh(cfg)
-        phys = dwh.introspect()
-        _add_examples(dwh, phys, cfg.examples)
-        classify_all(phys, cfg.eligibility)
+        phys = refresh_catalog(cfg)
     except Exception as e:
         typer.secho(f"ERRORE: {e}", fg=typer.colors.RED, err=True)
         raise typer.Exit(code=1)
-    phys.to_yaml(out)
     n_cols = sum(len(t.columns) for t in phys.tables.values())
     n_ignored = sum(
         1 for t in phys.tables.values() for c in t.columns.values() if not c.eligible