fix(preprocess): publish DWH artifacts atomically

harness/tht/cli/lsh_cmd.py

@@ -9,6 +9,11 @@ lsh_app = typer.Typer(help="Indice LSH su valori dei campi (derivato, rigenerabi
 
 
 def _lsh_dir(cfg) -> Path:
+    from tht.jobs.dwh_pipeline import active_generation_dir
+
+    active = active_generation_dir(cfg.paths.artifacts.parent)
+    if active is not None:
+        return active
     return cfg.paths.indexes / "lsh"
 
 
@@ -36,14 +41,17 @@ def _extract_lsh_values(dwh, physical, annotations, limit):
     return values, skipped, truncated
 
 
-def build_lsh_artifacts(cfg, *, dwh=None, verbose: bool = False):
+def build_lsh_artifacts(
+    cfg, *, dwh=None, verbose: bool = False, physical_file: Path | None = None,
+    output_dir: Path | None = None,
+):
     """Run the existing LSH extraction/build algorithm and persist its outputs."""
     from tht.adapters.factory import build_dwh
     from tht.cli.schema_cmd import annotations_path
     from tht.lshindex import build_index, save_index
     from tht.mschema.models import Annotations, PhysicalSchema
 
-    phys_file = physical_path(cfg)
+    phys_file = physical_file or physical_path(cfg)
     if not phys_file.exists():
         raise FileNotFoundError("physical catalog is missing; run schema introspect first")
     physical = PhysicalSchema.from_yaml(phys_file)
@@ -53,7 +61,10 @@ def build_lsh_artifacts(cfg, *, dwh=None, verbose: bool = False):
         target, physical, annotations, cfg.lsh.max_values_per_column
     )
     lsh, minhashes = build_index(values, cfg.lsh, verbose=verbose)
-    save_index(lsh, minhashes, cfg.lsh, _lsh_dir(cfg), name=cfg.database.db_schema)
+    save_index(
+        lsh, minhashes, cfg.lsh, output_dir or (cfg.paths.indexes / "lsh"),
+        name=cfg.database.db_schema,
+    )
     return minhashes, skipped, truncated, values
 
 
@@ -69,9 +80,25 @@ def build_cmd(config: Path = CONFIG_OPT) -> None:
         )
         raise typer.Exit(code=1)
     typer.echo("Estrazione valori (i più frequenti) dalle colonne testuali eligible...")
-    minhashes, skipped, truncated, values = build_lsh_artifacts(cfg, verbose=True)
-    n_values = sum(len(v) for t in values.values() for v in t.values())
-    typer.echo(f"  {n_values} valori da {sum(len(t) for t in values.values())} colonne")
+    from tht.jobs.dwh_pipeline import active_generation_dir
+
+    if active_generation_dir(cfg.paths.artifacts.parent) is None:
+        minhashes, skipped, truncated, values = build_lsh_artifacts(cfg, verbose=True)
+        n_values = sum(len(v) for table in values.values() for v in table.values())
+        n_columns = sum(len(table) for table in values.values())
+    else:
+        from tht.cli.preprocess_cmd import run_dwh_from_config
+        from tht.lshindex import load_index
+
+        report = run_dwh_from_config(config, steps=("lsh",))
+        if report.status != "succeeded":
+            typer.secho("ERRORE: DWH preprocessing failed", fg=typer.colors.RED, err=True)
+            raise typer.Exit(code=1)
+        _, minhashes, _ = load_index(_lsh_dir(cfg), name=cfg.database.db_schema)
+        skipped, truncated = [], []
+        n_values = len(minhashes)
+        n_columns = len({(entry[1], entry[2]) for entry in minhashes.values()})
+    typer.echo(f"  {n_values} valori da {n_columns} colonne")
     for s in skipped:
         typer.secho(f"  saltata {s.table}.{s.column}: {s.reason}", fg=typer.colors.YELLOW)
     for t in truncated:

harness/tht/cli/preprocess_cmd.py

@@ -19,18 +19,29 @@ def run_dwh_from_config(
     config: Path, *, steps: tuple[str, ...], resume: str | None = None,
 ):
     from tht.cli.lsh_cmd import build_lsh_artifacts
-    from tht.cli.schema_cmd import _load_config_or_exit, refresh_catalog
-    from tht.jobs.dwh_pipeline import DwhPreprocessPipeline, fingerprint
+    from tht.cli.schema_cmd import _load_config_or_exit, physical_path, refresh_catalog
+    from tht.jobs.dwh_pipeline import DwhPreprocessPipeline, active_generation_dir, fingerprint
 
     cfg = _load_config_or_exit(config)
     workspace_root = cfg.paths.artifacts.parent
+    active = active_generation_dir(workspace_root)
+    lsh_names = (
+        f"{cfg.database.db_schema}_lsh.pkl",
+        f"{cfg.database.db_schema}_minhashes.pkl",
+        f"{cfg.database.db_schema}_meta.json",
+    )
     pipeline = DwhPreprocessPipeline(
         workspace_id=config.stem.lower().replace(".", "-").replace("_", "-"),
         workspace_root=workspace_root,
         config_fingerprint=fingerprint(cfg.model_dump_json()),
         input_fingerprint=fingerprint(config.resolve().as_posix()),
-        introspect=lambda: refresh_catalog(cfg),
-        build_lsh=lambda: build_lsh_artifacts(cfg),
+        introspect=lambda output: refresh_catalog(cfg, output_path=output),
+        build_lsh=lambda physical, output: build_lsh_artifacts(
+            cfg, physical_file=physical, output_dir=output
+        ),
+        lsh_filenames=lsh_names,
+        current_physical=physical_path(cfg),
+        current_lsh_dir=active if active is not None else cfg.paths.indexes / "lsh",
     )
     return pipeline.run(steps, resume_run_id=resume)
 

harness/tht/cli/schema_cmd.py

@@ -37,6 +37,11 @@ def _load_config_or_exit(config: Path):
 
 
 def physical_path(cfg) -> Path:
+    from tht.jobs.dwh_pipeline import active_generation_dir
+
+    active = active_generation_dir(cfg.paths.artifacts.parent)
+    if active is not None:
+        return active / "physical.yaml"
     return cfg.paths.artifacts / "mschema" / "physical.yaml"
 
 
@@ -44,13 +49,13 @@ def annotations_path(cfg) -> Path:
     return cfg.paths.artifacts / "mschema" / "annotations.yaml"
 
 
-def refresh_catalog(cfg, *, dwh=None):
+def refresh_catalog(cfg, *, dwh=None, output_path: Path | None = None):
     """Run the existing catalog algorithm and persist its canonical output."""
     target = dwh if dwh is not None else build_dwh(cfg)
     physical = target.introspect()
     _add_examples(target, physical, cfg.examples)
     classify_all(physical, cfg.eligibility)
-    physical.to_yaml(physical_path(cfg))
+    physical.to_yaml(output_path or (cfg.paths.artifacts / "mschema" / "physical.yaml"))
     return physical
 
 
@@ -90,7 +95,18 @@ def introspect_cmd(
             )
             return
     try:
-        phys = refresh_catalog(cfg)
+        from tht.jobs.dwh_pipeline import active_generation_dir
+
+        if active_generation_dir(cfg.paths.artifacts.parent) is None:
+            phys = refresh_catalog(cfg)
+        else:
+            from tht.cli.preprocess_cmd import run_dwh_from_config
+            from tht.mschema.models import PhysicalSchema
+
+            report = run_dwh_from_config(config, steps=("introspect",))
+            if report.status != "succeeded":
+                raise RuntimeError("DWH preprocessing failed")
+            phys = PhysicalSchema.from_yaml(physical_path(cfg))
     except Exception as e:
         typer.secho(f"ERRORE: {e}", fg=typer.colors.RED, err=True)
         raise typer.Exit(code=1)