fix(preprocess): initialize fresh DWH writers safely

This commit is contained in:
2026-07-12 07:07:31 +02:00
parent 22e806a41b
commit 4cf0adbdd2
5 changed files with 190 additions and 70 deletions
+23 -25
View File
@@ -69,32 +69,30 @@ def build_lsh_artifacts(
def build_cmd(config: Path = CONFIG_OPT) -> None:
"""Costruisce l'indice LSH dai valori del database e lo salva su pickle."""
cfg = _load_config_or_exit(config)
phys_file = physical_path(cfg)
if not phys_file.exists():
typer.secho(
f"ERRORE: {phys_file} non trovato. Esegui prima `tht schema introspect`.",
fg=typer.colors.RED, err=True,
)
raise typer.Exit(code=1)
typer.echo("Estrazione valori (i più frequenti) dalle colonne testuali eligible...")
from tht.jobs.dwh_pipeline import resolve_dwh_snapshot
if resolve_dwh_snapshot(cfg).generation is None:
minhashes, skipped, truncated, values = build_lsh_artifacts(cfg, verbose=True)
n_values = sum(len(v) for table in values.values() for v in table.values())
n_columns = sum(len(table) for table in values.values())
else:
from tht.cli.preprocess_cmd import run_dwh_from_config
from tht.lshindex import load_index
report = run_dwh_from_config(config, steps=("lsh",))
if report.status != "succeeded":
typer.secho("ERRORE: DWH preprocessing failed", fg=typer.colors.RED, err=True)
dwh_root = cfg.paths.artifacts.parent / ".tht-dwh"
initialized = dwh_root.exists() or dwh_root.is_symlink()
if initialized:
phys_file = physical_path(cfg)
if not phys_file.exists():
typer.secho(
f"ERRORE: {phys_file} non trovato. Esegui prima `tht schema introspect`.",
fg=typer.colors.RED, err=True,
)
raise typer.Exit(code=1)
_, minhashes, _ = load_index(_lsh_dir(cfg), name=cfg.database.db_schema)
skipped, truncated = [], []
n_values = len(minhashes)
n_columns = len({(entry[1], entry[2]) for entry in minhashes.values()})
typer.echo("Estrazione valori (i più frequenti) dalle colonne testuali eligible...")
from tht.cli.preprocess_cmd import run_dwh_from_config
from tht.lshindex import load_index
report = run_dwh_from_config(
config, steps=("lsh",) if initialized else ("introspect", "lsh")
)
if report.status != "succeeded":
typer.secho("ERRORE: DWH preprocessing failed", fg=typer.colors.RED, err=True)
raise typer.Exit(code=1)
_, minhashes, _ = load_index(_lsh_dir(cfg), name=cfg.database.db_schema)
skipped, truncated = [], []
n_values = len(minhashes)
n_columns = len({(entry[1], entry[2]) for entry in minhashes.values()})
typer.echo(f" {n_values} valori da {n_columns} colonne")
for s in skipped:
typer.secho(f" saltata {s.table}.{s.column}: {s.reason}", fg=typer.colors.YELLOW)
+12 -13
View File
@@ -72,8 +72,11 @@ def introspect_cmd(
Se physical.yaml esiste già, esce subito (cache); usa --refresh per rigenerarlo.
"""
cfg = _load_config_or_exit(config)
out = physical_path(cfg)
if out.exists() and not refresh:
dwh_root = cfg.paths.artifacts.parent / ".tht-dwh"
out = cfg.paths.artifacts / "mschema" / "physical.yaml"
if dwh_root.exists() or dwh_root.is_symlink():
out = physical_path(cfg)
if (dwh_root.exists() or dwh_root.is_symlink()) and out.exists() and not refresh:
from datetime import UTC, datetime
from tht.mschema.models import PhysicalSchema
@@ -94,18 +97,14 @@ def introspect_cmd(
)
return
try:
from tht.jobs.dwh_pipeline import resolve_dwh_snapshot
from tht.cli.preprocess_cmd import run_dwh_from_config
from tht.mschema.models import PhysicalSchema
if resolve_dwh_snapshot(cfg).generation is None:
phys = refresh_catalog(cfg)
else:
from tht.cli.preprocess_cmd import run_dwh_from_config
from tht.mschema.models import PhysicalSchema
report = run_dwh_from_config(config, steps=("introspect",))
if report.status != "succeeded":
raise RuntimeError("DWH preprocessing failed")
phys = PhysicalSchema.from_yaml(physical_path(cfg))
report = run_dwh_from_config(config, steps=("introspect",))
if report.status != "succeeded":
raise RuntimeError("DWH preprocessing failed")
out = physical_path(cfg)
phys = PhysicalSchema.from_yaml(out)
except Exception as e:
typer.secho(f"ERRORE: {e}", fg=typer.colors.RED, err=True)
raise typer.Exit(code=1)