fix(preprocess): initialize fresh DWH writers safely
This commit is contained in:
+23
-25
@@ -69,32 +69,30 @@ def build_lsh_artifacts(
|
||||
def build_cmd(config: Path = CONFIG_OPT) -> None:
|
||||
"""Costruisce l'indice LSH dai valori del database e lo salva su pickle."""
|
||||
cfg = _load_config_or_exit(config)
|
||||
phys_file = physical_path(cfg)
|
||||
if not phys_file.exists():
|
||||
typer.secho(
|
||||
f"ERRORE: {phys_file} non trovato. Esegui prima `tht schema introspect`.",
|
||||
fg=typer.colors.RED, err=True,
|
||||
)
|
||||
raise typer.Exit(code=1)
|
||||
typer.echo("Estrazione valori (i più frequenti) dalle colonne testuali eligible...")
|
||||
from tht.jobs.dwh_pipeline import resolve_dwh_snapshot
|
||||
|
||||
if resolve_dwh_snapshot(cfg).generation is None:
|
||||
minhashes, skipped, truncated, values = build_lsh_artifacts(cfg, verbose=True)
|
||||
n_values = sum(len(v) for table in values.values() for v in table.values())
|
||||
n_columns = sum(len(table) for table in values.values())
|
||||
else:
|
||||
from tht.cli.preprocess_cmd import run_dwh_from_config
|
||||
from tht.lshindex import load_index
|
||||
|
||||
report = run_dwh_from_config(config, steps=("lsh",))
|
||||
if report.status != "succeeded":
|
||||
typer.secho("ERRORE: DWH preprocessing failed", fg=typer.colors.RED, err=True)
|
||||
dwh_root = cfg.paths.artifacts.parent / ".tht-dwh"
|
||||
initialized = dwh_root.exists() or dwh_root.is_symlink()
|
||||
if initialized:
|
||||
phys_file = physical_path(cfg)
|
||||
if not phys_file.exists():
|
||||
typer.secho(
|
||||
f"ERRORE: {phys_file} non trovato. Esegui prima `tht schema introspect`.",
|
||||
fg=typer.colors.RED, err=True,
|
||||
)
|
||||
raise typer.Exit(code=1)
|
||||
_, minhashes, _ = load_index(_lsh_dir(cfg), name=cfg.database.db_schema)
|
||||
skipped, truncated = [], []
|
||||
n_values = len(minhashes)
|
||||
n_columns = len({(entry[1], entry[2]) for entry in minhashes.values()})
|
||||
typer.echo("Estrazione valori (i più frequenti) dalle colonne testuali eligible...")
|
||||
from tht.cli.preprocess_cmd import run_dwh_from_config
|
||||
from tht.lshindex import load_index
|
||||
|
||||
report = run_dwh_from_config(
|
||||
config, steps=("lsh",) if initialized else ("introspect", "lsh")
|
||||
)
|
||||
if report.status != "succeeded":
|
||||
typer.secho("ERRORE: DWH preprocessing failed", fg=typer.colors.RED, err=True)
|
||||
raise typer.Exit(code=1)
|
||||
_, minhashes, _ = load_index(_lsh_dir(cfg), name=cfg.database.db_schema)
|
||||
skipped, truncated = [], []
|
||||
n_values = len(minhashes)
|
||||
n_columns = len({(entry[1], entry[2]) for entry in minhashes.values()})
|
||||
typer.echo(f" {n_values} valori da {n_columns} colonne")
|
||||
for s in skipped:
|
||||
typer.secho(f" saltata {s.table}.{s.column}: {s.reason}", fg=typer.colors.YELLOW)
|
||||
|
||||
@@ -72,8 +72,11 @@ def introspect_cmd(
|
||||
Se physical.yaml esiste già, esce subito (cache); usa --refresh per rigenerarlo.
|
||||
"""
|
||||
cfg = _load_config_or_exit(config)
|
||||
out = physical_path(cfg)
|
||||
if out.exists() and not refresh:
|
||||
dwh_root = cfg.paths.artifacts.parent / ".tht-dwh"
|
||||
out = cfg.paths.artifacts / "mschema" / "physical.yaml"
|
||||
if dwh_root.exists() or dwh_root.is_symlink():
|
||||
out = physical_path(cfg)
|
||||
if (dwh_root.exists() or dwh_root.is_symlink()) and out.exists() and not refresh:
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from tht.mschema.models import PhysicalSchema
|
||||
@@ -94,18 +97,14 @@ def introspect_cmd(
|
||||
)
|
||||
return
|
||||
try:
|
||||
from tht.jobs.dwh_pipeline import resolve_dwh_snapshot
|
||||
from tht.cli.preprocess_cmd import run_dwh_from_config
|
||||
from tht.mschema.models import PhysicalSchema
|
||||
|
||||
if resolve_dwh_snapshot(cfg).generation is None:
|
||||
phys = refresh_catalog(cfg)
|
||||
else:
|
||||
from tht.cli.preprocess_cmd import run_dwh_from_config
|
||||
from tht.mschema.models import PhysicalSchema
|
||||
|
||||
report = run_dwh_from_config(config, steps=("introspect",))
|
||||
if report.status != "succeeded":
|
||||
raise RuntimeError("DWH preprocessing failed")
|
||||
phys = PhysicalSchema.from_yaml(physical_path(cfg))
|
||||
report = run_dwh_from_config(config, steps=("introspect",))
|
||||
if report.status != "succeeded":
|
||||
raise RuntimeError("DWH preprocessing failed")
|
||||
out = physical_path(cfg)
|
||||
phys = PhysicalSchema.from_yaml(out)
|
||||
except Exception as e:
|
||||
typer.secho(f"ERRORE: {e}", fg=typer.colors.RED, err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
Reference in New Issue
Block a user