feat(preprocess): add resumable DWH jobs

This commit is contained in:
2026-07-12 05:34:38 +02:00
parent 2624d1d342
commit 3a9808f126
6 changed files with 323 additions and 20 deletions
+11 -5
View File
@@ -44,6 +44,16 @@ def annotations_path(cfg) -> Path:
return cfg.paths.artifacts / "mschema" / "annotations.yaml"
def refresh_catalog(cfg, *, dwh=None):
"""Run the existing catalog algorithm and persist its canonical output."""
target = dwh if dwh is not None else build_dwh(cfg)
physical = target.introspect()
_add_examples(target, physical, cfg.examples)
classify_all(physical, cfg.eligibility)
physical.to_yaml(physical_path(cfg))
return physical
@schema_app.command("introspect")
def introspect_cmd(
config: Path = CONFIG_OPT,
@@ -80,14 +90,10 @@ def introspect_cmd(
)
return
try:
dwh = build_dwh(cfg)
phys = dwh.introspect()
_add_examples(dwh, phys, cfg.examples)
classify_all(phys, cfg.eligibility)
phys = refresh_catalog(cfg)
except Exception as e:
typer.secho(f"ERRORE: {e}", fg=typer.colors.RED, err=True)
raise typer.Exit(code=1)
phys.to_yaml(out)
n_cols = sum(len(t.columns) for t in phys.tables.values())
n_ignored = sum(
1 for t in phys.tables.values() for c in t.columns.values() if not c.eligible