feat(preprocess): add resumable DWH jobs
This commit is contained in:
@@ -44,6 +44,16 @@ def annotations_path(cfg) -> Path:
|
||||
return cfg.paths.artifacts / "mschema" / "annotations.yaml"
|
||||
|
||||
|
||||
def refresh_catalog(cfg, *, dwh=None):
|
||||
"""Run the existing catalog algorithm and persist its canonical output."""
|
||||
target = dwh if dwh is not None else build_dwh(cfg)
|
||||
physical = target.introspect()
|
||||
_add_examples(target, physical, cfg.examples)
|
||||
classify_all(physical, cfg.eligibility)
|
||||
physical.to_yaml(physical_path(cfg))
|
||||
return physical
|
||||
|
||||
|
||||
@schema_app.command("introspect")
|
||||
def introspect_cmd(
|
||||
config: Path = CONFIG_OPT,
|
||||
@@ -80,14 +90,10 @@ def introspect_cmd(
|
||||
)
|
||||
return
|
||||
try:
|
||||
dwh = build_dwh(cfg)
|
||||
phys = dwh.introspect()
|
||||
_add_examples(dwh, phys, cfg.examples)
|
||||
classify_all(phys, cfg.eligibility)
|
||||
phys = refresh_catalog(cfg)
|
||||
except Exception as e:
|
||||
typer.secho(f"ERRORE: {e}", fg=typer.colors.RED, err=True)
|
||||
raise typer.Exit(code=1)
|
||||
phys.to_yaml(out)
|
||||
n_cols = sum(len(t.columns) for t in phys.tables.values())
|
||||
n_ignored = sum(
|
||||
1 for t in phys.tables.values() for c in t.columns.values() if not c.eligible
|
||||
|
||||
Reference in New Issue
Block a user