fix(preprocess): publish DWH artifacts atomically
This commit is contained in:
@@ -37,6 +37,11 @@ def _load_config_or_exit(config: Path):
|
||||
|
||||
|
||||
def physical_path(cfg) -> Path:
|
||||
from tht.jobs.dwh_pipeline import active_generation_dir
|
||||
|
||||
active = active_generation_dir(cfg.paths.artifacts.parent)
|
||||
if active is not None:
|
||||
return active / "physical.yaml"
|
||||
return cfg.paths.artifacts / "mschema" / "physical.yaml"
|
||||
|
||||
|
||||
@@ -44,13 +49,13 @@ def annotations_path(cfg) -> Path:
|
||||
return cfg.paths.artifacts / "mschema" / "annotations.yaml"
|
||||
|
||||
|
||||
def refresh_catalog(cfg, *, dwh=None):
|
||||
def refresh_catalog(cfg, *, dwh=None, output_path: Path | None = None):
|
||||
"""Run the existing catalog algorithm and persist its canonical output."""
|
||||
target = dwh if dwh is not None else build_dwh(cfg)
|
||||
physical = target.introspect()
|
||||
_add_examples(target, physical, cfg.examples)
|
||||
classify_all(physical, cfg.eligibility)
|
||||
physical.to_yaml(physical_path(cfg))
|
||||
physical.to_yaml(output_path or (cfg.paths.artifacts / "mschema" / "physical.yaml"))
|
||||
return physical
|
||||
|
||||
|
||||
@@ -90,7 +95,18 @@ def introspect_cmd(
|
||||
)
|
||||
return
|
||||
try:
|
||||
phys = refresh_catalog(cfg)
|
||||
from tht.jobs.dwh_pipeline import active_generation_dir
|
||||
|
||||
if active_generation_dir(cfg.paths.artifacts.parent) is None:
|
||||
phys = refresh_catalog(cfg)
|
||||
else:
|
||||
from tht.cli.preprocess_cmd import run_dwh_from_config
|
||||
from tht.mschema.models import PhysicalSchema
|
||||
|
||||
report = run_dwh_from_config(config, steps=("introspect",))
|
||||
if report.status != "succeeded":
|
||||
raise RuntimeError("DWH preprocessing failed")
|
||||
phys = PhysicalSchema.from_yaml(physical_path(cfg))
|
||||
except Exception as e:
|
||||
typer.secho(f"ERRORE: {e}", fg=typer.colors.RED, err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
Reference in New Issue
Block a user