fix(preprocess): bind DWH artifacts to workspace
This commit is contained in:
@@ -20,9 +20,12 @@ def run_dwh_from_config(
|
||||
):
|
||||
from tht.cli.lsh_cmd import build_lsh_artifacts
|
||||
from tht.cli.schema_cmd import _load_config_or_exit, physical_path, refresh_catalog
|
||||
from tht.jobs.dwh_pipeline import DwhPreprocessPipeline, active_generation_dir, fingerprint
|
||||
from tht.jobs.dwh_pipeline import (
|
||||
DwhPreprocessPipeline, active_generation_dir, config_dwh_binding,
|
||||
)
|
||||
|
||||
cfg = _load_config_or_exit(config)
|
||||
binding = config_dwh_binding(cfg)
|
||||
workspace_root = cfg.paths.artifacts.parent
|
||||
active = active_generation_dir(workspace_root)
|
||||
lsh_names = (
|
||||
@@ -31,10 +34,10 @@ def run_dwh_from_config(
|
||||
f"{cfg.database.db_schema}_meta.json",
|
||||
)
|
||||
pipeline = DwhPreprocessPipeline(
|
||||
workspace_id=config.stem.lower().replace(".", "-").replace("_", "-"),
|
||||
workspace_id=binding["workspace_id"],
|
||||
workspace_root=workspace_root,
|
||||
config_fingerprint=fingerprint(cfg.model_dump_json()),
|
||||
input_fingerprint=fingerprint(config.resolve().as_posix()),
|
||||
config_fingerprint=binding["config_fingerprint"],
|
||||
input_fingerprint=binding["input_fingerprint"],
|
||||
introspect=lambda output: refresh_catalog(cfg, output_path=output),
|
||||
build_lsh=lambda physical, output: build_lsh_artifacts(
|
||||
cfg, physical_file=physical, output_dir=output
|
||||
|
||||
Reference in New Issue
Block a user