fix(preprocess): publish DWH artifacts atomically

This commit is contained in:
2026-07-12 05:43:57 +02:00
parent 4028ef7821
commit 24f6a5daa6
6 changed files with 438 additions and 46 deletions
+15 -4
View File
@@ -19,18 +19,29 @@ def run_dwh_from_config(
config: Path, *, steps: tuple[str, ...], resume: str | None = None,
):
from tht.cli.lsh_cmd import build_lsh_artifacts
from tht.cli.schema_cmd import _load_config_or_exit, refresh_catalog
from tht.jobs.dwh_pipeline import DwhPreprocessPipeline, fingerprint
from tht.cli.schema_cmd import _load_config_or_exit, physical_path, refresh_catalog
from tht.jobs.dwh_pipeline import DwhPreprocessPipeline, active_generation_dir, fingerprint
cfg = _load_config_or_exit(config)
workspace_root = cfg.paths.artifacts.parent
active = active_generation_dir(workspace_root)
lsh_names = (
f"{cfg.database.db_schema}_lsh.pkl",
f"{cfg.database.db_schema}_minhashes.pkl",
f"{cfg.database.db_schema}_meta.json",
)
pipeline = DwhPreprocessPipeline(
workspace_id=config.stem.lower().replace(".", "-").replace("_", "-"),
workspace_root=workspace_root,
config_fingerprint=fingerprint(cfg.model_dump_json()),
input_fingerprint=fingerprint(config.resolve().as_posix()),
introspect=lambda: refresh_catalog(cfg),
build_lsh=lambda: build_lsh_artifacts(cfg),
introspect=lambda output: refresh_catalog(cfg, output_path=output),
build_lsh=lambda physical, output: build_lsh_artifacts(
cfg, physical_file=physical, output_dir=output
),
lsh_filenames=lsh_names,
current_physical=physical_path(cfg),
current_lsh_dir=active if active is not None else cfg.paths.indexes / "lsh",
)
return pipeline.run(steps, resume_run_id=resume)