fix(preprocess): restrict DWH root claims to writers
This commit is contained in:
@@ -77,9 +77,9 @@ def build_cmd(config: Path = CONFIG_OPT) -> None:
|
||||
)
|
||||
raise typer.Exit(code=1)
|
||||
typer.echo("Estrazione valori (i più frequenti) dalle colonne testuali eligible...")
|
||||
from tht.jobs.dwh_pipeline import active_generation_dir
|
||||
from tht.jobs.dwh_pipeline import resolve_dwh_snapshot
|
||||
|
||||
if active_generation_dir(cfg.paths.artifacts.parent) is None:
|
||||
if resolve_dwh_snapshot(cfg).generation is None:
|
||||
minhashes, skipped, truncated, values = build_lsh_artifacts(cfg, verbose=True)
|
||||
n_values = sum(len(v) for table in values.values() for v in table.values())
|
||||
n_columns = sum(len(table) for table in values.values())
|
||||
|
||||
@@ -19,15 +19,14 @@ def run_dwh_from_config(
|
||||
config: Path, *, steps: tuple[str, ...], resume: str | None = None,
|
||||
):
|
||||
from tht.cli.lsh_cmd import build_lsh_artifacts
|
||||
from tht.cli.schema_cmd import _load_config_or_exit, physical_path, refresh_catalog
|
||||
from tht.cli.schema_cmd import _load_config_or_exit, refresh_catalog
|
||||
from tht.jobs.dwh_pipeline import (
|
||||
DwhPreprocessPipeline, active_generation_dir, config_dwh_binding,
|
||||
DwhPreprocessPipeline, config_dwh_binding,
|
||||
)
|
||||
|
||||
cfg = _load_config_or_exit(config)
|
||||
binding = config_dwh_binding(cfg)
|
||||
workspace_root = cfg.paths.artifacts.parent
|
||||
active = active_generation_dir(workspace_root)
|
||||
lsh_names = (
|
||||
f"{cfg.database.db_schema}_lsh.pkl",
|
||||
f"{cfg.database.db_schema}_minhashes.pkl",
|
||||
@@ -43,8 +42,8 @@ def run_dwh_from_config(
|
||||
cfg, physical_file=physical, output_dir=output
|
||||
),
|
||||
lsh_filenames=lsh_names,
|
||||
current_physical=physical_path(cfg),
|
||||
current_lsh_dir=active if active is not None else cfg.paths.indexes / "lsh",
|
||||
current_physical=cfg.paths.artifacts / "mschema" / "physical.yaml",
|
||||
current_lsh_dir=cfg.paths.indexes / "lsh",
|
||||
)
|
||||
return pipeline.run(steps, resume_run_id=resume)
|
||||
|
||||
|
||||
@@ -39,6 +39,8 @@ def _load_config_or_exit(config: Path):
|
||||
def physical_path(cfg) -> Path:
|
||||
from tht.jobs.dwh_pipeline import resolve_dwh_snapshot
|
||||
|
||||
if not (cfg.paths.artifacts.parent / ".tht-dwh").exists():
|
||||
return cfg.paths.artifacts / "mschema" / "physical.yaml"
|
||||
return resolve_dwh_snapshot(cfg).physical
|
||||
|
||||
|
||||
@@ -92,9 +94,9 @@ def introspect_cmd(
|
||||
)
|
||||
return
|
||||
try:
|
||||
from tht.jobs.dwh_pipeline import active_generation_dir
|
||||
from tht.jobs.dwh_pipeline import resolve_dwh_snapshot
|
||||
|
||||
if active_generation_dir(cfg.paths.artifacts.parent) is None:
|
||||
if resolve_dwh_snapshot(cfg).generation is None:
|
||||
phys = refresh_catalog(cfg)
|
||||
else:
|
||||
from tht.cli.preprocess_cmd import run_dwh_from_config
|
||||
|
||||
Reference in New Issue
Block a user