mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(preprocess): add resumable DWH jobs

Browse Source
This commit is contained in:
marcopan
2026-07-12 05:34:38 +02:00
parent 2624d1d342
commit 3a9808f126
6 changed files with 323 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/tht/cli/lsh_cmd.py
+22 -15
View File
@@ -36,12 +36,30 @@ def _extract_lsh_values(dwh, physical, annotations, limit):
return values, skipped, truncated
@lsh_app.command("build")
def build_cmd(config: Path = CONFIG_OPT) -> None:
"""Costruisce l'indice LSH dai valori del database e lo salva su pickle."""
def build_lsh_artifacts(cfg, *, dwh=None, verbose: bool = False):
"""Run the existing LSH extraction/build algorithm and persist its outputs."""
from tht.adapters.factory import build_dwh
from tht.cli.schema_cmd import annotations_path
from tht.lshindex import build_index, save_index
from tht.mschema.models import Annotations, PhysicalSchema
phys_file = physical_path(cfg)
if not phys_file.exists():
raise FileNotFoundError("physical catalog is missing; run schema introspect first")
physical = PhysicalSchema.from_yaml(phys_file)
annotations = Annotations.from_yaml(annotations_path(cfg))
target = dwh if dwh is not None else build_dwh(cfg)
values, skipped, truncated = _extract_lsh_values(
target, physical, annotations, cfg.lsh.max_values_per_column
)
lsh, minhashes = build_index(values, cfg.lsh, verbose=verbose)
save_index(lsh, minhashes, cfg.lsh, _lsh_dir(cfg), name=cfg.database.db_schema)
return minhashes, skipped, truncated, values
@lsh_app.command("build")
def build_cmd(config: Path = CONFIG_OPT) -> None:
"""Costruisce l'indice LSH dai valori del database e lo salva su pickle."""
cfg = _load_config_or_exit(config)
phys_file = physical_path(cfg)
if not phys_file.exists():
@@ -50,17 +68,8 @@ def build_cmd(config: Path = CONFIG_OPT) -> None:
fg=typer.colors.RED, err=True,
)
raise typer.Exit(code=1)
physical = PhysicalSchema.from_yaml(phys_file)
from tht.cli.schema_cmd import annotations_path
annotations = Annotations.from_yaml(annotations_path(cfg))
typer.echo("Estrazione valori (i più frequenti) dalle colonne testuali eligible...")
from tht.adapters.factory import build_dwh
dwh = build_dwh(cfg)
values, skipped, truncated = _extract_lsh_values(
dwh, physical, annotations, cfg.lsh.max_values_per_column
)
minhashes, skipped, truncated, values = build_lsh_artifacts(cfg, verbose=True)
n_values = sum(len(v) for t in values.values() for v in t.values())
typer.echo(f" {n_values} valori da {sum(len(t) for t in values.values())} colonne")
for s in skipped:
@@ -72,8 +81,6 @@ def build_cmd(config: Path = CONFIG_OPT) -> None:
fg=typer.colors.YELLOW,
)
lsh, minhashes = build_index(values, cfg.lsh, verbose=True)
save_index(lsh, minhashes, cfg.lsh, _lsh_dir(cfg), name=cfg.database.db_schema)
typer.secho(
f"OK: indice LSH ({len(minhashes)} entry) -> {_lsh_dir(cfg)}", fg=typer.colors.GREEN
)