fix(preprocess): reconcile durable DWH generations
This commit is contained in:
@@ -9,12 +9,9 @@ lsh_app = typer.Typer(help="Indice LSH su valori dei campi (derivato, rigenerabi
|
||||
|
||||
|
||||
def _lsh_dir(cfg) -> Path:
|
||||
from tht.jobs.dwh_pipeline import active_generation_dir
|
||||
from tht.jobs.dwh_pipeline import resolve_dwh_snapshot
|
||||
|
||||
active = active_generation_dir(cfg.paths.artifacts.parent)
|
||||
if active is not None:
|
||||
return active
|
||||
return cfg.paths.indexes / "lsh"
|
||||
return resolve_dwh_snapshot(cfg).lsh_dir
|
||||
|
||||
|
||||
def _extract_lsh_values(dwh, physical, annotations, limit):
|
||||
|
||||
@@ -37,12 +37,9 @@ def _load_config_or_exit(config: Path):
|
||||
|
||||
|
||||
def physical_path(cfg) -> Path:
|
||||
from tht.jobs.dwh_pipeline import active_generation_dir
|
||||
from tht.jobs.dwh_pipeline import resolve_dwh_snapshot
|
||||
|
||||
active = active_generation_dir(cfg.paths.artifacts.parent)
|
||||
if active is not None:
|
||||
return active / "physical.yaml"
|
||||
return cfg.paths.artifacts / "mschema" / "physical.yaml"
|
||||
return resolve_dwh_snapshot(cfg).physical
|
||||
|
||||
|
||||
def annotations_path(cfg) -> Path:
|
||||
|
||||
@@ -47,6 +47,9 @@ def search_cmd(
|
||||
from tht.search import combined_search
|
||||
|
||||
cfg = _load_config_or_exit(config)
|
||||
from tht.jobs.dwh_pipeline import resolve_dwh_snapshot
|
||||
|
||||
dwh_snapshot = resolve_dwh_snapshot(cfg)
|
||||
require_vector_cfg(cfg)
|
||||
from tht.search.evidence import active_searcher
|
||||
|
||||
@@ -88,7 +91,7 @@ def search_cmd(
|
||||
lsh_hits = None
|
||||
try:
|
||||
lsh, minhashes, meta = load_index(
|
||||
cfg.paths.indexes / "lsh", name=cfg.database.db_schema
|
||||
dwh_snapshot.lsh_dir, name=cfg.database.db_schema
|
||||
)
|
||||
hits = query_index(lsh, minhashes, keyword, meta, top_n=top * 3)
|
||||
lsh_hits = [(h.table, h.column, h.value, h.score) for h in hits]
|
||||
@@ -100,12 +103,12 @@ def search_cmd(
|
||||
)
|
||||
|
||||
if kind == "schema":
|
||||
from tht.cli.schema_cmd import annotations_path, physical_path
|
||||
from tht.cli.schema_cmd import annotations_path
|
||||
from tht.mschema.models import Annotations, PhysicalSchema
|
||||
from tht.mschema.render import to_mschema_text
|
||||
from tht.search import schema_tables
|
||||
|
||||
phys_file = physical_path(cfg)
|
||||
phys_file = dwh_snapshot.physical
|
||||
if not phys_file.exists():
|
||||
typer.secho(
|
||||
f"ERRORE: {phys_file} non trovato. Esegui prima `tht schema introspect`.",
|
||||
@@ -241,6 +244,9 @@ def pack_cmd(
|
||||
from tht.vectorstore.rest_client import VectorRestError
|
||||
|
||||
cfg = _load_config_or_exit(config)
|
||||
from tht.jobs.dwh_pipeline import resolve_dwh_snapshot
|
||||
|
||||
dwh_snapshot = resolve_dwh_snapshot(cfg)
|
||||
require_vector_cfg(cfg)
|
||||
|
||||
tables: list[dict] = []
|
||||
@@ -261,10 +267,8 @@ def pack_cmd(
|
||||
warnings.append(f"retrieval non disponibile ({e}): prosegui con le ricerche live")
|
||||
|
||||
if vec is not None:
|
||||
from tht.cli.schema_cmd import physical_path
|
||||
|
||||
descriptions: dict[str, str] = {}
|
||||
phys_file = physical_path(cfg)
|
||||
phys_file = dwh_snapshot.physical
|
||||
if phys_file.exists():
|
||||
from tht.mschema.models import PhysicalSchema
|
||||
|
||||
|
||||
Reference in New Issue
Block a user