refactor(core): route integrations through adapter factory
This commit is contained in:
+23
-13
@@ -32,20 +32,30 @@ def build_cmd(config: Path = CONFIG_OPT) -> None:
|
||||
annotations = Annotations.from_yaml(annotations_path(cfg))
|
||||
|
||||
typer.echo("Estrazione valori (i più frequenti) dalle colonne testuali eligible...")
|
||||
if cfg.database.transport == "rest":
|
||||
from tht.db.sampling import unique_values_for_lsh_rest
|
||||
from tht.rest.client import RestClient
|
||||
from tht.adapters.factory import build_dwh
|
||||
from tht.db.sampling import SkippedColumn, TruncatedColumn, is_text_type
|
||||
from tht.mschema.eligibility import effective_eligibility
|
||||
|
||||
values, skipped, truncated = unique_values_for_lsh_rest(
|
||||
RestClient(cfg.rest), physical, cfg.lsh, annotations
|
||||
)
|
||||
else:
|
||||
from tht.db.connection import make_engine
|
||||
from tht.db.sampling import unique_values_for_lsh
|
||||
|
||||
values, skipped, truncated = unique_values_for_lsh(
|
||||
make_engine(cfg.database), physical, cfg.lsh, annotations
|
||||
)
|
||||
dwh = build_dwh(cfg)
|
||||
values: dict[str, dict[str, list[str]]] = {}
|
||||
skipped: list[SkippedColumn] = []
|
||||
truncated: list[TruncatedColumn] = []
|
||||
for table_name, table in physical.tables.items():
|
||||
table_ann = annotations.tables.get(table_name)
|
||||
for column_name, column in table.columns.items():
|
||||
ann_col = table_ann.columns.get(column_name) if table_ann else None
|
||||
if not is_text_type(column.type) or not effective_eligibility(column, ann_col)[0]:
|
||||
continue
|
||||
try:
|
||||
distinct = dwh.distinct_values(table_name, column_name)
|
||||
except Exception as e:
|
||||
skipped.append(SkippedColumn(table_name, column_name, f"errore: {e}"))
|
||||
continue
|
||||
vals = [str(value) for value in distinct.values if value not in (None, "")]
|
||||
if vals:
|
||||
values.setdefault(table_name, {})[column_name] = vals
|
||||
if distinct.truncated or len(vals) >= cfg.lsh.max_values_per_column:
|
||||
truncated.append(TruncatedColumn(table_name, column_name, len(vals)))
|
||||
n_values = sum(len(v) for t in values.values() for v in t.values())
|
||||
typer.echo(f" {n_values} valori da {sum(len(t) for t in values.values())} colonne")
|
||||
for s in skipped:
|
||||
|
||||
Reference in New Issue
Block a user