feat: complete catalog-driven preprocessing

harness/tht/cli/lsh_cmd.py

@@ -3,7 +3,7 @@ from pathlib import Path
 from tht.cli.schema_cmd import physical_path
 
 
-def _extract_lsh_values(dwh, physical, annotations, limit):
+def _extract_lsh_values(dwh, physical, annotations, limit, eligibility_cfg=None):
     from tht.db.sampling import SkippedColumn, TruncatedColumn, is_text_type
     from tht.mschema.eligibility import effective_eligibility
 
@@ -12,7 +12,16 @@ def _extract_lsh_values(dwh, physical, annotations, limit):
         table_ann = annotations.tables.get(table_name)
         for column_name, column in table.columns.items():
             ann_col = table_ann.columns.get(column_name) if table_ann else None
-            if not is_text_type(column.type) or not effective_eligibility(column, ann_col)[0]:
+            from_catalog = column.eligibility_reason in {"catalog", "sensitive"}
+            if not is_text_type(column.type):
+                continue
+            if from_catalog and column.eligibility_reason == "sensitive":
+                continue
+            if from_catalog and eligibility_cfg is not None and (
+                column_name.lower() in {name.lower() for name in eligibility_cfg.ignore_columns}
+            ):
+                continue
+            if not from_catalog and not effective_eligibility(column, ann_col)[0]:
                 continue
             try:
                 distinct = dwh.distinct_values(table_name, column_name, limit=limit)
@@ -20,6 +29,21 @@ def _extract_lsh_values(dwh, physical, annotations, limit):
                 skipped.append(SkippedColumn(table_name, column_name, f"errore: {exc}"))
                 continue
             vals = [str(value) for value in distinct.values if value not in (None, "")]
+            if from_catalog and eligibility_cfg is not None:
+                from tht.mschema.eligibility import classify_column
+
+                lengths = [len(value) for value in vals]
+                eligible, reason = classify_column(
+                    column.type,
+                    column.is_enum,
+                    (sum(lengths) / len(lengths)) if lengths else None,
+                    max(lengths) if lengths else None,
+                    eligibility_cfg,
+                )
+                column.eligible = eligible
+                column.eligibility_reason = reason
+                if not eligible:
+                    continue
             if vals:
                 values.setdefault(table_name, {})[column_name] = vals
             if distinct.truncated:
@@ -33,18 +57,25 @@ def build_lsh_artifacts(
 ):
     """Run the existing LSH extraction/build algorithm and persist its outputs."""
     from tht.adapters.factory import build_dwh
-    from tht.cli.schema_cmd import annotations_path
     from tht.lshindex import build_index, save_index
-    from tht.mschema.models import Annotations, PhysicalSchema
 
-    phys_file = physical_file or physical_path(cfg)
-    if not phys_file.exists():
-        raise FileNotFoundError("physical catalog is missing; run schema introspect first")
-    physical = PhysicalSchema.from_yaml(phys_file)
-    annotations = Annotations.from_yaml(annotations_path(cfg))
+    if physical_file is None and cfg.paths.catalog_metadata_snapshot is not None:
+        from tht.mschema.context import load_schema_context
+
+        context = load_schema_context(cfg)
+        physical, annotations = context.physical, context.annotations
+    else:
+        from tht.cli.schema_cmd import annotations_path
+        from tht.mschema.models import Annotations, PhysicalSchema
+
+        phys_file = physical_file or physical_path(cfg)
+        if not phys_file.exists():
+            raise FileNotFoundError("physical catalog is missing; run schema introspect first")
+        physical = PhysicalSchema.from_yaml(phys_file)
+        annotations = Annotations.from_yaml(annotations_path(cfg))
     target = dwh if dwh is not None else build_dwh(cfg)
     values, skipped, truncated = _extract_lsh_values(
-        target, physical, annotations, cfg.lsh.max_values_per_column
+        target, physical, annotations, cfg.lsh.max_values_per_column, cfg.eligibility
     )
     lsh, minhashes = build_index(values, cfg.lsh, verbose=verbose)
     save_index(