feat: complete catalog-driven preprocessing

2026-09-06 17:49:35 +02:00
parent 8707ae1d46
commit cffa60772e
141 changed files with 5898 additions and 3015 deletions
@@ -15,7 +15,7 @@ MAX_EXAMPLES_IN_RECORD = 5

 class VectorRecord(BaseModel):
    id: str
-    kind: str  # evidence | schema_table | schema_column
+    kind: str  # evidence | schema_table | schema_column | schema_relationship
    ref: str   # file/chiave canonica di provenienza
    title: str
    content: str
@@ -23,7 +23,7 @@ class VectorRecord(BaseModel):


 def qdrant_semantic_kind(kind: str) -> str:
-    if kind in {"schema_table", "schema_column"}:
+    if kind in {"schema_table", "schema_column", "schema_relationship"}:
        return "schema"
    if kind in {"memory", "solved_question"}:
        return "memory"
@@ -136,3 +136,65 @@ def schema_records(physical: PhysicalSchema, annotations: Annotations) -> list[V
                )
            )
    return records
+
+
+def catalog_schema_records(snapshot) -> list[VectorRecord]:
+    """Build the complete schema slice from a Catalog Metadata Snapshot."""
+    records: list[VectorRecord] = []
+    for table in snapshot.tables:
+        column_names = [column.name for column in table.columns]
+        lines = [f"Tabella {table.name}"]
+        if table.description:
+            lines.append(table.description)
+        lines.append("Colonne: " + ", ".join(column_names))
+        records.append(
+            VectorRecord(
+                id=f"schema_table:{table.id}",
+                kind="schema_table",
+                ref=table.name,
+                title=table.name,
+                content="\n".join(lines),
+                metadata={"tables": [table.name]},
+            )
+        )
+        for column in table.columns:
+            lines = [f"Colonna {table.name}.{column.name}", f"Tipo: {column.data_type}"]
+            if column.description:
+                lines.append(column.description)
+            if column.sensitive:
+                lines.append("Dato sensibile")
+            records.append(
+                VectorRecord(
+                    id=f"schema_column:{column.id}",
+                    kind="schema_column",
+                    ref=f"{table.name}.{column.name}",
+                    title=f"{table.name}.{column.name}",
+                    content="\n".join(lines),
+                    metadata={"tables": [table.name], "sensitive": column.sensitive},
+                )
+            )
+    for relationship in snapshot.relationships:
+        pairs = ", ".join(
+            f"{relationship.source_table}.{source} -> "
+            f"{relationship.target_table}.{target}"
+            for source, target in zip(
+                relationship.source_columns, relationship.target_columns, strict=True
+            )
+        )
+        ref = f"{relationship.source_table}->{relationship.target_table}"
+        records.append(
+            VectorRecord(
+                id=f"schema_relationship:{relationship.id}",
+                kind="schema_relationship",
+                ref=ref,
+                title=ref,
+                content=f"Relazione {relationship.origin}: {pairs}",
+                metadata={
+                    "tables": [relationship.source_table, relationship.target_table],
+                    "origin": relationship.origin,
+                    "source_table": relationship.source_table,
+                    "target_table": relationship.target_table,
+                },
+            )
+        )
+    return records