test(harness): L0 testcontainers + L1 contract tests for ported db/mschema/rest (A9, spec §1)
Ports the leaf data-layer modules and validates them: - mschema/ (models, eligibility, merge, render), db/ (connection, sampling, introspect, fetch_ca), rest/client.py -- renamed psdwp3->nsp, verbatim. - L0 (testcontainers, real Postgres): db connection read-only enforcement (psd_ro cannot CREATE/INSERT), introspect against a known schema (tables, columns, types, comments, FKs, enum, composite PK), sampling most-frequent values + truncation reporting. 15 tests, ~4s. - L1 (fake data): rest/client RPC contract (mocked transport -- X-API-Key header, payloads, base_url slash handling, HTTP/network error surfacing), mschema/render 3 formats (markdown, mschema-text, schema-dict) + eligibility rules (wide_text excluded, short_text/numeric/enum/temporal/ boolean eligible, annotation override wins). 25 tests. pyproject registers l0/l2 markers + addopts '-m not l2' (L2 opt-in). Deferred to their dependency-porting tasks: test_rrf.py (search needs vectorstore, B3) and the 11 CLI contract tests (need _guards/session, wired when each command lands). 'Not assumed reliable' now has real teeth for the data layer; CLI/search contracts follow.
This commit is contained in:
@@ -0,0 +1,37 @@
|
||||
from sqlalchemy import Engine, create_engine, text
|
||||
|
||||
from nsp.config import DatabaseConfig
|
||||
|
||||
|
||||
def make_engine(cfg: DatabaseConfig) -> Engine:
|
||||
url = (
|
||||
f"postgresql+psycopg2://{cfg.user}:{cfg.password}"
|
||||
f"@{cfg.host}:{cfg.port}/{cfg.database}"
|
||||
)
|
||||
return create_engine(url, echo=False)
|
||||
|
||||
|
||||
def ping(engine: Engine) -> None:
|
||||
with engine.connect() as conn:
|
||||
conn.execute(text("SELECT 1"))
|
||||
|
||||
|
||||
def writable_tables(engine: Engine, schema: str) -> list[str]:
|
||||
"""Tabelle dello schema su cui l'utente corrente ha privilegi di scrittura."""
|
||||
q = text("""
|
||||
SELECT c.relname
|
||||
FROM pg_class c
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = :schema
|
||||
AND c.relkind IN ('r', 'p')
|
||||
AND has_table_privilege(current_user, c.oid, 'INSERT, UPDATE, DELETE')
|
||||
ORDER BY c.relname
|
||||
""")
|
||||
with engine.connect() as conn:
|
||||
return [row[0] for row in conn.execute(q, {"schema": schema})]
|
||||
|
||||
|
||||
def can_create_in_schema(engine: Engine, schema: str) -> bool:
|
||||
q = text("SELECT has_schema_privilege(current_user, :schema, 'CREATE')")
|
||||
with engine.connect() as conn:
|
||||
return bool(conn.execute(q, {"schema": schema}).scalar())
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Recupero della catena di certificati presentata da un endpoint HTTPS.
|
||||
|
||||
Serve al setup di una postazione *workstation* dietro una CA interna: scarica la
|
||||
catena TLS del server REST e la salva in un bundle PEM da puntare con `PSD_SSL_CA`
|
||||
(consumato da `requests` via `verify=`). NON installa nulla nel trust store dell'OS.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import _ssl
|
||||
import socket
|
||||
import ssl
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
# Encoding atteso da Certificate.public_bytes() per la catena TLS non verificata.
|
||||
_PEM_ENCODING = getattr(_ssl, "ENCODING_PEM", 1)
|
||||
|
||||
|
||||
class CaFetchError(Exception):
|
||||
"""Errore azionabile durante il recupero della catena CA."""
|
||||
|
||||
|
||||
def parse_host_port(base_url: str) -> tuple[str, int]:
|
||||
"""Estrae (host, port) da un URL REST https. Porta di default 443."""
|
||||
parts = urlsplit(base_url)
|
||||
if parts.scheme != "https":
|
||||
raise CaFetchError(
|
||||
f"URL non https: {base_url!r}. Il recupero CA ha senso solo su HTTPS."
|
||||
)
|
||||
if not parts.hostname:
|
||||
raise CaFetchError(f"Host mancante nell'URL: {base_url!r}.")
|
||||
return parts.hostname, parts.port or 443
|
||||
|
||||
|
||||
def fetch_chain_pem(host: str, port: int = 443, timeout: int = 30) -> list[str]:
|
||||
"""Restituisce la catena di certificati presentata da host:port come lista di PEM.
|
||||
|
||||
L'handshake è volutamente *non verificato* (CERT_NONE): stiamo recuperando la catena
|
||||
per poter poi *stabilire* la fiducia, non per fidarci adesso. La verifica vera avviene
|
||||
in seguito quando `PSD_SSL_CA` punta al bundle salvato (es. `nsp db ping`).
|
||||
"""
|
||||
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=timeout) as sock:
|
||||
with ctx.wrap_socket(sock, server_hostname=host) as tls:
|
||||
certs = _unverified_chain(tls)
|
||||
except (OSError, ssl.SSLError) as e:
|
||||
raise CaFetchError(
|
||||
f"Impossibile connettersi a {host}:{port} per recuperare i certificati: {e}"
|
||||
) from e
|
||||
|
||||
if not certs:
|
||||
raise CaFetchError(
|
||||
f"Nessun certificato presentato da {host}:{port}. "
|
||||
f"In alternativa, estrai la catena a mano con: "
|
||||
f"openssl s_client -showcerts -connect {host}:{port} -servername {host}"
|
||||
)
|
||||
return [_to_pem(c) for c in certs]
|
||||
|
||||
|
||||
def _unverified_chain(tls: ssl.SSLSocket) -> list:
|
||||
"""Catena presentata dal server. Metodo pubblico su Python >= 3.13, API interna su 3.12."""
|
||||
public = getattr(tls, "get_unverified_chain", None)
|
||||
if public is not None:
|
||||
return list(public() or [])
|
||||
sslobj = getattr(tls, "_sslobj", None)
|
||||
getter = getattr(sslobj, "get_unverified_chain", None) if sslobj is not None else None
|
||||
if getter is None:
|
||||
raise CaFetchError(
|
||||
"Questa versione di Python non espone la catena TLS. "
|
||||
"Estrai la catena a mano con `openssl s_client -showcerts`."
|
||||
)
|
||||
return list(getter() or [])
|
||||
|
||||
|
||||
def describe_pem(pem: str) -> str:
|
||||
"""Riassunto leggibile (subject / issuer) di un certificato PEM, best-effort.
|
||||
|
||||
Serve a far riconoscere all'utente la CA interna attesa (verifica out-of-band).
|
||||
Restituisce "" se il certificato non è decodificabile.
|
||||
"""
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".pem", delete=False) as fh:
|
||||
fh.write(pem)
|
||||
tmp = fh.name
|
||||
try:
|
||||
info = _ssl._test_decode_cert(tmp)
|
||||
finally:
|
||||
Path(tmp).unlink(missing_ok=True)
|
||||
except (OSError, ssl.SSLError, ValueError):
|
||||
return ""
|
||||
subject = _name(info.get("subject"))
|
||||
issuer = _name(info.get("issuer"))
|
||||
return f"subject={subject} issuer={issuer}"
|
||||
|
||||
|
||||
def _name(rdns) -> str:
|
||||
"""Estrae il CN (o l'intero RDN) da una struttura subject/issuer di _test_decode_cert."""
|
||||
if not rdns:
|
||||
return "?"
|
||||
parts = {k: v for rdn in rdns for (k, v) in rdn}
|
||||
return parts.get("commonName") or ", ".join(f"{k}={v}" for k, v in parts.items())
|
||||
|
||||
|
||||
def _to_pem(cert) -> str:
|
||||
"""Converte un certificato (_ssl.Certificate o DER bytes) in PEM."""
|
||||
if isinstance(cert, (bytes, bytearray)):
|
||||
return ssl.DER_cert_to_PEM_cert(bytes(cert))
|
||||
pem = cert.public_bytes(_PEM_ENCODING)
|
||||
return pem if isinstance(pem, str) else pem.decode("ascii")
|
||||
@@ -0,0 +1,202 @@
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Engine, text
|
||||
|
||||
from nsp.mschema.models import (
|
||||
ColumnPhysical,
|
||||
ForeignKey,
|
||||
Index,
|
||||
PhysicalSchema,
|
||||
TablePhysical,
|
||||
)
|
||||
|
||||
# Query adattate da thoth_sqldb2 (Apache 2.0) — vedi src/nsp/vendor/VENDORED.md.
|
||||
|
||||
_TABLES_Q = text("""
|
||||
SELECT c.relname AS table_name,
|
||||
COALESCE(d.description, '') AS comment,
|
||||
GREATEST(c.reltuples::bigint, 0) AS row_count
|
||||
FROM pg_class c
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
LEFT JOIN pg_description d ON d.objoid = c.oid AND d.objsubid = 0
|
||||
WHERE c.relkind IN ('r', 'p') AND n.nspname = :schema
|
||||
ORDER BY c.relname
|
||||
""")
|
||||
|
||||
_COLUMNS_Q = text("""
|
||||
SELECT a.attname AS column_name,
|
||||
format_type(a.atttypid, a.atttypmod) AS data_type,
|
||||
(NOT a.attnotnull) AS is_nullable,
|
||||
pg_get_expr(d.adbin, d.adrelid) AS column_default,
|
||||
COALESCE(pgd.description, '') AS comment,
|
||||
(ty.typtype = 'e') AS is_enum,
|
||||
EXISTS (
|
||||
SELECT 1 FROM pg_index i
|
||||
WHERE i.indrelid = c.oid AND i.indisprimary AND a.attnum = ANY (i.indkey)
|
||||
) AS is_pk
|
||||
FROM pg_class c
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
JOIN pg_attribute a ON a.attrelid = c.oid
|
||||
JOIN pg_type ty ON ty.oid = a.atttypid
|
||||
LEFT JOIN pg_attrdef d ON d.adrelid = c.oid AND d.adnum = a.attnum
|
||||
LEFT JOIN pg_description pgd ON pgd.objoid = c.oid AND pgd.objsubid = a.attnum
|
||||
WHERE c.relname = :table_name AND n.nspname = :schema
|
||||
AND a.attnum > 0 AND NOT a.attisdropped
|
||||
ORDER BY a.attnum
|
||||
""")
|
||||
|
||||
_FOREIGN_KEYS_Q = text("""
|
||||
SELECT con.conname AS constraint_name,
|
||||
rel.relname AS source_table,
|
||||
a.attname AS source_column,
|
||||
frel.relname AS target_table,
|
||||
fa.attname AS target_column,
|
||||
src.ord
|
||||
FROM pg_constraint con
|
||||
JOIN pg_class rel ON rel.oid = con.conrelid
|
||||
JOIN pg_namespace ns ON ns.oid = rel.relnamespace
|
||||
JOIN pg_class frel ON frel.oid = con.confrelid
|
||||
JOIN unnest(con.conkey) WITH ORDINALITY AS src(attnum, ord) ON true
|
||||
JOIN pg_attribute a ON a.attrelid = con.conrelid AND a.attnum = src.attnum
|
||||
JOIN unnest(con.confkey) WITH ORDINALITY AS dst(attnum, ord) ON dst.ord = src.ord
|
||||
JOIN pg_attribute fa ON fa.attrelid = con.confrelid AND fa.attnum = dst.attnum
|
||||
WHERE con.contype = 'f' AND ns.nspname = :schema
|
||||
ORDER BY rel.relname, con.conname, src.ord
|
||||
""")
|
||||
|
||||
_INDEXES_Q = text("""
|
||||
SELECT i.relname AS index_name,
|
||||
t.relname AS table_name,
|
||||
ix.indisunique AS is_unique,
|
||||
ix.indisprimary AS is_primary,
|
||||
am.amname AS index_type,
|
||||
array_agg(a.attname ORDER BY a.attnum) AS columns
|
||||
FROM pg_index ix
|
||||
JOIN pg_class i ON i.oid = ix.indexrelid
|
||||
JOIN pg_class t ON t.oid = ix.indrelid
|
||||
JOIN pg_namespace n ON n.oid = t.relnamespace
|
||||
JOIN pg_am am ON am.oid = i.relam
|
||||
JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY (ix.indkey)
|
||||
WHERE n.nspname = :schema
|
||||
GROUP BY i.relname, t.relname, ix.indisunique, ix.indisprimary, am.amname
|
||||
ORDER BY t.relname, i.relname
|
||||
""")
|
||||
|
||||
|
||||
class IntrospectionError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def introspect(engine: Engine, database: str, schema: str) -> PhysicalSchema:
|
||||
with engine.connect() as conn:
|
||||
exists = conn.execute(
|
||||
text("SELECT 1 FROM pg_namespace WHERE nspname = :schema"), {"schema": schema}
|
||||
).scalar()
|
||||
if not exists:
|
||||
raise IntrospectionError(f"Schema inesistente: {schema}")
|
||||
|
||||
tables: dict[str, TablePhysical] = {}
|
||||
for trow in conn.execute(_TABLES_Q, {"schema": schema}):
|
||||
columns: dict[str, ColumnPhysical] = {}
|
||||
for crow in conn.execute(
|
||||
_COLUMNS_Q, {"table_name": trow.table_name, "schema": schema}
|
||||
):
|
||||
columns[crow.column_name] = ColumnPhysical(
|
||||
type=crow.data_type,
|
||||
nullable=bool(crow.is_nullable),
|
||||
pk=bool(crow.is_pk),
|
||||
default=crow.column_default,
|
||||
comment=crow.comment,
|
||||
is_enum=bool(crow.is_enum),
|
||||
)
|
||||
tables[trow.table_name] = TablePhysical(
|
||||
comment=trow.comment, row_count=trow.row_count, columns=columns
|
||||
)
|
||||
|
||||
# FK raggruppate per (tabella, constraint), ordinate per posizione
|
||||
grouped: dict[tuple[str, str], ForeignKey] = {}
|
||||
for row in conn.execute(_FOREIGN_KEYS_Q, {"schema": schema}):
|
||||
key = (row.source_table, row.constraint_name)
|
||||
fk = grouped.setdefault(
|
||||
key,
|
||||
ForeignKey(
|
||||
columns=[], ref_table=row.target_table, ref_columns=[],
|
||||
name=row.constraint_name,
|
||||
),
|
||||
)
|
||||
fk.columns.append(row.source_column)
|
||||
fk.ref_columns.append(row.target_column)
|
||||
for (table_name, _), fk in grouped.items():
|
||||
if table_name in tables:
|
||||
tables[table_name].foreign_keys.append(fk)
|
||||
|
||||
for row in conn.execute(_INDEXES_Q, {"schema": schema}):
|
||||
if row.table_name in tables:
|
||||
tables[row.table_name].indexes.append(
|
||||
Index(
|
||||
name=row.index_name,
|
||||
columns=list(row.columns),
|
||||
unique=bool(row.is_unique),
|
||||
primary=bool(row.is_primary),
|
||||
type=row.index_type,
|
||||
)
|
||||
)
|
||||
|
||||
return PhysicalSchema(
|
||||
database=database,
|
||||
schema=schema,
|
||||
introspected_at=datetime.now(UTC),
|
||||
tables=tables,
|
||||
)
|
||||
|
||||
|
||||
def introspect_rest(client, database: str, schema: str) -> PhysicalSchema:
|
||||
"""Introspezione via REST (rpc `list_tables`/`table_columns`/`table_comments`/
|
||||
`table_foreign_keys`). Limiti rispetto al diretto: niente indici (nessun rpc) e
|
||||
`is_enum` non disponibile (default False)."""
|
||||
tables: dict[str, TablePhysical] = {}
|
||||
for trow in client.list_tables(schema):
|
||||
if trow.get("type") != "TABLE":
|
||||
continue # le viste sono fuori scope (come l'introspezione diretta)
|
||||
table_name = trow["table"]
|
||||
|
||||
col_comments = {
|
||||
c["name"]: (c.get("comment") or "")
|
||||
for c in client.table_comments(schema, table_name)
|
||||
if c.get("object") == "COLUMN"
|
||||
}
|
||||
columns: dict[str, ColumnPhysical] = {}
|
||||
for crow in client.table_columns(schema, table_name):
|
||||
name = crow["column"]
|
||||
columns[name] = ColumnPhysical(
|
||||
type=crow["type"],
|
||||
nullable=bool(crow.get("nullable", True)),
|
||||
pk=bool(crow.get("pk", False)),
|
||||
default=crow.get("default"),
|
||||
comment=col_comments.get(name, ""),
|
||||
)
|
||||
|
||||
foreign_keys: list[ForeignKey] = []
|
||||
for fk in client.table_foreign_keys(schema, table_name):
|
||||
foreign_keys.append(
|
||||
ForeignKey(
|
||||
columns=fk.get("columns") or [fk["column"]],
|
||||
ref_table=fk.get("ref_table") or fk["target_table"],
|
||||
ref_columns=fk.get("ref_columns") or [fk["target_column"]],
|
||||
name=fk.get("name", ""),
|
||||
)
|
||||
)
|
||||
|
||||
tables[table_name] = TablePhysical(
|
||||
comment=trow.get("comment") or "",
|
||||
row_count=int(trow.get("rows") or 0),
|
||||
columns=columns,
|
||||
foreign_keys=foreign_keys,
|
||||
)
|
||||
|
||||
return PhysicalSchema(
|
||||
database=database,
|
||||
schema=schema,
|
||||
introspected_at=datetime.now(UTC),
|
||||
tables=tables,
|
||||
)
|
||||
@@ -0,0 +1,160 @@
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import Engine, text
|
||||
|
||||
from nsp.config import ExamplesConfig, LshConfig
|
||||
from nsp.mschema.models import Annotations, PhysicalSchema
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TEXT_TYPE_PREFIXES = ("text", "varchar", "character", "char")
|
||||
|
||||
|
||||
def is_text_type(pg_type: str) -> bool:
|
||||
return pg_type.lower().startswith(TEXT_TYPE_PREFIXES)
|
||||
|
||||
|
||||
def add_examples(engine: Engine, physical: PhysicalSchema, cfg: ExamplesConfig) -> None:
|
||||
"""Campiona i valori distinti piu' frequenti delle colonne testuali (in-place)."""
|
||||
schema = physical.db_schema
|
||||
with engine.connect() as conn:
|
||||
for table_name, table in physical.tables.items():
|
||||
for column_name, column in table.columns.items():
|
||||
if not is_text_type(column.type):
|
||||
continue
|
||||
q = text(f'''
|
||||
SELECT "{column_name}" FROM (
|
||||
SELECT "{column_name}", count(*) AS _freq
|
||||
FROM "{schema}"."{table_name}"
|
||||
WHERE "{column_name}" IS NOT NULL AND length("{column_name}") > 0
|
||||
GROUP BY "{column_name}"
|
||||
ORDER BY _freq DESC
|
||||
LIMIT :lim
|
||||
) AS sub
|
||||
''')
|
||||
try:
|
||||
rows = conn.execute(q, {"lim": cfg.max_per_column}).fetchall()
|
||||
except Exception as e: # colonna non leggibile: si salta, non si interrompe
|
||||
logger.warning("Campionamento saltato per %s.%s: %s", table_name, column_name, e)
|
||||
continue
|
||||
column.examples = [str(r[0]) for r in rows]
|
||||
|
||||
|
||||
def add_examples_rest(client, physical: PhysicalSchema, cfg: ExamplesConfig) -> None:
|
||||
"""Variante REST di add_examples: valori più frequenti via rpc `top_values`."""
|
||||
schema = physical.db_schema
|
||||
for table_name, table in physical.tables.items():
|
||||
for column_name, column in table.columns.items():
|
||||
if not is_text_type(column.type):
|
||||
continue
|
||||
rows = client.top_values(schema, table_name, column_name, cfg.max_per_column)
|
||||
column.examples = [str(r["value"]) for r in rows if r["value"] not in (None, "")]
|
||||
|
||||
|
||||
@dataclass
|
||||
class SkippedColumn:
|
||||
table: str
|
||||
column: str
|
||||
reason: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class TruncatedColumn:
|
||||
table: str
|
||||
column: str
|
||||
indexed: int # quanti valori (i più frequenti) sono stati indicizzati
|
||||
|
||||
|
||||
def unique_values_for_lsh(
|
||||
engine: Engine,
|
||||
physical: PhysicalSchema,
|
||||
cfg: LshConfig,
|
||||
annotations: Annotations | None = None,
|
||||
) -> tuple[dict[str, dict[str, list[str]]], list[SkippedColumn], list[TruncatedColumn]]:
|
||||
"""Valori delle colonne testuali *eligible* per l'indice LSH.
|
||||
|
||||
Indicizza solo colonne con eligibilità effettiva True (le `wide_text` sono escluse:
|
||||
vedi principio di column eligibility). Estrae i valori distinti *più frequenti*
|
||||
(ORDER BY frequenza); se superano `max_values_per_column` la colonna è troncata e
|
||||
segnalata (mai tagliata in silenzio).
|
||||
"""
|
||||
from nsp.mschema.eligibility import effective_eligibility
|
||||
|
||||
annotations = annotations or Annotations()
|
||||
schema = physical.db_schema
|
||||
values: dict[str, dict[str, list[str]]] = {}
|
||||
skipped: list[SkippedColumn] = []
|
||||
truncated: list[TruncatedColumn] = []
|
||||
with engine.connect() as conn:
|
||||
for table_name, table in physical.tables.items():
|
||||
table_ann = annotations.tables.get(table_name)
|
||||
for column_name, column in table.columns.items():
|
||||
if not is_text_type(column.type):
|
||||
continue
|
||||
ann_col = table_ann.columns.get(column_name) if table_ann else None
|
||||
if not effective_eligibility(column, ann_col)[0]:
|
||||
continue
|
||||
q = text(f'''
|
||||
SELECT "{column_name}" FROM (
|
||||
SELECT "{column_name}", count(*) AS _freq
|
||||
FROM "{schema}"."{table_name}"
|
||||
WHERE "{column_name}" IS NOT NULL AND length("{column_name}") > 0
|
||||
GROUP BY "{column_name}"
|
||||
ORDER BY _freq DESC, "{column_name}"
|
||||
LIMIT :lim
|
||||
) AS sub
|
||||
''')
|
||||
try:
|
||||
rows = conn.execute(q, {"lim": cfg.max_values_per_column}).fetchall()
|
||||
except Exception as e:
|
||||
skipped.append(SkippedColumn(table_name, column_name, f"errore: {e}"))
|
||||
continue
|
||||
vals = [str(r[0]) for r in rows]
|
||||
if not vals:
|
||||
continue
|
||||
values.setdefault(table_name, {})[column_name] = vals
|
||||
if len(vals) >= cfg.max_values_per_column:
|
||||
truncated.append(TruncatedColumn(table_name, column_name, len(vals)))
|
||||
return values, skipped, truncated
|
||||
|
||||
|
||||
def unique_values_for_lsh_rest(
|
||||
client,
|
||||
physical: PhysicalSchema,
|
||||
cfg: LshConfig,
|
||||
annotations: Annotations | None = None,
|
||||
) -> tuple[dict[str, dict[str, list[str]]], list[SkippedColumn], list[TruncatedColumn]]:
|
||||
"""Variante REST di unique_values_for_lsh: valori più frequenti via rpc `top_values`.
|
||||
|
||||
Stessa logica di eligibility e di segnalazione del troncamento del transport diretto.
|
||||
"""
|
||||
from nsp.mschema.eligibility import effective_eligibility
|
||||
|
||||
annotations = annotations or Annotations()
|
||||
schema = physical.db_schema
|
||||
values: dict[str, dict[str, list[str]]] = {}
|
||||
skipped: list[SkippedColumn] = []
|
||||
truncated: list[TruncatedColumn] = []
|
||||
for table_name, table in physical.tables.items():
|
||||
table_ann = annotations.tables.get(table_name)
|
||||
for column_name, column in table.columns.items():
|
||||
if not is_text_type(column.type):
|
||||
continue
|
||||
ann_col = table_ann.columns.get(column_name) if table_ann else None
|
||||
if not effective_eligibility(column, ann_col)[0]:
|
||||
continue
|
||||
try:
|
||||
rows = client.top_values(
|
||||
schema, table_name, column_name, cfg.max_values_per_column
|
||||
)
|
||||
except Exception as e:
|
||||
skipped.append(SkippedColumn(table_name, column_name, f"errore: {e}"))
|
||||
continue
|
||||
vals = [str(r["value"]) for r in rows if r["value"] not in (None, "")]
|
||||
if not vals:
|
||||
continue
|
||||
values.setdefault(table_name, {})[column_name] = vals
|
||||
if len(vals) >= cfg.max_values_per_column:
|
||||
truncated.append(TruncatedColumn(table_name, column_name, len(vals)))
|
||||
return values, skipped, truncated
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Classificazione di column eligibility (principio trasversale PsdWp3).
|
||||
|
||||
Vedi docs/superpowers/specs/2026-06-13-nsp-column-eligibility-principle.md.
|
||||
Il testo ampio (lettere di dimissione, note, anamnesi) è ignorato ovunque; i dati
|
||||
provengono solo da numerici, enum, temporali, booleani e testo breve.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
from nsp.config import EligibilityConfig
|
||||
from nsp.db.sampling import is_text_type
|
||||
from nsp.mschema.models import ColumnAnnotation, ColumnPhysical, PhysicalSchema
|
||||
|
||||
_NUMERIC_PREFIXES = (
|
||||
"smallint", "integer", "bigint", "numeric", "decimal", "real", "double", "money",
|
||||
)
|
||||
_TEMPORAL_PREFIXES = ("date", "time", "timestamp", "interval")
|
||||
_LEN_RE = re.compile(r"\((\d+)\)")
|
||||
|
||||
|
||||
def _declared_len(pg_type: str) -> int | None:
|
||||
m = _LEN_RE.search(pg_type)
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
|
||||
def classify_column(
|
||||
pg_type: str,
|
||||
is_enum: bool,
|
||||
sampled_avg: float | None,
|
||||
sampled_max: int | None,
|
||||
cfg: EligibilityConfig,
|
||||
) -> tuple[bool, str]:
|
||||
"""Classifica una colonna come (eligible, reason). Funzione pura, senza I/O."""
|
||||
t = pg_type.strip().lower()
|
||||
if t.endswith("[]"):
|
||||
return False, "wide_text"
|
||||
if is_enum:
|
||||
return True, "enum"
|
||||
if t.startswith(_NUMERIC_PREFIXES):
|
||||
return True, "numeric"
|
||||
if t.startswith("boolean"):
|
||||
return True, "boolean"
|
||||
if t.startswith(_TEMPORAL_PREFIXES):
|
||||
return True, "temporal"
|
||||
if t.startswith("uuid"):
|
||||
return True, "code"
|
||||
if is_text_type(t):
|
||||
declared = _declared_len(t)
|
||||
if declared is not None and declared <= cfg.max_declared_len:
|
||||
return True, "short_text"
|
||||
# bound grande o text/varchar non vincolato: decide il dato campionato
|
||||
if sampled_avg is None or sampled_max is None:
|
||||
return False, "wide_text"
|
||||
if sampled_avg <= cfg.max_avg_length and sampled_max <= cfg.max_sampled_len:
|
||||
return True, "short_text"
|
||||
return False, "wide_text"
|
||||
return False, "wide_text"
|
||||
|
||||
|
||||
def _sampled_stats(examples: list[str]) -> tuple[float | None, int | None]:
|
||||
if not examples:
|
||||
return None, None
|
||||
lengths = [len(v) for v in examples]
|
||||
return sum(lengths) / len(lengths), max(lengths)
|
||||
|
||||
|
||||
def classify_all(physical: PhysicalSchema, cfg: EligibilityConfig) -> None:
|
||||
"""Assegna eligible/eligibility_reason a ogni colonna (in-place) e azzera gli
|
||||
examples delle colonne ignored. Da chiamare DOPO add_examples."""
|
||||
ignore_by_name = {name.lower() for name in cfg.ignore_columns}
|
||||
for table in physical.tables.values():
|
||||
for column_name, column in table.columns.items():
|
||||
if column_name.lower() in ignore_by_name:
|
||||
# colonna di servizio (ETL/audit): ignorata a prescindere dal tipo
|
||||
column.eligible = False
|
||||
column.eligibility_reason = "ignored_by_name"
|
||||
column.examples = []
|
||||
continue
|
||||
avg, mx = _sampled_stats(column.examples)
|
||||
eligible, reason = classify_column(column.type, column.is_enum, avg, mx, cfg)
|
||||
column.eligible = eligible
|
||||
column.eligibility_reason = reason
|
||||
if not eligible:
|
||||
column.examples = []
|
||||
|
||||
|
||||
def effective_eligibility(
|
||||
column: ColumnPhysical, annotation: ColumnAnnotation | None
|
||||
) -> tuple[bool, str]:
|
||||
"""Eligibilità effettiva: l'override in annotations.yaml vince sul fisico."""
|
||||
if annotation is not None and annotation.eligible is not None:
|
||||
return annotation.eligible, "override"
|
||||
return column.eligible, column.eligibility_reason
|
||||
@@ -0,0 +1,15 @@
|
||||
from nsp.mschema.models import Annotations, PhysicalSchema
|
||||
|
||||
|
||||
def find_orphans(physical: PhysicalSchema, annotations: Annotations) -> list[str]:
|
||||
"""Annotazioni che puntano a oggetti spariti dal fisico. Non le rimuove mai."""
|
||||
orphans: list[str] = []
|
||||
for table_name, table_ann in annotations.tables.items():
|
||||
table = physical.tables.get(table_name)
|
||||
if table is None:
|
||||
orphans.append(table_name)
|
||||
continue
|
||||
for column_name in table_ann.columns:
|
||||
if column_name not in table.columns:
|
||||
orphans.append(f"{table_name}.{column_name}")
|
||||
return orphans
|
||||
@@ -0,0 +1,90 @@
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Self
|
||||
|
||||
import yaml
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class _YamlModel(BaseModel):
|
||||
def to_yaml(self, path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = self.model_dump(by_alias=True, mode="json", exclude_defaults=False)
|
||||
path.write_text(
|
||||
yaml.safe_dump(data, sort_keys=False, allow_unicode=True, width=120)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_yaml(cls, path: Path) -> Self:
|
||||
raw = yaml.safe_load(path.read_text())
|
||||
return cls.model_validate(raw)
|
||||
|
||||
|
||||
class ColumnPhysical(BaseModel):
|
||||
type: str
|
||||
nullable: bool = True
|
||||
pk: bool = False
|
||||
default: str | None = None
|
||||
comment: str = ""
|
||||
examples: list[str] = []
|
||||
is_enum: bool = False
|
||||
eligible: bool = True
|
||||
eligibility_reason: str = ""
|
||||
|
||||
|
||||
class ForeignKey(BaseModel):
|
||||
columns: list[str]
|
||||
ref_table: str
|
||||
ref_columns: list[str]
|
||||
name: str = ""
|
||||
|
||||
|
||||
class Index(BaseModel):
|
||||
name: str
|
||||
columns: list[str]
|
||||
unique: bool = False
|
||||
primary: bool = False
|
||||
type: str = "btree"
|
||||
|
||||
|
||||
class TablePhysical(BaseModel):
|
||||
comment: str = ""
|
||||
row_count: int = 0 # stima da pg_class.reltuples
|
||||
columns: dict[str, ColumnPhysical] = {}
|
||||
foreign_keys: list[ForeignKey] = []
|
||||
indexes: list[Index] = []
|
||||
|
||||
|
||||
class PhysicalSchema(_YamlModel):
|
||||
database: str
|
||||
db_schema: str = Field(alias="schema")
|
||||
introspected_at: datetime
|
||||
tables: dict[str, TablePhysical] = {}
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
|
||||
class ColumnAnnotation(BaseModel):
|
||||
description: str = ""
|
||||
synonyms: list[str] = []
|
||||
concepts: list[str] = []
|
||||
evidence: list[str] = []
|
||||
notes: str = ""
|
||||
eligible: bool | None = None
|
||||
|
||||
|
||||
class TableAnnotation(BaseModel):
|
||||
description: str = ""
|
||||
concepts: list[str] = []
|
||||
notes: str = ""
|
||||
columns: dict[str, ColumnAnnotation] = {}
|
||||
|
||||
|
||||
class Annotations(_YamlModel):
|
||||
tables: dict[str, TableAnnotation] = {}
|
||||
|
||||
@classmethod
|
||||
def from_yaml(cls, path: Path) -> "Annotations":
|
||||
if not path.exists():
|
||||
return cls()
|
||||
return super().from_yaml(path)
|
||||
@@ -0,0 +1,141 @@
|
||||
from typing import Any
|
||||
|
||||
from nsp.mschema.eligibility import effective_eligibility
|
||||
from nsp.mschema.models import Annotations, ColumnAnnotation, PhysicalSchema
|
||||
|
||||
MAX_EXAMPLES_IN_PROMPT = 5
|
||||
|
||||
|
||||
def _ann_col(annotations: Annotations, table: str, column: str) -> ColumnAnnotation | None:
|
||||
ann = annotations.tables.get(table)
|
||||
if ann is None:
|
||||
return None
|
||||
return ann.columns.get(column)
|
||||
|
||||
|
||||
def _table_description(physical: PhysicalSchema, annotations: Annotations, table: str) -> str:
|
||||
ann = annotations.tables.get(table)
|
||||
if ann and ann.description:
|
||||
return ann.description
|
||||
return physical.tables[table].comment
|
||||
|
||||
|
||||
def _column_description(
|
||||
physical: PhysicalSchema, annotations: Annotations, table: str, column: str
|
||||
) -> str:
|
||||
ann = annotations.tables.get(table)
|
||||
if ann and column in ann.columns and ann.columns[column].description:
|
||||
return ann.columns[column].description
|
||||
return physical.tables[table].columns[column].comment
|
||||
|
||||
|
||||
def to_mschema_text(
|
||||
physical: PhysicalSchema,
|
||||
annotations: Annotations | None = None,
|
||||
tables: list[str] | None = None,
|
||||
) -> str:
|
||||
"""Serializzazione testuale in stile ThothAI (【Schema】/【Foreign keys】)."""
|
||||
annotations = annotations or Annotations()
|
||||
selected = [t for t in physical.tables if tables is None or t in tables]
|
||||
lines: list[str] = ["【Schema】"]
|
||||
fk_lines: list[str] = []
|
||||
for table_name in selected:
|
||||
table = physical.tables[table_name]
|
||||
desc = _table_description(physical, annotations, table_name)
|
||||
if desc:
|
||||
lines.append(f"-- {desc}")
|
||||
lines.append(f"CREATE TABLE {table_name} (")
|
||||
for column_name, column in table.columns.items():
|
||||
if not effective_eligibility(column, _ann_col(annotations, table_name, column_name))[0]:
|
||||
continue
|
||||
line = f" {column_name} {column.type.upper()}"
|
||||
if column.pk:
|
||||
line += " -- PRIMARY KEY"
|
||||
lines.append(line)
|
||||
cdesc = _column_description(physical, annotations, table_name, column_name)
|
||||
if cdesc:
|
||||
lines.append(f" -- {cdesc}")
|
||||
if column.examples:
|
||||
shown = ", ".join(column.examples[:MAX_EXAMPLES_IN_PROMPT])
|
||||
lines.append(f" -- Examples: {shown}")
|
||||
lines.append(");")
|
||||
for fk in table.foreign_keys:
|
||||
for src, dst in zip(fk.columns, fk.ref_columns):
|
||||
fk_lines.append(f"{table_name}.{src}={fk.ref_table}.{dst}")
|
||||
lines.extend(["", "【Foreign keys】", *fk_lines])
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def to_schema_dict(
|
||||
physical: PhysicalSchema, annotations: Annotations | None = None
|
||||
) -> dict[str, Any]:
|
||||
"""Vista compatibile con le logiche AV-SQL (schema_dict)."""
|
||||
annotations = annotations or Annotations()
|
||||
out: dict[str, Any] = {}
|
||||
for table_name, table in physical.tables.items():
|
||||
cols = [
|
||||
c
|
||||
for c in table.columns
|
||||
if effective_eligibility(table.columns[c], _ann_col(annotations, table_name, c))[0]
|
||||
]
|
||||
out[table_name] = {
|
||||
"columns_name": cols,
|
||||
"columns_type": [table.columns[c].type for c in cols],
|
||||
"columns_description": [
|
||||
_column_description(physical, annotations, table_name, c) for c in cols
|
||||
],
|
||||
"example_values": [table.columns[c].examples for c in cols],
|
||||
"table_to_tablefullname": f"{physical.db_schema}.{table_name}",
|
||||
"primary_keys": [c for c in cols if table.columns[c].pk],
|
||||
"foreign_keys": [
|
||||
{"columns": fk.columns, "ref_table": fk.ref_table, "ref_columns": fk.ref_columns}
|
||||
for fk in table.foreign_keys
|
||||
],
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
def to_markdown(physical: PhysicalSchema, annotations: Annotations | None = None) -> str:
|
||||
"""Report leggibile per il reviewer."""
|
||||
annotations = annotations or Annotations()
|
||||
lines = [
|
||||
f"# Schema {physical.db_schema} ({physical.database})",
|
||||
"",
|
||||
f"Introspezione: {physical.introspected_at.isoformat()} — "
|
||||
f"{len(physical.tables)} tabelle",
|
||||
]
|
||||
for table_name, table in physical.tables.items():
|
||||
lines += ["", f"## {table_name}", ""]
|
||||
desc = _table_description(physical, annotations, table_name)
|
||||
if desc:
|
||||
lines += [desc, ""]
|
||||
lines += [
|
||||
f"Righe (stima): {table.row_count}",
|
||||
"",
|
||||
"| Colonna | Tipo | Null | PK | Descrizione | Esempi |",
|
||||
"|---|---|---|---|---|---|",
|
||||
]
|
||||
for column_name, column in table.columns.items():
|
||||
eligible, reason = effective_eligibility(
|
||||
column, _ann_col(annotations, table_name, column_name)
|
||||
)
|
||||
cdesc = _column_description(physical, annotations, table_name, column_name)
|
||||
if not eligible:
|
||||
lines.append(
|
||||
f"| ~~{column_name}~~ | {column.type} | "
|
||||
f"{'sì' if column.nullable else 'no'} | "
|
||||
f"{'sì' if column.pk else ''} | {cdesc} | _ignorata: {reason}_ |"
|
||||
)
|
||||
continue
|
||||
examples = ", ".join(column.examples[:3])
|
||||
lines.append(
|
||||
f"| {column_name} | {column.type} | {'sì' if column.nullable else 'no'} "
|
||||
f"| {'sì' if column.pk else ''} | {cdesc} | {examples} |"
|
||||
)
|
||||
if table.foreign_keys:
|
||||
lines += ["", "Foreign keys:"]
|
||||
for fk in table.foreign_keys:
|
||||
lines.append(
|
||||
f"- ({', '.join(fk.columns)}) → {fk.ref_table} ({', '.join(fk.ref_columns)})"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Client per il DWH Chirone esposto via Supabase/PostgREST.
|
||||
|
||||
Incapsula i 10 rpc verificati contro produzione. Errori sempre in italiano e azionabili
|
||||
(stile `vectorstore/embeddings.py`). Lettura sola: l'API ammette solo SELECT/WITH.
|
||||
"""
|
||||
|
||||
import requests
|
||||
|
||||
from nsp.config import RestConfig
|
||||
|
||||
|
||||
class RestError(Exception):
|
||||
"""Errore di accesso al DWH via REST, con messaggio leggibile per il reviewer."""
|
||||
|
||||
|
||||
class RestClient:
|
||||
def __init__(self, cfg: RestConfig):
|
||||
self.cfg = cfg
|
||||
self._base = cfg.base_url.rstrip("/")
|
||||
|
||||
# -- trasporto -----------------------------------------------------------------
|
||||
|
||||
def _post(self, fn: str, args: dict) -> requests.Response:
|
||||
url = f"{self._base}/rpc/{fn}"
|
||||
verify: bool | str = self.cfg.ssl_ca if self.cfg.ssl_ca else True
|
||||
try:
|
||||
return requests.post(
|
||||
url,
|
||||
json=args,
|
||||
headers={"X-API-Key": self.cfg.api_key},
|
||||
timeout=self.cfg.timeout,
|
||||
verify=verify,
|
||||
)
|
||||
except requests.RequestException as e:
|
||||
raise RestError(
|
||||
f"DWH REST non raggiungibile su {self.cfg.base_url} (rpc {fn}): {e}"
|
||||
) from e
|
||||
|
||||
def _error_msg(self, fn: str, resp: requests.Response) -> str:
|
||||
try:
|
||||
body = resp.json()
|
||||
detail = body.get("message") or body.get("details") or resp.text
|
||||
except Exception:
|
||||
detail = resp.text
|
||||
return f"DWH REST rpc {fn} → HTTP {resp.status_code}: {detail}"
|
||||
|
||||
def _call(self, fn: str, args: dict):
|
||||
resp = self._post(fn, args)
|
||||
if not resp.ok:
|
||||
raise RestError(self._error_msg(fn, resp))
|
||||
if resp.status_code == 204 or not resp.text:
|
||||
return None
|
||||
return resp.json()
|
||||
|
||||
# -- rpc -----------------------------------------------------------------------
|
||||
|
||||
def ping(self) -> dict:
|
||||
return self._call("ping", {})
|
||||
|
||||
def run_query(self, query_text: str) -> list[dict]:
|
||||
"""Esegue una SELECT/WITH e restituisce le righe (read-only lato server)."""
|
||||
return self._call("run_query", {"query_text": query_text}) or []
|
||||
|
||||
def explain_query(self, query_text: str) -> list[str]:
|
||||
"""EXPLAIN di una SELECT/WITH: ritorna le righe testuali del piano."""
|
||||
rows = self._call("explain_query", {"query_text": query_text}) or []
|
||||
return [r["line"] for r in rows]
|
||||
|
||||
def validate_select(self, query_text: str) -> bool:
|
||||
"""True se la query è una SELECT/WITH accettata, False se è una write."""
|
||||
resp = self._post("_validate_select", {"query_text": query_text})
|
||||
if resp.status_code == 204:
|
||||
return True
|
||||
if resp.status_code == 400:
|
||||
return False
|
||||
raise RestError(self._error_msg("_validate_select", resp))
|
||||
|
||||
def list_tables(self, schema_name: str) -> list[dict]:
|
||||
return self._call("list_tables", {"schema_name": schema_name}) or []
|
||||
|
||||
def table_columns(self, schema_name: str, table_name: str) -> list[dict]:
|
||||
return self._call(
|
||||
"table_columns", {"schema_name": schema_name, "table_name": table_name}
|
||||
) or []
|
||||
|
||||
def table_comments(self, schema_name: str, table_name: str) -> list[dict]:
|
||||
return self._call(
|
||||
"table_comments", {"schema_name": schema_name, "table_name": table_name}
|
||||
) or []
|
||||
|
||||
def table_foreign_keys(self, schema_name: str, table_name: str) -> list[dict]:
|
||||
return self._call(
|
||||
"table_foreign_keys", {"schema_name": schema_name, "table_name": table_name}
|
||||
) or []
|
||||
|
||||
def top_values(
|
||||
self, schema_name: str, table_name: str, column_name: str, max_values: int
|
||||
) -> list[dict]:
|
||||
return self._call(
|
||||
"top_values",
|
||||
{
|
||||
"schema_name": schema_name,
|
||||
"table_name": table_name,
|
||||
"column_name": column_name,
|
||||
"max_values": max_values,
|
||||
},
|
||||
) or []
|
||||
|
||||
def column_stats(self, schema_name: str, table_name: str, column_name: str) -> dict:
|
||||
return self._call(
|
||||
"column_stats",
|
||||
{"schema_name": schema_name, "table_name": table_name, "column_name": column_name},
|
||||
)
|
||||
Reference in New Issue
Block a user