marcopan
eb3bde90e2
Ports the leaf data-layer modules and validates them: - mschema/ (models, eligibility, merge, render), db/ (connection, sampling, introspect, fetch_ca), rest/client.py -- renamed psdwp3->nsp, verbatim. - L0 (testcontainers, real Postgres): db connection read-only enforcement (psd_ro cannot CREATE/INSERT), introspect against a known schema (tables, columns, types, comments, FKs, enum, composite PK), sampling most-frequent values + truncation reporting. 15 tests, ~4s. - L1 (fake data): rest/client RPC contract (mocked transport -- X-API-Key header, payloads, base_url slash handling, HTTP/network error surfacing), mschema/render 3 formats (markdown, mschema-text, schema-dict) + eligibility rules (wide_text excluded, short_text/numeric/enum/temporal/ boolean eligible, annotation override wins). 25 tests. pyproject registers l0/l2 markers + addopts '-m not l2' (L2 opt-in). Deferred to their dependency-porting tasks: test_rrf.py (search needs vectorstore, B3) and the 11 CLI contract tests (need _guards/session, wired when each command lands). 'Not assumed reliable' now has real teeth for the data layer; CLI/search contracts follow.
55 lines
2.2 KiB
Python
55 lines
2.2 KiB
Python
"""L0: db/sampling against known data (testcontainers).
|
|
Verifies unique_values_for_lsh returns the expected most-frequent values for text
|
|
columns, and that wide_text / non-text columns are excluded.
|
|
"""
|
|
import pytest
|
|
|
|
from nsp.config import LshConfig
|
|
from nsp.db.introspect import introspect
|
|
from nsp.db.sampling import is_text_type, unique_values_for_lsh
|
|
|
|
pytestmark = [pytest.mark.l0]
|
|
|
|
|
|
def test_is_text_type():
|
|
assert is_text_type("text")
|
|
assert is_text_type("varchar(100)")
|
|
assert is_text_type("character varying")
|
|
assert not is_text_type("integer")
|
|
assert not is_text_type("bigint")
|
|
assert not is_text_type("timestamp without time zone")
|
|
|
|
|
|
def test_unique_values_for_lsh_returns_most_frequent(admin_engine):
|
|
schema = introspect(admin_engine, "testdb", "dw")
|
|
# Before classify_all, all text columns are eligible=True by default. Sampling
|
|
# only touches text types regardless.
|
|
values, skipped, truncated = unique_values_for_lsh(
|
|
admin_engine, schema, LshConfig(max_values_per_column=100)
|
|
)
|
|
# dim_pazienti.citta: Milano, Bergamo, Brescia (3 distinct, all eligible text)
|
|
citta = values.get("dim_pazienti", {}).get("citta")
|
|
assert citta is not None
|
|
assert set(citta) == {"Milano", "Bergamo", "Brescia"}
|
|
|
|
|
|
def test_unique_values_for_lsh_excludes_non_text(admin_engine):
|
|
schema = introspect(admin_engine, "testdb", "dw")
|
|
values, _, _ = unique_values_for_lsh(
|
|
admin_engine, schema, LshConfig(max_values_per_column=100)
|
|
)
|
|
# id_paziente is bigint — must never appear in the LSH values.
|
|
assert "id_paziente" not in values.get("dim_pazienti", {})
|
|
|
|
|
|
def test_unique_values_for_lsh_truncation_reported(admin_engine):
|
|
schema = introspect(admin_engine, "testdb", "dw")
|
|
# Force a tiny cap so procedura/diagnosi columns (which have >2 distinct values)
|
|
# are reported as truncated rather than silently cut.
|
|
_, _, truncated = unique_values_for_lsh(
|
|
admin_engine, schema, LshConfig(max_values_per_column=1)
|
|
)
|
|
truncated_cols = {(t.table, t.column) for t in truncated}
|
|
# fct_ricoveri has several eligible text columns with distinct values
|
|
assert any(t[0] == "fct_ricoveri" for t in truncated_cols)
|