284 lines
11 KiB
Python
284 lines
11 KiB
Python
# ruff: noqa: DTZ001
|
|
from datetime import datetime
|
|
|
|
import yaml
|
|
from typer.testing import CliRunner
|
|
|
|
from tht.cli import app
|
|
from tht.mschema.merge import find_orphans
|
|
from tht.mschema.models import (
|
|
Annotations,
|
|
ColumnPhysical,
|
|
ForeignKey,
|
|
PhysicalSchema,
|
|
TableAnnotation,
|
|
TablePhysical,
|
|
)
|
|
from tht.mschema.render import to_mschema_text, to_schema_dict
|
|
|
|
|
|
def _physical():
|
|
return PhysicalSchema(
|
|
database="d", schema="s", introspected_at=datetime(2026, 1, 1),
|
|
tables={
|
|
"dim_patient": TablePhysical(
|
|
columns={"cod_paz": ColumnPhysical(type="bigint", pk=True)},
|
|
),
|
|
"dim_time": TablePhysical(
|
|
columns={"day_key": ColumnPhysical(type="integer", pk=True)},
|
|
),
|
|
"fact_ablazione": TablePhysical(
|
|
columns={
|
|
"cod_paz": ColumnPhysical(type="bigint"),
|
|
"data_time_key": ColumnPhysical(type="integer"),
|
|
"esito": ColumnPhysical(type="text"),
|
|
},
|
|
),
|
|
},
|
|
)
|
|
|
|
|
|
def _annotations_with_fks():
|
|
return Annotations(
|
|
tables={
|
|
"fact_ablazione": TableAnnotation(
|
|
foreign_keys=[
|
|
ForeignKey(columns=["cod_paz"], ref_table="dim_patient",
|
|
ref_columns=["cod_paz"]),
|
|
ForeignKey(columns=["data_time_key"], ref_table="dim_time",
|
|
ref_columns=["day_key"]),
|
|
],
|
|
)
|
|
}
|
|
)
|
|
|
|
|
|
def test_mschema_text_renders_annotation_fks():
|
|
text = to_mschema_text(_physical(), _annotations_with_fks())
|
|
assert "fact_ablazione.cod_paz=dim_patient.cod_paz" in text
|
|
assert "fact_ablazione.data_time_key=dim_time.day_key" in text
|
|
|
|
|
|
def test_schema_dict_merges_annotation_fks():
|
|
d = to_schema_dict(_physical(), _annotations_with_fks())
|
|
fks = d["fact_ablazione"]["foreign_keys"]
|
|
assert {"columns": ["cod_paz"], "ref_table": "dim_patient",
|
|
"ref_columns": ["cod_paz"]} in fks
|
|
|
|
|
|
def test_find_orphans_flags_broken_annotation_fk():
|
|
ann = Annotations(
|
|
tables={
|
|
"fact_ablazione": TableAnnotation(
|
|
foreign_keys=[
|
|
ForeignKey(columns=["cod_paz"], ref_table="dim_sparita",
|
|
ref_columns=["x"]),
|
|
ForeignKey(columns=["colonna_sparita"], ref_table="dim_time",
|
|
ref_columns=["day_key"]),
|
|
],
|
|
)
|
|
}
|
|
)
|
|
orphans = find_orphans(_physical(), ann)
|
|
assert "fact_ablazione.fk(cod_paz)->dim_sparita" in orphans
|
|
assert "fact_ablazione.fk(colonna_sparita)->dim_time" in orphans
|
|
|
|
|
|
def test_find_orphans_ok_with_valid_fk():
|
|
assert find_orphans(_physical(), _annotations_with_fks()) == []
|
|
|
|
|
|
def _write_workspace(tmp_path):
|
|
_physical().to_yaml(tmp_path / "artifacts" / "mschema" / "physical.yaml")
|
|
cfg = tmp_path / "workspace.yaml"
|
|
cfg.write_text(
|
|
"database: {database: d, schema: s, user: u, password: p, transport: direct}\n"
|
|
f"paths: {{artifacts: {tmp_path/'artifacts'}, indexes: {tmp_path/'i'}, sessions: {tmp_path/'s'}}}\n"
|
|
)
|
|
return cfg
|
|
|
|
|
|
def test_suggest_fks_prints_candidates(tmp_path):
|
|
cfg = _write_workspace(tmp_path)
|
|
res = CliRunner().invoke(app, ["schema", "suggest-fks", "-c", str(cfg)])
|
|
assert res.exit_code == 0, res.output
|
|
data = yaml.safe_load(res.output.rsplit("\n", 2)[0].split("FK candidate")[0])
|
|
fks = data["tables"]["fact_ablazione"]["foreign_keys"]
|
|
assert {"columns": ["cod_paz"], "ref_table": "dim_patient",
|
|
"ref_columns": ["cod_paz"]} in fks
|
|
assert {"columns": ["data_time_key"], "ref_table": "dim_time",
|
|
"ref_columns": ["day_key"]} in fks
|
|
|
|
|
|
def test_mine_join_pairs_from_approved_sql():
|
|
from tht.mschema.fkmine import mine_join_pairs
|
|
|
|
sql = """
|
|
WITH abl AS (
|
|
SELECT sea.cod_paz, dt.year
|
|
FROM datawarehouse.fact_ablazione AS sea
|
|
JOIN datawarehouse.dim_time AS dt ON sea.data_time_key = dt.day_key
|
|
)
|
|
SELECT * FROM abl JOIN abl b ON abl.year = b.year;
|
|
"""
|
|
pairs = mine_join_pairs(sql, _physical())
|
|
assert pairs[("fact_ablazione", "data_time_key", "dim_time", "day_key")] == 1
|
|
# il join CTE-CTE (abl.year=b.year) non produce coppie
|
|
assert len(pairs) == 1
|
|
|
|
|
|
def test_mine_join_pairs_ignores_non_pk_pairs_and_bad_sql():
|
|
from tht.mschema.fkmine import mine_join_pairs
|
|
|
|
# esito=esito: nessun lato e' PK -> scartato
|
|
sql = ("SELECT * FROM fact_ablazione a JOIN fact_ablazione b "
|
|
"ON a.esito = b.esito")
|
|
assert len(mine_join_pairs(sql, _physical())) == 0
|
|
assert len(mine_join_pairs("WITH broken (", _physical())) == 0
|
|
|
|
|
|
def test_suggest_fks_skips_generic_and_ambiguous_pks(tmp_path):
|
|
phys = PhysicalSchema(
|
|
database="d", schema="s", introspected_at=datetime(2026, 1, 1),
|
|
tables={
|
|
"dim_a": TablePhysical(columns={"id": ColumnPhysical(type="int", pk=True)}),
|
|
"dim_b": TablePhysical(columns={"id": ColumnPhysical(type="int", pk=True)}),
|
|
"dim_c1": TablePhysical(columns={"cod_x": ColumnPhysical(type="int", pk=True)}),
|
|
"dim_c2": TablePhysical(columns={"cod_x": ColumnPhysical(type="int", pk=True)}),
|
|
"fact_f": TablePhysical(
|
|
columns={
|
|
"id": ColumnPhysical(type="int"),
|
|
"cod_x": ColumnPhysical(type="int"),
|
|
},
|
|
),
|
|
},
|
|
)
|
|
phys.to_yaml(tmp_path / "artifacts" / "mschema" / "physical.yaml")
|
|
cfg = tmp_path / "workspace.yaml"
|
|
cfg.write_text(
|
|
"database: {database: d, schema: s, user: u, password: p, transport: direct}\n"
|
|
f"paths: {{artifacts: {tmp_path/'artifacts'}, indexes: {tmp_path/'i'}, sessions: {tmp_path/'s'}}}\n"
|
|
)
|
|
res = CliRunner().invoke(app, ["schema", "suggest-fks", "-c", str(cfg)])
|
|
assert res.exit_code == 0, res.output
|
|
assert "nessuna FK da suggerire" in res.output # id generico, cod_x ambigua
|
|
assert "cod_x" in res.output # segnalata come ambigua saltata
|
|
|
|
# --assume disambigua la PK multi-proprietario
|
|
res2 = CliRunner().invoke(
|
|
app, ["schema", "suggest-fks", "-c", str(cfg), "--assume", "cod_x=dim_c1"]
|
|
)
|
|
assert res2.exit_code == 0, res2.output
|
|
yaml_text = "\n".join(
|
|
line for line in res2.output.splitlines() if "FK candidate" not in line
|
|
)
|
|
data = yaml.safe_load(yaml_text)
|
|
fact_fks = data["tables"]["fact_f"]["foreign_keys"]
|
|
assert {"columns": ["cod_x"], "ref_table": "dim_c1",
|
|
"ref_columns": ["cod_x"]} in fact_fks
|
|
# dim_c2.cod_x -> dim_c1 (estensione 1:1), ma NON dim_c1 -> se stessa
|
|
assert "dim_c1" not in data["tables"] or all(
|
|
fk["ref_table"] != "dim_c1" for fk in data["tables"].get("dim_c1", {}).get("foreign_keys", [])
|
|
)
|
|
|
|
# --assume con tabella inesistente -> errore chiaro
|
|
res3 = CliRunner().invoke(
|
|
app, ["schema", "suggest-fks", "-c", str(cfg), "--assume", "cod_x=nope"]
|
|
)
|
|
assert res3.exit_code == 1
|
|
assert "non valido" in res3.output
|
|
|
|
|
|
def test_suggest_fks_from_sql_mines_joins(tmp_path):
|
|
cfg = _write_workspace(tmp_path)
|
|
sqldir = tmp_path / "approved"
|
|
sqldir.mkdir()
|
|
(sqldir / "q1.sql").write_text(
|
|
"SELECT f.esito FROM datawarehouse.fact_ablazione f "
|
|
"JOIN datawarehouse.dim_patient p ON f.cod_paz = p.cod_paz"
|
|
)
|
|
res = CliRunner().invoke(
|
|
app, ["schema", "suggest-fks", "-c", str(cfg), "--from-sql", str(sqldir)]
|
|
)
|
|
assert res.exit_code == 0, res.output
|
|
assert "Minati 1 equi-join da 1 file SQL" in res.output
|
|
assert "ref_table: dim_patient" in res.output
|
|
|
|
|
|
def test_suggest_fks_write_merges_and_is_idempotent(tmp_path):
|
|
cfg = _write_workspace(tmp_path)
|
|
ann_path = tmp_path / "artifacts" / "mschema" / "annotations.yaml"
|
|
Annotations(
|
|
tables={"fact_ablazione": TableAnnotation(description="Ablazioni")}
|
|
).to_yaml(ann_path)
|
|
|
|
res = CliRunner().invoke(app, ["schema", "suggest-fks", "-c", str(cfg), "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
ann = Annotations.from_yaml(ann_path)
|
|
assert ann.tables["fact_ablazione"].description == "Ablazioni" # non distrutta
|
|
assert len(ann.tables["fact_ablazione"].foreign_keys) == 2
|
|
|
|
res2 = CliRunner().invoke(app, ["schema", "suggest-fks", "-c", str(cfg), "--write"])
|
|
assert "nessuna FK da suggerire" in res2.output
|
|
ann2 = Annotations.from_yaml(ann_path)
|
|
assert len(ann2.tables["fact_ablazione"].foreign_keys) == 2
|
|
|
|
|
|
def test_suggest_fks_json_is_single_deterministic_document_without_writing(tmp_path):
|
|
import hashlib
|
|
import json
|
|
|
|
cfg = _write_workspace(tmp_path)
|
|
staged = tmp_path / "staged"
|
|
staged.mkdir()
|
|
(staged / "z.sql").write_text(
|
|
"SELECT * FROM fact_ablazione f JOIN dim_patient p ON f.cod_paz = p.cod_paz"
|
|
)
|
|
(staged / "a.sql").write_text(
|
|
"SELECT * FROM fact_ablazione f JOIN dim_time p ON f.data_time_key = p.day_key"
|
|
)
|
|
response = CliRunner().invoke(
|
|
app, ["schema", "suggest-fks", "--json", "-c", str(cfg), "--from-sql", str(staged)]
|
|
)
|
|
assert response.exit_code == 0, response.output
|
|
assert response.stdout.count("\n") == 1
|
|
payload = json.loads(response.stdout)
|
|
assert payload["status"] == "succeeded"
|
|
assert payload["code"] == "ok"
|
|
assert payload["candidate_count"] == 2
|
|
assert payload["candidate_digest"].startswith("sha256:")
|
|
assert payload["orphan_count"] == 0
|
|
# The digest is over the canonical candidate export and --json never writes annotations.
|
|
expected = json.dumps(payload["candidates"], ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
assert payload["candidate_digest"] == "sha256:" + hashlib.sha256(expected.encode()).hexdigest()
|
|
assert not (tmp_path / "artifacts" / "mschema" / "annotations.yaml").exists()
|
|
|
|
|
|
def test_suggest_fks_json_failure_is_safe_and_single_document(tmp_path):
|
|
import json
|
|
|
|
cfg = _write_workspace(tmp_path)
|
|
(tmp_path / "artifacts" / "mschema" / "physical.yaml").unlink()
|
|
response = CliRunner().invoke(app, ["schema", "suggest-fks", "--json", "-c", str(cfg)])
|
|
assert response.exit_code != 0
|
|
assert response.stdout.count("\n") == 1
|
|
payload = json.loads(response.stdout)
|
|
assert payload == {"status": "failed", "code": "physical_schema_missing"}
|
|
assert str(tmp_path) not in response.stdout
|
|
|
|
|
|
def test_schema_check_json_reports_orphan_count_without_prose(tmp_path):
|
|
import json
|
|
|
|
cfg = _write_workspace(tmp_path)
|
|
Annotations(tables={"gone": TableAnnotation(description="x")}).to_yaml(
|
|
tmp_path / "artifacts" / "mschema" / "annotations.yaml"
|
|
)
|
|
response = CliRunner().invoke(app, ["schema", "check", "--json", "-c", str(cfg)])
|
|
assert response.exit_code == 3
|
|
assert response.stdout.count("\n") == 1
|
|
assert json.loads(response.stdout) == {
|
|
"status": "failed", "code": "annotation_orphans", "orphan_count": 1,
|
|
"orphans": ["gone"],
|
|
}
|