feat(opt): three efficiency levers for NL→SQL workflow
Lever 1: Join-graph via FK logics in annotations + suggest-fks command
- TableAnnotation.foreign_keys field stores curated logical FKs (DWH has no FK constraints)
- tht schema suggest-fks: mine from approved SQL, heuristics (time_key → dim_time),
same-name discovery + explicit --assume flag for multi-owner PKs
- mschema renders 【Foreign keys】 section populated; validation in merge.py
- SKILL.md F4 now reads FKs from mschema-text, no custom data_time_key logic
Lever 2: Context-pack consolidation at kickoff (tht search pack)
- Single embedding of question, reused for schema + evidence + solved searches
- One command: tht search pack <question> --session <id> → retrieval_pack.md
- Graceful degradation when Ollama/vector store unreachable (exit 0, empty sections)
- SKILL.md F1 prescribes as first call; reduces model thinking turns via pre-retrieval
Lever 3: Phase-summary recap v2 auto-construction from session ledger
- tht session show --json includes full decisions ledger
- tht phase meta --json exports 'emits' (substantive decision types per phase)
- Gate appends deterministic 【Decisioni registrate in questa fase】 section (appendLedgerSection)
- Model authors only summary + checks; recap table comes from persisted state (exact by construction)
- SKILL.md Disciplina 6: brief model output, gate fills the rest
Tests: 358 Python (including 10 FK + 3 pack + 1 session-ledger tests) + 111 JS gate tests, all pass.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5
@@ -141,6 +141,174 @@ def check_cmd(config: Path = CONFIG_OPT) -> None:
|
||||
typer.secho("OK: nessuna annotazione orfana.", fg=typer.colors.GREEN)
|
||||
|
||||
|
||||
# PK con questi nomi sono identificatori generici: la regola same-name non si applica
|
||||
# (nel DWH reale `id` e' la PK di ~50 tabelle e produrrebbe migliaia di falsi positivi).
|
||||
_GENERIC_PK_NAMES = {"id", "key", "code"}
|
||||
|
||||
|
||||
@schema_app.command("suggest-fks")
|
||||
def suggest_fks_cmd(
|
||||
config: Path = CONFIG_OPT,
|
||||
from_sql: list[Path] = typer.Option(
|
||||
None, "--from-sql",
|
||||
help="Directory di .sql approvati da cui minare i join reali (ripetibile).",
|
||||
),
|
||||
assume: list[str] = typer.Option(
|
||||
None, "--assume",
|
||||
help="Disambigua una PK con piu' proprietari: col=tabella_ref "
|
||||
"(es. cod_paz=dim_patient). Ripetibile.",
|
||||
),
|
||||
write: bool = typer.Option(
|
||||
False, "--write",
|
||||
help="Fonde i suggerimenti in annotations.yaml (aggiunge solo FK mancanti).",
|
||||
),
|
||||
) -> None:
|
||||
"""Suggerisce FK logiche per la curazione umana in annotations.yaml.
|
||||
|
||||
Tre regole, in ordine di confidenza: (1) equi-join minati dall'SQL gia'
|
||||
approvato (--from-sql); (2) colonna `*time_key` verso la PK di dim_time;
|
||||
(3) colonna con lo stesso nome della PK di UN'ALTRA tabella, solo se quel
|
||||
nome ha un unico proprietario e non e' generico (id/key/code) — salvo
|
||||
disambiguazione esplicita con --assume.
|
||||
"""
|
||||
import yaml as _yaml
|
||||
|
||||
from tht.mschema.fkmine import mine_join_pairs
|
||||
from tht.mschema.models import Annotations, ForeignKey, PhysicalSchema, TableAnnotation
|
||||
|
||||
cfg = _load_config_or_exit(config)
|
||||
phys_file = physical_path(cfg)
|
||||
if not phys_file.exists():
|
||||
typer.secho(
|
||||
f"ERRORE: {phys_file} non trovato. Esegui prima `tht schema introspect`.",
|
||||
fg=typer.colors.RED, err=True,
|
||||
)
|
||||
raise typer.Exit(code=1)
|
||||
physical = PhysicalSchema.from_yaml(phys_file)
|
||||
ann_path = annotations_path(cfg)
|
||||
annotations = Annotations.from_yaml(ann_path)
|
||||
|
||||
assumed: dict[str, str] = {}
|
||||
for a in assume or []:
|
||||
col, _, ref = a.partition("=")
|
||||
if not ref or ref not in physical.tables:
|
||||
typer.secho(
|
||||
f"ERRORE: --assume '{a}' non valido (atteso col=tabella nel catalogo).",
|
||||
fg=typer.colors.RED, err=True,
|
||||
)
|
||||
raise typer.Exit(code=1)
|
||||
assumed[col] = ref
|
||||
|
||||
def _single_pk(table) -> str | None:
|
||||
pks = [c for c, col in table.columns.items() if col.pk]
|
||||
return pks[0] if len(pks) == 1 else None
|
||||
|
||||
pk_owners: dict[str, list[str]] = {}
|
||||
for tname, table in physical.tables.items():
|
||||
pk = _single_pk(table)
|
||||
if pk:
|
||||
pk_owners.setdefault(pk, []).append(tname)
|
||||
|
||||
dim_time_pk = None
|
||||
if "dim_time" in physical.tables:
|
||||
dim_time_pk = _single_pk(physical.tables["dim_time"])
|
||||
|
||||
def _known(tname: str) -> set:
|
||||
keys = set()
|
||||
for fk in physical.tables[tname].foreign_keys:
|
||||
keys.add((tuple(fk.columns), fk.ref_table, tuple(fk.ref_columns)))
|
||||
ann = annotations.tables.get(tname)
|
||||
if ann:
|
||||
for fk in ann.foreign_keys:
|
||||
keys.add((tuple(fk.columns), fk.ref_table, tuple(fk.ref_columns)))
|
||||
return keys
|
||||
|
||||
known_by_table: dict[str, set] = {t: _known(t) for t in physical.tables}
|
||||
suggested: dict[str, list[ForeignKey]] = {}
|
||||
|
||||
def _add(tname: str, col: str, ref_table: str, ref_col: str) -> None:
|
||||
key = ((col,), ref_table, (ref_col,))
|
||||
if key in known_by_table[tname]:
|
||||
return
|
||||
known_by_table[tname].add(key)
|
||||
suggested.setdefault(tname, []).append(
|
||||
ForeignKey(columns=[col], ref_table=ref_table, ref_columns=[ref_col])
|
||||
)
|
||||
|
||||
# Regola 1: join minati dall'SQL approvato.
|
||||
n_sql_files = 0
|
||||
mined_total = 0
|
||||
for d in from_sql or []:
|
||||
for sql_file in sorted(d.rglob("*.sql")):
|
||||
n_sql_files += 1
|
||||
pairs = mine_join_pairs(sql_file.read_text(), physical)
|
||||
mined_total += sum(pairs.values())
|
||||
for (src_t, src_c, ref_t, ref_c) in pairs:
|
||||
_add(src_t, src_c, ref_t, ref_c)
|
||||
|
||||
# Regole 2 e 3: convenzioni di naming.
|
||||
ambiguous_skipped: set[str] = set()
|
||||
for tname, table in physical.tables.items():
|
||||
for cname in table.columns:
|
||||
if dim_time_pk and cname.endswith("time_key") and tname != "dim_time":
|
||||
_add(tname, cname, "dim_time", dim_time_pk)
|
||||
continue
|
||||
if cname in assumed:
|
||||
if assumed[cname] != tname:
|
||||
_add(tname, cname, assumed[cname], cname)
|
||||
continue
|
||||
owners = [o for o in pk_owners.get(cname, []) if o != tname]
|
||||
if not owners or cname in _GENERIC_PK_NAMES:
|
||||
continue
|
||||
if len(pk_owners[cname]) > 1:
|
||||
ambiguous_skipped.add(cname)
|
||||
continue
|
||||
_add(tname, cname, owners[0], cname)
|
||||
|
||||
if n_sql_files:
|
||||
typer.secho(
|
||||
f"Minati {mined_total} equi-join da {n_sql_files} file SQL.",
|
||||
fg=typer.colors.BLUE, err=True,
|
||||
)
|
||||
if ambiguous_skipped:
|
||||
typer.secho(
|
||||
"PK ambigue saltate dalla regola same-name (piu' tabelle proprietarie): "
|
||||
+ ", ".join(sorted(ambiguous_skipped))
|
||||
+ ". Se servono, aggiungile a mano o passa --from-sql.",
|
||||
fg=typer.colors.YELLOW, err=True,
|
||||
)
|
||||
|
||||
n_fks = sum(len(v) for v in suggested.values())
|
||||
if not suggested:
|
||||
typer.secho("OK: nessuna FK da suggerire.", fg=typer.colors.GREEN)
|
||||
return
|
||||
|
||||
if write:
|
||||
for tname, fks in suggested.items():
|
||||
ann = annotations.tables.setdefault(tname, TableAnnotation())
|
||||
ann.foreign_keys.extend(fks)
|
||||
annotations.to_yaml(ann_path)
|
||||
typer.secho(
|
||||
f"OK: {n_fks} FK suggerite aggiunte a {ann_path} "
|
||||
f"({len(suggested)} tabelle). Rivedile a mano prima dell'uso.",
|
||||
fg=typer.colors.GREEN,
|
||||
)
|
||||
return
|
||||
|
||||
payload = {
|
||||
"tables": {
|
||||
tname: {"foreign_keys": [fk.model_dump(exclude_defaults=True) for fk in fks]}
|
||||
for tname, fks in suggested.items()
|
||||
}
|
||||
}
|
||||
typer.echo(_yaml.safe_dump(payload, sort_keys=False, allow_unicode=True))
|
||||
typer.secho(
|
||||
f"{n_fks} FK candidate ({len(suggested)} tabelle). "
|
||||
f"Usa --write per fonderle in annotations.yaml, poi curale a mano.",
|
||||
fg=typer.colors.YELLOW,
|
||||
)
|
||||
|
||||
|
||||
@schema_app.command("render")
|
||||
def render_cmd(
|
||||
config: Path = CONFIG_OPT,
|
||||
|
||||
Reference in New Issue
Block a user