mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

refactor(evidence): unify formulas with typed evidence

Browse Source
This commit is contained in:
marcopan
2026-08-25 01:23:04 +02:00
parent d5d65f3659
commit cc30148b69
11 changed files with 342 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/tht/cli/search_cmd.py
+62 -23
View File
@@ -11,7 +11,7 @@ KIND_MAP = {
"evidence": ["evidence"],
"schema": ["schema_table", "schema_column"],
"values": [], # solo LSH
"formula": [], # solo formula store (D14b), niente LSH/vector
"formula": ["evidence"],
}
_STAGE_PURPOSES = {
@@ -30,6 +30,20 @@ DEFAULT_TOP_FALLBACK = 10
search_app = typer.Typer(help="Ricerca semantica (evidence/schema/values) nel vectorstore")
def search_formula_evidence(keyword: str, *, searcher, embedder, top: int):
"""Search only published Formula Evidence through the shared typed facade."""
from tht.evidence import EvidenceSearchContext, search_evidence
return search_evidence(
keyword,
"sql_generation",
EvidenceSearchContext(required_kinds=("formula",)),
searcher=searcher,
embedder=embedder,
top_n=top,
)
def _leased_dwh_snapshot(cfg, context: typer.Context):
from tht.jobs.dwh_pipeline import lease_dwh_snapshot
@@ -143,12 +157,6 @@ def search_cmd(
workspace_id = workspace_id_for_config(cfg, config)
validate_corpus_workspace(cfg, workspace_id)
dwh_snapshot = _leased_dwh_snapshot(cfg, ctx)
require_vector_cfg(cfg)
runtime_searcher = active_searcher(
cfg, open_searcher(cfg),
workspace_id=workspace_id,
)
if kind is not None and kind not in KIND_MAP:
typer.secho(
f"ERRORE: --kind sconosciuto: {kind} (validi: {', '.join(KIND_MAP)})",
@@ -160,29 +168,60 @@ def search_cmd(
top = cfg.search.top_schema_tables if kind == "schema" else DEFAULT_TOP_FALLBACK
if kind == "formula":
# D14b: recupero formule di concetto dallo store locale (niente LSH/vector).
from tht.cli.evidence_cmd import evidence_root
from tht.evidence.formula_store import search_formulas
formulas = search_formulas(evidence_root(cfg), keyword)[:top]
require_vector_cfg(cfg)
outcome = search_formula_evidence(
keyword,
searcher=active_searcher(cfg, open_searcher(cfg), workspace_id=workspace_id),
embedder=make_embedder(cfg.embeddings),
top=top,
)
if outcome.status == "unavailable":
payload = {"status": outcome.status, "code": outcome.code, "message": outcome.message}
if json_out:
typer.echo(json.dumps(payload, ensure_ascii=False))
else:
typer.secho(f"ERRORE: {outcome.message}", fg=typer.colors.RED, err=True)
raise typer.Exit(1)
if json_out:
typer.echo(json.dumps(
[f.model_dump(mode="json") for f in formulas], ensure_ascii=False, indent=2))
typer.echo(json.dumps([
{
"evidence_id": result.evidence_id,
"title": result.title,
"kind": result.kind,
"excerpts": list(result.excerpts),
"provenance": result.provenance,
"citation": result.citation,
"document_id": result.document_id,
}
for result in outcome.results
], ensure_ascii=False, indent=2))
return
if not formulas:
typer.secho(
"ATTENZIONE: lo store formule legacy non viene più consultato; "
"sono disponibili solo Formula Evidence pubblicate.",
fg=typer.colors.YELLOW,
err=True,
)
if not outcome.results:
typer.secho(f"Nessuna formula per '{keyword}'.", fg=typer.colors.YELLOW)
return
table = Table(title=f"Formule per '{keyword}'")
table.add_column("Concetto")
table.add_column("Status")
table.add_column("Colonne")
table.add_column("SQL")
for f in formulas:
sql_preview = (f.sql[:80] + "…") if len(f.sql) > 80 else f.sql
table.add_row(f.concept, f.status, ", ".join(f.columns), sql_preview)
table = Table(title=f"Formula Evidence per '{keyword}'")
table.add_column("Formula")
table.add_column("Provenienza")
table.add_column("Estratto")
for result in outcome.results:
excerpt = result.excerpts[0] if result.excerpts else ""
table.add_row(result.title, result.citation, excerpt[:120])
Console().print(table)
return
dwh_snapshot = _leased_dwh_snapshot(cfg, ctx)
require_vector_cfg(cfg)
runtime_searcher = active_searcher(
cfg, open_searcher(cfg),
workspace_id=workspace_id,
)
lsh_hits = None
try:
lsh, minhashes, meta = load_index(
harness/tht/evidence/formula_store.py
+47 -13
View File
@@ -1,17 +1,12 @@
"""SQL concept->formula evidence store (spec D14b, §4.7).
"""Legacy ConceptFormula reader and one-way migration into Curated Evidence.
A concept (e.g. 'fascia pediatrica', 'ablazione') maps to a reusable SQL formula
(a CASE WHEN ...) that derives it from physical columns. These are reviewable
units: the gate surfaces a candidate formula, the reviewer approves or rejects it
(decision types concept_formula_approved / concept_formula_rejected), and approved
formulas travel with the schema-linking artifact.
Storage: one file per formula, frontmatter YAML + SQL body (same shape as
EvidenceDoc.parse). Directory layout: <root>/formulas/<slug>-<n>.sql.md.
Retrieve is by concept (may return several, e.g. competing drafts vs reviewed).
The ``formulas/*.sql.md`` store is retained only for the migration window. Runtime
lookup uses typed, published ``kind=formula`` Evidence instead. A session reviewer
may still approve a formula locally; that is a proposal, not publication.
"""
from __future__ import annotations
import hashlib
import re
from pathlib import Path
from typing import Literal
@@ -19,6 +14,8 @@ from typing import Literal
import yaml
from pydantic import BaseModel
from tht.evidence.canonical import CuratedEvidence
FORMULAS_SUBDIR = "formulas"
_SUFFIX_RE = re.compile(r"^(.*?)-(\d+)\.sql\.md$")
@@ -104,8 +101,45 @@ def retrieve_formula(root: Path | str, concept: str) -> list[ConceptFormula]:
def search_formulas(root: Path | str, query: str) -> list[ConceptFormula]:
"""Formulas whose concept contains `query` (case-insensitive). Used by
`tht search find --kind formula` (D14b retrieval, §4.7.2): the reviewer searches a
concept term and gets the candidate formulas to approve before they reach the CTE."""
"""Read legacy formulas for migration tooling only (case-insensitive concept match)."""
q = query.strip().lower()
return [f for f in _load_all(Path(root)) if q in f.concept.lower()]
def legacy_formula_to_curated(
formula: ConceptFormula,
*,
legacy_path: str,
) -> CuratedEvidence | None:
"""Convert one reviewed legacy formula into its deterministic curated counterpart.
Drafts and model-generated formulas have no global publication status. Their
caller must project them as session-local Formula proposals instead.
"""
if formula.status != "reviewed":
return None
source_notes = tuple(formula.sources) or (
"Legacy formula migrated without a recorded provenance note.",
)
source_sha256 = hashlib.sha256(formula.dump().encode("utf-8")).hexdigest()
source_file = legacy_path if legacy_path.startswith("source/") else f"source/{legacy_path}"
return CuratedEvidence.model_validate({
"schema_version": 1,
"id": f"evidence:{formula._slug}",
"title": formula.concept[:1].upper() + formula.concept[1:],
"kind": "formula",
"purposes": ["schema_linking", "sql_generation"],
"applies_to": {"concepts": [formula.concept], "columns": formula.columns},
"language": "it",
"provenance": {
"source_file": source_file,
"source_sha256": f"sha256:{source_sha256}",
"supporting_excerpts": source_notes,
},
"review_items": [],
"payload": {
"concept": formula.concept,
"columns": formula.columns,
"sql": formula.sql,
},
})
harness/tht/evidence/session.py
+39 -1
View File
@@ -55,7 +55,45 @@ def project_session(
"esito": "accettata" if decision.type == "evidence_accepted" else "scartata",
"decision_seq": decision.seq,
}
return list(entries.values())
return list(entries.values()) + _formula_proposals(decisions, linking)
def _formula_proposals(decisions: list["DecisionRecord"], linking: "SchemaLinking") -> list[dict]:
"""Project locally approved F4 formulas without representing them as Evidence.
A proposal remains in the persisted session artifact until a separate curator
imports, reviews, and publishes it in the workspace repository.
"""
approved = {
decision.detail: decision.seq
for decision in decisions
if decision.type == "concept_formula_approved" and decision.detail
}
proposals = []
for formula in linking.concept_formulas:
if not isinstance(formula, dict):
continue
concept = formula.get("concept")
sql = formula.get("sql")
columns = formula.get("columns")
if not isinstance(concept, str) or not isinstance(sql, str) or not isinstance(columns, list):
continue
# A referenced published Formula Evidence is already represented by its
# Evidence receipt/citation, so it must not be reintroduced as a proposal.
evidence_id = formula.get("evidence_id", formula.get("id", ""))
if isinstance(evidence_id, str) and evidence_id.startswith("evidence:"):
continue
proposals.append({
"schema_version": 1,
"kind": "formula_proposal",
"publication": "session_only",
"concept": concept,
"columns": columns,
"sql": sql,
"sources": formula.get("sources", []),
"decision_seq": approved.get(concept),
})
return proposals
@dataclass(frozen=True)
harness/tht/pi_skill_projection.py
+1
View File
@@ -19,6 +19,7 @@ FRAGMENT_ORDER = (
("{{DISAMBIGUATION_REWRITING_INSTRUCTIONS}}", "disambiguation/phase-3.md"),
("{{MEMORY_SOLVED_SEARCH_F4}}", "memory/solved-search-f4.md"),
("{{DISAMBIGUATION_SCHEMA_GROUNDING}}", "disambiguation/schema-grounding.md"),
("{{EVIDENCE_FORMULA_PROPOSALS}}", "evidence/formula-proposals.md"),
("{{MEMORY_SOLVED_SEARCH_F6}}", "memory/solved-search-f6.md"),
("{{MEMORY_SOLVED_SEARCH_F7}}", "memory/solved-search-f7.md"),
("{{MEMORY_PROMOTION_F8}}", "memory/phase-8-promotion.md"),