feat(evidence): build semantic fragments from typed units

2026-08-24 21:33:48 +02:00
parent 8adc085746
commit 3420c57c8b
9 changed files with 642 additions and 14 deletions
@@ -2,6 +2,7 @@ import hashlib

 import pytest

+from tht.evidence.canonical import CuratedEvidence
 from tht.evidence.corpus.chunk import ChunkPolicy, chunk
 from tht.evidence.corpus.models import CanonicalDocument, CorpusManifest

@@ -33,6 +34,50 @@ def other_document(content: str) -> CanonicalDocument:
    )


+def curated_formula_document(*, sql: str = "CASE WHEN age < 18 THEN 'pediatric' END") -> CanonicalDocument:
+    evidence = CuratedEvidence.model_validate(
+        {
+            "schema_version": 1,
+            "id": "evidence:fascia-pediatrica",
+            "title": "Fascia pediatrica",
+            "kind": "formula",
+            "purposes": ["sql_generation", "schema_linking"],
+            "applies_to": {
+                "concepts": ["fascia pediatrica"],
+                "tables": ["clinical.patient"],
+                "columns": ["clinical.patient.birth_date"],
+            },
+            "language": "it",
+            "provenance": {
+                "source_file": "source/paziente.md",
+                "source_sha256": "sha256:" + "a" * 64,
+                "supporting_excerpts": ["I pazienti pediatrici hanno età inferiore a 18 anni."],
+            },
+            "review_items": [],
+            "payload": {
+                "concept": "fascia pediatrica",
+                "columns": ["clinical.patient.birth_date"],
+                "sql": sql,
+            },
+        }
+    )
+    return curated_document(evidence)
+
+
+def curated_document(evidence: CuratedEvidence) -> CanonicalDocument:
+    content = "canonical curated Evidence"
+    return CanonicalDocument(
+        document_id="doc:" + evidence.id.removeprefix("evidence:"),
+        source_id="curated:" + evidence.id.removeprefix("evidence:"),
+        source_uri=f"file:///safe/curated/{evidence.kind}/{evidence.id.removeprefix('evidence:')}.md",
+        source_fingerprint="sha256:" + "b" * 64,
+        content_hash="sha256:" + hashlib.sha256(content.encode()).hexdigest(),
+        title=evidence.title,
+        content=content,
+        media_type="text/markdown",
+        pipeline_version="pipe:v1",
+        metadata={"curated_evidence": evidence.model_dump(mode="json")},
+    )
 def test_chunk_ids_are_stable_for_same_content_and_repeat_runs():
    policy = ChunkPolicy(version="paragraph:v1", max_chars=8)
    first = chunk(document("A\n\nB"), policy)
@@ -116,3 +161,128 @@ def test_empty_document_has_no_chunks_and_invalid_policy_is_rejected():
    assert chunk(document(""), ChunkPolicy(version="v1", max_chars=4)) == []
    with pytest.raises(ValueError):
        ChunkPolicy(version="v1", max_chars=0)
+
+
+def test_typed_formula_is_rendered_as_one_traceable_semantic_fragment():
+    fragments = chunk(curated_formula_document(), ChunkPolicy(version="semantic:v1", max_chars=4000))
+
+    assert len(fragments) == 1
+    fragment = fragments[0]
+    assert "Formula: Fascia pediatrica" in fragment.content
+    assert "Scopi: sql_generation, schema_linking" in fragment.content
+    assert "Concetto: fascia pediatrica" in fragment.content
+    assert "Colonne: clinical.patient.birth_date" in fragment.content
+    assert "SQL: CASE WHEN age < 18 THEN 'pediatric' END" in fragment.content
+    assert "Provenienza: source/paziente.md" in fragment.content
+    assert fragment.metadata["evidence_id"] == "evidence:fascia-pediatrica"
+    assert fragment.metadata["evidence_kind"] == "formula"
+    assert fragment.metadata["purposes"] == ["sql_generation", "schema_linking"]
+    assert fragment.metadata["scope"] == {
+        "concepts": ["fascia pediatrica"],
+        "tables": ["clinical.patient"],
+        "columns": ["clinical.patient.birth_date"],
+    }
+    assert fragment.metadata["language"] == "it"
+    assert fragment.metadata["provenance"]["source_file"] == "source/paziente.md"
+
+
+def test_oversized_typed_atomic_content_fails_instead_of_being_split():
+    document = curated_formula_document(sql="CASE WHEN age < 18 THEN " + "x" * 200 + " END")
+
+    with pytest.raises(ValueError, match="atomic_content_too_large") as caught:
+        chunk(document, ChunkPolicy(version="semantic:v1", max_chars=120))
+
+    assert caught.value.review_item.code == "atomic_content_too_large"
+    assert caught.value.review_item.field == "formula.sql"
+
+
+def test_enum_value_meaning_pairs_are_atomic_and_fragment_ids_are_deterministic():
+    payload = curated_formula_document().metadata["curated_evidence"]
+    evidence = CuratedEvidence.model_validate({
+        **payload,
+        "id": "evidence:stato-ricovero",
+        "title": "Stato ricovero",
+        "kind": "enum",
+        "payload": {
+            "column": "clinical.admission.status",
+            "values": {"A": "Attivo", "D": "Dimesso"},
+        },
+    })
+    document = curated_document(evidence)
+
+    first = chunk(document, ChunkPolicy(version="semantic:v1", max_chars=4000))
+    second = chunk(document, ChunkPolicy(version="semantic:v1", max_chars=4000))
+
+    assert first == second
+    assert [fragment.ordinal for fragment in first] == [0, 1]
+    assert "Valore: A\nSignificato: Attivo" in first[0].content
+    assert "Valore: D\nSignificato: Dimesso" in first[1].content
+
+
+def test_italian_evidence_uses_an_italian_kind_heading():
+    base = curated_formula_document().metadata["curated_evidence"]
+    evidence = CuratedEvidence.model_validate({
+        **base,
+        "id": "evidence:regola-ricovero",
+        "title": "Regola ricovero",
+        "kind": "domain",
+        "payload": {"rule": "Il ricovero richiede una data di ammissione."},
+    })
+
+    fragment = chunk(curated_document(evidence), ChunkPolicy(version="semantic:v1", max_chars=4000))[0]
+
+    assert fragment.content.startswith("Dominio: Regola ricovero\n")
+
+
+def test_domain_rule_remains_atomic_across_blank_paragraphs():
+    base = curated_formula_document().metadata["curated_evidence"]
+    evidence = CuratedEvidence.model_validate({
+        **base,
+        "id": "evidence:regole-ricovero",
+        "title": "Regole ricovero",
+        "kind": "domain",
+        "payload": {"rule": "La data di ammissione è obbligatoria.\n\nLa data di dimissione segue l'ammissione."},
+    })
+
+    fragments = chunk(curated_document(evidence), ChunkPolicy(version="semantic:v1", max_chars=4000))
+
+    assert len(fragments) == 1
+    assert "Regola: La data di ammissione è obbligatoria.\n\nLa data di dimissione segue l'ammissione." in fragments[0].content
+
+
+@pytest.mark.parametrize(
+    ("kind", "payload", "field"),
+    [
+        ("domain", {"rule": "x" * 200}, "domain.rule"),
+        (
+            "mapping",
+            {
+                "concept": "ricovero",
+                "tables": ["clinical.admission"],
+                "columns": ["clinical.admission.status"],
+            },
+            "mapping",
+        ),
+        (
+            "reference",
+            {"url": "https://example.test/guide", "label": "Guida", "description": "x" * 200},
+            "reference.url",
+        ),
+        ("normalization", {"input": "a", "output": "b", "rule": "x" * 200}, "normalization.rule"),
+        ("glossary", {"definition": "x" * 200}, "glossary.definition"),
+        ("example", {"question": "x" * 200, "interpretation": "attesa"}, "example"),
+    ],
+)
+def test_other_typed_atomic_content_blocks_with_a_stable_review_item(kind, payload, field):
+    base = curated_formula_document().metadata["curated_evidence"]
+    evidence = CuratedEvidence.model_validate({
+        **base,
+        "id": f"evidence:{kind}-test",
+        "kind": kind,
+        "payload": payload,
+    })
+
+    with pytest.raises(ValueError, match="atomic_content_too_large") as caught:
+        chunk(curated_document(evidence), ChunkPolicy(version="semantic:v1", max_chars=120))
+
+    assert caught.value.review_item.field == field
@@ -148,6 +148,34 @@ def test_manifest_rejects_inconsistent_pipeline_versions():
        CorpusManifest(pipeline_version="evidence-v1", documents=[wrong])


+def test_typed_evidence_fragment_metadata_must_be_complete_and_allowlisted():
+    metadata = {
+        "evidence_id": "evidence:fascia-pediatrica",
+        "evidence_kind": "formula",
+        "purposes": ["sql_generation"],
+        "scope": {"concepts": ["fascia pediatrica"], "tables": [], "columns": []},
+        "language": "it",
+        "provenance": {
+            "source_file": "source/paziente.md",
+            "source_sha256": "sha256:" + "a" * 64,
+            "supporting_excerpts": ["Pazienti con età inferiore a 18 anni."],
+        },
+    }
+    assert CanonicalChunk.model_validate({**chunk().model_dump(), "metadata": metadata}).metadata == metadata
+
+    with pytest.raises(ValidationError, match="typed Evidence metadata"):
+        CanonicalChunk.model_validate({
+            **chunk().model_dump(),
+            "metadata": {**metadata, "unreviewed_evidence_note": "not allowed"},
+        })
+
+    with pytest.raises(ValidationError, match="typed Evidence metadata"):
+        CanonicalChunk.model_validate({
+            **chunk().model_dump(),
+            "metadata": {"evidence_kind": "formula"},
+        })
+
+
 def test_vector_generation_requires_embedding_compatibility():
    with pytest.raises(ValidationError, match="vector_generation"):
        CorpusManifest(pipeline_version="evidence-v1", vector_generation="generation:one")
@@ -52,6 +52,59 @@ def test_frontmatter_can_end_at_eof_without_inventing_content():
    assert document.content == ""


+def test_normalize_preserves_validated_curated_evidence_for_semantic_projection():
+    source = SourceObject(
+        source_id="filesystem:fascia-pediatrica",
+        uri="file:///safe/evidence/curated/formula/fascia-pediatrica.md",
+        fingerprint="sha256:" + "a" * 64,
+        metadata={"relative_path": "curated/formula/fascia-pediatrica.md"},
+    )
+    raw = (
+        "---\n"
+        "schema_version: 1\n"
+        "id: evidence:fascia-pediatrica\n"
+        "title: Fascia pediatrica\n"
+        "kind: formula\n"
+        "purposes: [sql_generation]\n"
+        "applies_to:\n"
+        "  columns: [clinical.patient.birth_date]\n"
+        "language: it\n"
+        "provenance:\n"
+        "  source_file: source/paziente.md\n"
+        "  source_sha256: sha256:" + "b" * 64 + "\n"
+        "  supporting_excerpts: [Pazienti con età inferiore a 18 anni.]\n"
+        "review_items: []\n"
+        "formula:\n"
+        "  concept: fascia pediatrica\n"
+        "  columns: [clinical.patient.birth_date]\n"
+        "  sql: CASE WHEN age < 18 THEN 'pediatric' END\n"
+        "---\n"
+    ).encode()
+
+    document = normalize(AcquiredDocument(source=source, content=raw, media_type="text/markdown"), "pipe:v1")
+
+    assert document.content == raw.decode()
+    assert document.title == "Fascia pediatrica"
+    assert document.metadata["curated_evidence"]["id"] == "evidence:fascia-pediatrica"
+    assert document.metadata["curated_evidence"]["payload"]["concept"] == "fascia pediatrica"
+
+
+def test_only_curated_markdown_is_treated_as_canonical_evidence_without_relative_metadata():
+    source = SourceObject(
+        source_id="filesystem:notes",
+        uri="file:///safe/evidence/curated/formula/notes.txt",
+        fingerprint="sha256:" + "a" * 64,
+    )
+
+    document = normalize(
+        AcquiredDocument(source=source, content=b"ordinary note", media_type="text/plain"),
+        "pipe:v1",
+    )
+
+    assert document.content == "ordinary note"
+    assert "curated_evidence" not in document.metadata
+
+
@pytest.mark.parametrize(
    "frontmatter",
    [
@@ -2,6 +2,7 @@ from datetime import UTC, datetime, timedelta

 import pytest

+from tht.evidence.canonical import CuratedEvidence, dump_curated_markdown
 from tht.evidence.contracts import AcquiredDocument, SourceObject
 from tht.evidence.corpus.chunk import ChunkPolicy
 from tht.evidence.corpus.models import CanonicalChunk, CanonicalDocument, CorpusManifest
@@ -116,6 +117,131 @@ def pipeline(tmp_path, source, *, embedder=None, vectors=None, model="model-a",
    )


+def test_pipeline_embeds_validated_curated_evidence_as_semantic_fragments(tmp_path):
+    evidence = CuratedEvidence.model_validate(
+        {
+            "schema_version": 1,
+            "id": "evidence:fascia-pediatrica",
+            "title": "Fascia pediatrica",
+            "kind": "formula",
+            "purposes": ["sql_generation"],
+            "applies_to": {"columns": ["clinical.patient.birth_date"]},
+            "language": "it",
+            "provenance": {
+                "source_file": "source/paziente.md",
+                "source_sha256": "sha256:" + "a" * 64,
+                "supporting_excerpts": ["Pazienti con età inferiore a 18 anni."],
+            },
+            "review_items": [],
+            "payload": {
+                "concept": "fascia pediatrica",
+                "columns": ["clinical.patient.birth_date"],
+                "sql": "CASE WHEN age < 18 THEN 'pediatric' END",
+            },
+        }
+    )
+    source_item = SourceObject(
+        source_id="fs:curated-formula",
+        uri="file:///safe/curated/formula/fascia-pediatrica.md",
+        fingerprint="sha256:" + "b" * 64,
+        metadata={"relative_path": "curated/formula/fascia-pediatrica.md"},
+    )
+    embedder = Embedder()
+    vectors = Vectors()
+
+    result = pipeline(
+        tmp_path,
+        Source([(source_item, dump_curated_markdown(evidence))]),
+        embedder=embedder,
+        vectors=vectors,
+        policy=ChunkPolicy(version="chunk-v1", max_chars=4000),
+    ).run()
+
+    assert result.status == "succeeded"
+    assert len(result.manifest.chunks) == 1
+    assert "Formula: Fascia pediatrica" in embedder.calls[0]
+    assert vectors.records[0].record.metadata["evidence_id"] == evidence.id
+    assert vectors.records[0].record.metadata["provenance"]["source_file"] == "source/paziente.md"
+
+
+def test_pipeline_exposes_atomic_content_review_code_when_candidate_is_blocked(tmp_path):
+    evidence = CuratedEvidence.model_validate(
+        {
+            "schema_version": 1,
+            "id": "evidence:formula-lunga",
+            "title": "Formula lunga",
+            "kind": "formula",
+            "purposes": ["sql_generation"],
+            "language": "it",
+            "provenance": {
+                "source_file": "source/paziente.md",
+                "source_sha256": "sha256:" + "a" * 64,
+                "supporting_excerpts": ["Una formula molto lunga."],
+            },
+            "review_items": [],
+            "payload": {"concept": "formula lunga", "columns": [], "sql": "x" * 200},
+        }
+    )
+    source_item = SourceObject(
+        source_id="fs:formula-lunga",
+        uri="file:///safe/curated/formula/formula-lunga.md",
+        fingerprint="sha256:" + "b" * 64,
+        metadata={"relative_path": "curated/formula/formula-lunga.md"},
+    )
+
+    result = pipeline(
+        tmp_path,
+        Source([(source_item, dump_curated_markdown(evidence))]),
+        policy=ChunkPolicy(version="chunk-v1", max_chars=120),
+    ).run()
+
+    assert result.status == "blocked"
+    assert result.published is False
+    assert [item.code for item in result.review_items] == ["atomic_content_too_large"]
+    assert result.review_items[0].field == "formula.sql"
+
+
+def test_job_pipeline_persists_atomic_content_review_item_when_candidate_is_blocked(tmp_path):
+    evidence = CuratedEvidence.model_validate(
+        {
+            "schema_version": 1,
+            "id": "evidence:formula-lunga-job",
+            "title": "Formula lunga",
+            "kind": "formula",
+            "purposes": ["sql_generation"],
+            "language": "it",
+            "provenance": {
+                "source_file": "source/paziente.md",
+                "source_sha256": "sha256:" + "a" * 64,
+                "supporting_excerpts": ["Una formula molto lunga."],
+            },
+            "review_items": [],
+            "payload": {"concept": "formula lunga", "columns": [], "sql": "x" * 200},
+        }
+    )
+    source_item = SourceObject(
+        source_id="fs:formula-lunga-job",
+        uri="file:///safe/curated/formula/formula-lunga-job.md",
+        fingerprint="sha256:" + "b" * 64,
+        metadata={"relative_path": "curated/formula/formula-lunga-job.md"},
+    )
+
+    result = pipeline(
+        tmp_path,
+        Source([(source_item, dump_curated_markdown(evidence))]),
+        policy=ChunkPolicy(version="chunk-v1", max_chars=120),
+    ).run_as_job(
+        workspace_id="demo",
+        workspace_root=tmp_path,
+        config_fingerprint="sha256:" + "1" * 64,
+        input_fingerprint="sha256:" + "2" * 64,
+    )
+
+    assert result.status == "blocked"
+    assert result.published is False
+    assert [item.code for item in result.review_items] == ["atomic_content_too_large"]
+
+
 def test_pipeline_routes_source_io_through_evidence_facade(tmp_path, monkeypatch):
    import tht.evidence.acquisition as evidence_acquisition