289 lines
11 KiB
Python
289 lines
11 KiB
Python
import hashlib
|
|
|
|
import pytest
|
|
|
|
from tht.evidence.canonical import CuratedEvidence
|
|
from tht.evidence.corpus.chunk import ChunkPolicy, chunk
|
|
from tht.evidence.corpus.models import CanonicalDocument, CorpusManifest
|
|
|
|
|
|
def document(content: str) -> CanonicalDocument:
|
|
normalized = content.replace("\r\n", "\n").replace("\r", "\n")
|
|
digest = hashlib.sha256(normalized.encode()).hexdigest()
|
|
return CanonicalDocument(
|
|
document_id="doc:abc",
|
|
source_id="source:a",
|
|
source_uri="https://host/a.md",
|
|
source_fingerprint="etag:abc",
|
|
content_hash=f"sha256:{digest}",
|
|
title="A",
|
|
content=normalized,
|
|
media_type="text/markdown",
|
|
pipeline_version="pipe:v1",
|
|
metadata={"owner": "docs"},
|
|
)
|
|
|
|
|
|
def other_document(content: str) -> CanonicalDocument:
|
|
return document(content).model_copy(
|
|
update={
|
|
"document_id": "doc:def",
|
|
"source_id": "source:b",
|
|
"source_uri": "https://host/b.md",
|
|
}
|
|
)
|
|
|
|
|
|
def curated_formula_document(*, sql: str = "CASE WHEN age < 18 THEN 'pediatric' END") -> CanonicalDocument:
|
|
evidence = CuratedEvidence.model_validate(
|
|
{
|
|
"schema_version": 1,
|
|
"id": "evidence:fascia-pediatrica",
|
|
"title": "Fascia pediatrica",
|
|
"kind": "formula",
|
|
"purposes": ["sql_generation", "schema_linking"],
|
|
"applies_to": {
|
|
"concepts": ["fascia pediatrica"],
|
|
"tables": ["clinical.patient"],
|
|
"columns": ["clinical.patient.birth_date"],
|
|
},
|
|
"language": "it",
|
|
"provenance": {
|
|
"source_file": "source/paziente.md",
|
|
"source_sha256": "sha256:" + "a" * 64,
|
|
"supporting_excerpts": ["I pazienti pediatrici hanno età inferiore a 18 anni."],
|
|
},
|
|
"review_items": [],
|
|
"payload": {
|
|
"concept": "fascia pediatrica",
|
|
"columns": ["clinical.patient.birth_date"],
|
|
"sql": sql,
|
|
},
|
|
}
|
|
)
|
|
return curated_document(evidence)
|
|
|
|
|
|
def curated_document(evidence: CuratedEvidence) -> CanonicalDocument:
|
|
content = "canonical curated Evidence"
|
|
return CanonicalDocument(
|
|
document_id="doc:" + evidence.id.removeprefix("evidence:"),
|
|
source_id="curated:" + evidence.id.removeprefix("evidence:"),
|
|
source_uri=f"file:///safe/curated/{evidence.kind}/{evidence.id.removeprefix('evidence:')}.md",
|
|
source_fingerprint="sha256:" + "b" * 64,
|
|
content_hash="sha256:" + hashlib.sha256(content.encode()).hexdigest(),
|
|
title=evidence.title,
|
|
content=content,
|
|
media_type="text/markdown",
|
|
pipeline_version="pipe:v1",
|
|
metadata={"curated_evidence": evidence.model_dump(mode="json")},
|
|
)
|
|
def test_chunk_ids_are_stable_for_same_content_and_repeat_runs():
|
|
policy = ChunkPolicy(version="paragraph:v1", max_chars=8)
|
|
first = chunk(document("A\n\nB"), policy)
|
|
second = chunk(document("A\r\n\r\nB"), policy)
|
|
repeated = chunk(document("A\n\nB"), policy)
|
|
assert first == second == repeated
|
|
|
|
|
|
def test_policy_version_changes_ids_without_changing_boundaries():
|
|
doc = document("alpha\n\nbeta")
|
|
first = chunk(doc, ChunkPolicy(version="paragraph:v1", max_chars=6))
|
|
second = chunk(doc, ChunkPolicy(version="paragraph:v2", max_chars=6))
|
|
assert [item.content for item in first] == [item.content for item in second]
|
|
assert [item.chunk_id for item in first] != [item.chunk_id for item in second]
|
|
|
|
|
|
def test_same_policy_version_with_different_boundary_config_changes_ids():
|
|
doc = document("alpha beta")
|
|
first = chunk(doc, ChunkPolicy(version="paragraph:v1", max_chars=6))
|
|
second = chunk(doc, ChunkPolicy(version="paragraph:v1", max_chars=7))
|
|
assert first[0].chunk_id != second[0].chunk_id
|
|
|
|
|
|
def test_identical_content_in_different_documents_cannot_collide_in_manifest():
|
|
policy = ChunkPolicy(version="paragraph:v1", max_chars=20)
|
|
first = document("same")
|
|
second = other_document("same")
|
|
chunks = [*chunk(first, policy), *chunk(second, policy)]
|
|
manifest = CorpusManifest(
|
|
pipeline_version="pipe:v1", documents=[first, second], chunks=chunks
|
|
)
|
|
assert len({item.chunk_id for item in manifest.chunks}) == 2
|
|
|
|
|
|
def test_long_non_ascii_tokens_are_hard_split_by_unicode_characters():
|
|
chunks = chunk(document("ééééé世界"), ChunkPolicy(version="chars:v1", max_chars=3))
|
|
assert [item.content for item in chunks] == ["ééé", "éé世", "界"]
|
|
assert all(len(item.content) <= 3 for item in chunks)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"content",
|
|
[
|
|
"alpha beta\tgamma\n\ndelta",
|
|
"line with markdown hard break \nnext line\n```\na b\n```",
|
|
" \t\n\n \n",
|
|
"supercalifragilisticexpialidocious",
|
|
"é 世界\r\nnext",
|
|
],
|
|
)
|
|
def test_chunks_preserve_every_character_and_respect_max_chars(content):
|
|
doc = document(content)
|
|
chunks = chunk(doc, ChunkPolicy(version="exact:v1", max_chars=9))
|
|
assert "".join(item.content for item in chunks) == doc.content
|
|
assert all(0 < len(item.content) <= 9 for item in chunks)
|
|
|
|
|
|
def test_chunks_have_contiguous_ordinals_hashes_and_provenance_metadata():
|
|
doc = document("alpha beta gamma")
|
|
chunks = chunk(doc, ChunkPolicy(version="words:v1", max_chars=7))
|
|
assert [item.ordinal for item in chunks] == list(range(len(chunks)))
|
|
assert len({item.chunk_id for item in chunks}) == len(chunks)
|
|
for item in chunks:
|
|
assert item.source_uri == doc.source_uri
|
|
assert item.document_id == doc.document_id
|
|
assert item.pipeline_version == doc.pipeline_version
|
|
assert item.metadata["chunk_policy"]["max_chars"] == 7
|
|
assert item.metadata["chunk_policy"]["version"] == "words:v1"
|
|
assert item.metadata["chunk_policy"]["fingerprint"].startswith("sha256:")
|
|
assert item.metadata["document"] == {"owner": "docs"}
|
|
assert item.content_hash == "sha256:" + hashlib.sha256(item.content.encode()).hexdigest()
|
|
|
|
|
|
def test_duplicate_chunk_content_cannot_collide_across_ordinals():
|
|
chunks = chunk(document("samesame"), ChunkPolicy(version="paragraph:v1", max_chars=4))
|
|
assert [item.content for item in chunks] == ["same", "same"]
|
|
assert chunks[0].chunk_id != chunks[1].chunk_id
|
|
|
|
|
|
def test_empty_document_has_no_chunks_and_invalid_policy_is_rejected():
|
|
assert chunk(document(""), ChunkPolicy(version="v1", max_chars=4)) == []
|
|
with pytest.raises(ValueError):
|
|
ChunkPolicy(version="v1", max_chars=0)
|
|
|
|
|
|
def test_typed_formula_is_rendered_as_one_traceable_semantic_fragment():
|
|
fragments = chunk(curated_formula_document(), ChunkPolicy(version="semantic:v1", max_chars=4000))
|
|
|
|
assert len(fragments) == 1
|
|
fragment = fragments[0]
|
|
assert "Formula: Fascia pediatrica" in fragment.content
|
|
assert "Scopi: sql_generation, schema_linking" in fragment.content
|
|
assert "Concetto: fascia pediatrica" in fragment.content
|
|
assert "Colonne: clinical.patient.birth_date" in fragment.content
|
|
assert "SQL: CASE WHEN age < 18 THEN 'pediatric' END" in fragment.content
|
|
assert "Provenienza: source/paziente.md" in fragment.content
|
|
assert fragment.metadata["evidence_id"] == "evidence:fascia-pediatrica"
|
|
assert fragment.metadata["evidence_kind"] == "formula"
|
|
assert fragment.metadata["purposes"] == ["sql_generation", "schema_linking"]
|
|
assert fragment.metadata["scope"] == {
|
|
"concepts": ["fascia pediatrica"],
|
|
"tables": ["clinical.patient"],
|
|
"columns": ["clinical.patient.birth_date"],
|
|
}
|
|
assert fragment.metadata["language"] == "it"
|
|
assert fragment.metadata["provenance"]["source_file"] == "source/paziente.md"
|
|
|
|
|
|
def test_oversized_typed_atomic_content_fails_instead_of_being_split():
|
|
document = curated_formula_document(sql="CASE WHEN age < 18 THEN " + "x" * 200 + " END")
|
|
|
|
with pytest.raises(ValueError, match="atomic_content_too_large") as caught:
|
|
chunk(document, ChunkPolicy(version="semantic:v1", max_chars=120))
|
|
|
|
assert caught.value.review_item.code == "atomic_content_too_large"
|
|
assert caught.value.review_item.field == "formula.sql"
|
|
|
|
|
|
def test_enum_value_meaning_pairs_are_atomic_and_fragment_ids_are_deterministic():
|
|
payload = curated_formula_document().metadata["curated_evidence"]
|
|
evidence = CuratedEvidence.model_validate({
|
|
**payload,
|
|
"id": "evidence:stato-ricovero",
|
|
"title": "Stato ricovero",
|
|
"kind": "enum",
|
|
"payload": {
|
|
"column": "clinical.admission.status",
|
|
"values": {"A": "Attivo", "D": "Dimesso"},
|
|
},
|
|
})
|
|
document = curated_document(evidence)
|
|
|
|
first = chunk(document, ChunkPolicy(version="semantic:v1", max_chars=4000))
|
|
second = chunk(document, ChunkPolicy(version="semantic:v1", max_chars=4000))
|
|
|
|
assert first == second
|
|
assert [fragment.ordinal for fragment in first] == [0, 1]
|
|
assert "Valore: A\nSignificato: Attivo" in first[0].content
|
|
assert "Valore: D\nSignificato: Dimesso" in first[1].content
|
|
|
|
|
|
def test_italian_evidence_uses_an_italian_kind_heading():
|
|
base = curated_formula_document().metadata["curated_evidence"]
|
|
evidence = CuratedEvidence.model_validate({
|
|
**base,
|
|
"id": "evidence:regola-ricovero",
|
|
"title": "Regola ricovero",
|
|
"kind": "domain",
|
|
"payload": {"rule": "Il ricovero richiede una data di ammissione."},
|
|
})
|
|
|
|
fragment = chunk(curated_document(evidence), ChunkPolicy(version="semantic:v1", max_chars=4000))[0]
|
|
|
|
assert fragment.content.startswith("Dominio: Regola ricovero\n")
|
|
|
|
|
|
def test_domain_rule_remains_atomic_across_blank_paragraphs():
|
|
base = curated_formula_document().metadata["curated_evidence"]
|
|
evidence = CuratedEvidence.model_validate({
|
|
**base,
|
|
"id": "evidence:regole-ricovero",
|
|
"title": "Regole ricovero",
|
|
"kind": "domain",
|
|
"payload": {"rule": "La data di ammissione è obbligatoria.\n\nLa data di dimissione segue l'ammissione."},
|
|
})
|
|
|
|
fragments = chunk(curated_document(evidence), ChunkPolicy(version="semantic:v1", max_chars=4000))
|
|
|
|
assert len(fragments) == 1
|
|
assert "Regola: La data di ammissione è obbligatoria.\n\nLa data di dimissione segue l'ammissione." in fragments[0].content
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("kind", "payload", "field"),
|
|
[
|
|
("domain", {"rule": "x" * 200}, "domain.rule"),
|
|
(
|
|
"mapping",
|
|
{
|
|
"concept": "ricovero",
|
|
"tables": ["clinical.admission"],
|
|
"columns": ["clinical.admission.status"],
|
|
},
|
|
"mapping",
|
|
),
|
|
(
|
|
"reference",
|
|
{"url": "https://example.test/guide", "label": "Guida", "description": "x" * 200},
|
|
"reference.url",
|
|
),
|
|
("normalization", {"input": "a", "output": "b", "rule": "x" * 200}, "normalization.rule"),
|
|
("glossary", {"definition": "x" * 200}, "glossary.definition"),
|
|
("example", {"question": "x" * 200, "interpretation": "attesa"}, "example"),
|
|
],
|
|
)
|
|
def test_other_typed_atomic_content_blocks_with_a_stable_review_item(kind, payload, field):
|
|
base = curated_formula_document().metadata["curated_evidence"]
|
|
evidence = CuratedEvidence.model_validate({
|
|
**base,
|
|
"id": f"evidence:{kind}-test",
|
|
"kind": kind,
|
|
"payload": payload,
|
|
})
|
|
|
|
with pytest.raises(ValueError, match="atomic_content_too_large") as caught:
|
|
chunk(curated_document(evidence), ChunkPolicy(version="semantic:v1", max_chars=120))
|
|
|
|
assert caught.value.review_item.field == field
|