import hashlib import pytest from tht.evidence.canonical import CuratedEvidence from tht.evidence.corpus.chunk import ChunkPolicy, chunk from tht.evidence.corpus.models import CanonicalDocument, CorpusManifest def document(content: str) -> CanonicalDocument: normalized = content.replace("\r\n", "\n").replace("\r", "\n") digest = hashlib.sha256(normalized.encode()).hexdigest() return CanonicalDocument( document_id="doc:abc", source_id="source:a", source_uri="https://host/a.md", source_fingerprint="etag:abc", content_hash=f"sha256:{digest}", title="A", content=normalized, media_type="text/markdown", pipeline_version="pipe:v1", metadata={"owner": "docs"}, ) def other_document(content: str) -> CanonicalDocument: return document(content).model_copy( update={ "document_id": "doc:def", "source_id": "source:b", "source_uri": "https://host/b.md", } ) def curated_formula_document(*, sql: str = "CASE WHEN age < 18 THEN 'pediatric' END") -> CanonicalDocument: evidence = CuratedEvidence.model_validate( { "schema_version": 1, "id": "evidence:fascia-pediatrica", "title": "Fascia pediatrica", "kind": "formula", "purposes": ["sql_generation", "schema_linking"], "applies_to": { "concepts": ["fascia pediatrica"], "tables": ["clinical.patient"], "columns": ["clinical.patient.birth_date"], }, "language": "it", "provenance": { "source_file": "source/paziente.md", "source_sha256": "sha256:" + "a" * 64, "supporting_excerpts": ["I pazienti pediatrici hanno età inferiore a 18 anni."], }, "review_items": [], "payload": { "concept": "fascia pediatrica", "columns": ["clinical.patient.birth_date"], "sql": sql, }, } ) return curated_document(evidence) def curated_document(evidence: CuratedEvidence) -> CanonicalDocument: content = "canonical curated Evidence" return CanonicalDocument( document_id="doc:" + evidence.id.removeprefix("evidence:"), source_id="curated:" + evidence.id.removeprefix("evidence:"), source_uri=f"file:///safe/curated/{evidence.kind}/{evidence.id.removeprefix('evidence:')}.md", source_fingerprint="sha256:" + "b" * 64, content_hash="sha256:" + hashlib.sha256(content.encode()).hexdigest(), title=evidence.title, content=content, media_type="text/markdown", pipeline_version="pipe:v1", metadata={"curated_evidence": evidence.model_dump(mode="json")}, ) def test_chunk_ids_are_stable_for_same_content_and_repeat_runs(): policy = ChunkPolicy(version="paragraph:v1", max_chars=8) first = chunk(document("A\n\nB"), policy) second = chunk(document("A\r\n\r\nB"), policy) repeated = chunk(document("A\n\nB"), policy) assert first == second == repeated def test_policy_version_changes_ids_without_changing_boundaries(): doc = document("alpha\n\nbeta") first = chunk(doc, ChunkPolicy(version="paragraph:v1", max_chars=6)) second = chunk(doc, ChunkPolicy(version="paragraph:v2", max_chars=6)) assert [item.content for item in first] == [item.content for item in second] assert [item.chunk_id for item in first] != [item.chunk_id for item in second] def test_same_policy_version_with_different_boundary_config_changes_ids(): doc = document("alpha beta") first = chunk(doc, ChunkPolicy(version="paragraph:v1", max_chars=6)) second = chunk(doc, ChunkPolicy(version="paragraph:v1", max_chars=7)) assert first[0].chunk_id != second[0].chunk_id def test_identical_content_in_different_documents_cannot_collide_in_manifest(): policy = ChunkPolicy(version="paragraph:v1", max_chars=20) first = document("same") second = other_document("same") chunks = [*chunk(first, policy), *chunk(second, policy)] manifest = CorpusManifest( pipeline_version="pipe:v1", documents=[first, second], chunks=chunks ) assert len({item.chunk_id for item in manifest.chunks}) == 2 def test_long_non_ascii_tokens_are_hard_split_by_unicode_characters(): chunks = chunk(document("ééééé世界"), ChunkPolicy(version="chars:v1", max_chars=3)) assert [item.content for item in chunks] == ["ééé", "éé世", "界"] assert all(len(item.content) <= 3 for item in chunks) @pytest.mark.parametrize( "content", [ "alpha beta\tgamma\n\ndelta", "line with markdown hard break \nnext line\n```\na b\n```", " \t\n\n \n", "supercalifragilisticexpialidocious", "é 世界\r\nnext", ], ) def test_chunks_preserve_every_character_and_respect_max_chars(content): doc = document(content) chunks = chunk(doc, ChunkPolicy(version="exact:v1", max_chars=9)) assert "".join(item.content for item in chunks) == doc.content assert all(0 < len(item.content) <= 9 for item in chunks) def test_chunks_have_contiguous_ordinals_hashes_and_provenance_metadata(): doc = document("alpha beta gamma") chunks = chunk(doc, ChunkPolicy(version="words:v1", max_chars=7)) assert [item.ordinal for item in chunks] == list(range(len(chunks))) assert len({item.chunk_id for item in chunks}) == len(chunks) for item in chunks: assert item.source_uri == doc.source_uri assert item.document_id == doc.document_id assert item.pipeline_version == doc.pipeline_version assert item.metadata["chunk_policy"]["max_chars"] == 7 assert item.metadata["chunk_policy"]["version"] == "words:v1" assert item.metadata["chunk_policy"]["fingerprint"].startswith("sha256:") assert item.metadata["document"] == {"owner": "docs"} assert item.content_hash == "sha256:" + hashlib.sha256(item.content.encode()).hexdigest() def test_duplicate_chunk_content_cannot_collide_across_ordinals(): chunks = chunk(document("samesame"), ChunkPolicy(version="paragraph:v1", max_chars=4)) assert [item.content for item in chunks] == ["same", "same"] assert chunks[0].chunk_id != chunks[1].chunk_id def test_empty_document_has_no_chunks_and_invalid_policy_is_rejected(): assert chunk(document(""), ChunkPolicy(version="v1", max_chars=4)) == [] with pytest.raises(ValueError): ChunkPolicy(version="v1", max_chars=0) def test_typed_formula_is_rendered_as_one_traceable_semantic_fragment(): fragments = chunk(curated_formula_document(), ChunkPolicy(version="semantic:v1", max_chars=4000)) assert len(fragments) == 1 fragment = fragments[0] assert "Formula: Fascia pediatrica" in fragment.content assert "Scopi: sql_generation, schema_linking" in fragment.content assert "Concetto: fascia pediatrica" in fragment.content assert "Colonne: clinical.patient.birth_date" in fragment.content assert "SQL: CASE WHEN age < 18 THEN 'pediatric' END" in fragment.content assert "Provenienza: source/paziente.md" in fragment.content assert fragment.metadata["evidence_id"] == "evidence:fascia-pediatrica" assert fragment.metadata["evidence_kind"] == "formula" assert fragment.metadata["purposes"] == ["sql_generation", "schema_linking"] assert fragment.metadata["scope"] == { "concepts": ["fascia pediatrica"], "tables": ["clinical.patient"], "columns": ["clinical.patient.birth_date"], } assert fragment.metadata["language"] == "it" assert fragment.metadata["provenance"]["source_file"] == "source/paziente.md" def test_oversized_typed_atomic_content_fails_instead_of_being_split(): document = curated_formula_document(sql="CASE WHEN age < 18 THEN " + "x" * 200 + " END") with pytest.raises(ValueError, match="atomic_content_too_large") as caught: chunk(document, ChunkPolicy(version="semantic:v1", max_chars=120)) assert caught.value.review_item.code == "atomic_content_too_large" assert caught.value.review_item.field == "formula.sql" def test_enum_value_meaning_pairs_are_atomic_and_fragment_ids_are_deterministic(): payload = curated_formula_document().metadata["curated_evidence"] evidence = CuratedEvidence.model_validate({ **payload, "id": "evidence:stato-ricovero", "title": "Stato ricovero", "kind": "enum", "payload": { "column": "clinical.admission.status", "values": {"A": "Attivo", "D": "Dimesso"}, }, }) document = curated_document(evidence) first = chunk(document, ChunkPolicy(version="semantic:v1", max_chars=4000)) second = chunk(document, ChunkPolicy(version="semantic:v1", max_chars=4000)) assert first == second assert [fragment.ordinal for fragment in first] == [0, 1] assert "Valore: A\nSignificato: Attivo" in first[0].content assert "Valore: D\nSignificato: Dimesso" in first[1].content def test_italian_evidence_uses_an_italian_kind_heading(): base = curated_formula_document().metadata["curated_evidence"] evidence = CuratedEvidence.model_validate({ **base, "id": "evidence:regola-ricovero", "title": "Regola ricovero", "kind": "domain", "payload": {"rule": "Il ricovero richiede una data di ammissione."}, }) fragment = chunk(curated_document(evidence), ChunkPolicy(version="semantic:v1", max_chars=4000))[0] assert fragment.content.startswith("Dominio: Regola ricovero\n") def test_domain_rule_remains_atomic_across_blank_paragraphs(): base = curated_formula_document().metadata["curated_evidence"] evidence = CuratedEvidence.model_validate({ **base, "id": "evidence:regole-ricovero", "title": "Regole ricovero", "kind": "domain", "payload": {"rule": "La data di ammissione è obbligatoria.\n\nLa data di dimissione segue l'ammissione."}, }) fragments = chunk(curated_document(evidence), ChunkPolicy(version="semantic:v1", max_chars=4000)) assert len(fragments) == 1 assert "Regola: La data di ammissione è obbligatoria.\n\nLa data di dimissione segue l'ammissione." in fragments[0].content @pytest.mark.parametrize( ("kind", "payload", "field"), [ ("domain", {"rule": "x" * 200}, "domain.rule"), ( "mapping", { "concept": "ricovero", "tables": ["clinical.admission"], "columns": ["clinical.admission.status"], }, "mapping", ), ( "reference", {"url": "https://example.test/guide", "label": "Guida", "description": "x" * 200}, "reference.url", ), ("normalization", {"input": "a", "output": "b", "rule": "x" * 200}, "normalization.rule"), ("glossary", {"definition": "x" * 200}, "glossary.definition"), ("example", {"question": "x" * 200, "interpretation": "attesa"}, "example"), ], ) def test_other_typed_atomic_content_blocks_with_a_stable_review_item(kind, payload, field): base = curated_formula_document().metadata["curated_evidence"] evidence = CuratedEvidence.model_validate({ **base, "id": f"evidence:{kind}-test", "kind": kind, "payload": payload, }) with pytest.raises(ValueError, match="atomic_content_too_large") as caught: chunk(curated_document(evidence), ChunkPolicy(version="semantic:v1", max_chars=120)) assert caught.value.review_item.field == field