import hashlib import pytest from tht.corpus.chunk import ChunkPolicy, chunk from tht.corpus.models import CanonicalDocument def document(content: str) -> CanonicalDocument: normalized = content.replace("\r\n", "\n").replace("\r", "\n") digest = hashlib.sha256(normalized.encode()).hexdigest() return CanonicalDocument( document_id="doc:abc", source_id="source:a", source_uri="https://host/a.md", source_fingerprint="etag:abc", content_hash=f"sha256:{digest}", title="A", content=normalized, media_type="text/markdown", pipeline_version="pipe:v1", metadata={"owner": "docs"}, ) def test_chunk_ids_are_stable_for_same_content_and_repeat_runs(): policy = ChunkPolicy(version="paragraph:v1", max_chars=8) first = chunk(document("A\n\nB"), policy) second = chunk(document("A\r\n\r\nB"), policy) repeated = chunk(document("A\n\nB"), policy) assert first == second == repeated def test_policy_version_changes_ids_without_changing_boundaries(): doc = document("alpha\n\nbeta") first = chunk(doc, ChunkPolicy(version="paragraph:v1", max_chars=6)) second = chunk(doc, ChunkPolicy(version="paragraph:v2", max_chars=6)) assert [item.content for item in first] == [item.content for item in second] assert [item.chunk_id for item in first] != [item.chunk_id for item in second] def test_long_non_ascii_tokens_are_hard_split_by_unicode_characters(): chunks = chunk(document("ééééé世界"), ChunkPolicy(version="chars:v1", max_chars=3)) assert [item.content for item in chunks] == ["ééé", "éé世", "界"] assert all(len(item.content) <= 3 for item in chunks) def test_chunks_have_contiguous_ordinals_hashes_and_provenance_metadata(): doc = document("alpha beta gamma") chunks = chunk(doc, ChunkPolicy(version="words:v1", max_chars=7)) assert [item.ordinal for item in chunks] == list(range(len(chunks))) assert len({item.chunk_id for item in chunks}) == len(chunks) for item in chunks: assert item.source_uri == doc.source_uri assert item.document_id == doc.document_id assert item.pipeline_version == doc.pipeline_version assert item.metadata["chunk_policy"] == {"max_chars": 7, "version": "words:v1"} assert item.metadata["document"] == {"owner": "docs"} assert item.content_hash == "sha256:" + hashlib.sha256(item.content.encode()).hexdigest() def test_duplicate_chunk_content_cannot_collide_across_ordinals(): chunks = chunk(document("same\n\nsame"), ChunkPolicy(version="paragraph:v1", max_chars=8)) assert [item.content for item in chunks] == ["same", "same"] assert chunks[0].chunk_id != chunks[1].chunk_id def test_empty_document_has_no_chunks_and_invalid_policy_is_rejected(): assert chunk(document(""), ChunkPolicy(version="v1", max_chars=4)) == [] with pytest.raises(ValueError): ChunkPolicy(version="v1", max_chars=0)