feat(corpus): add deterministic normalization and chunking

harness/tests/test_corpus_chunk.py

@@ -0,0 +1,71 @@
+import hashlib
+
+import pytest
+
+from tht.corpus.chunk import ChunkPolicy, chunk
+from tht.corpus.models import CanonicalDocument
+
+
+def document(content: str) -> CanonicalDocument:
+    normalized = content.replace("\r\n", "\n").replace("\r", "\n")
+    digest = hashlib.sha256(normalized.encode()).hexdigest()
+    return CanonicalDocument(
+        document_id="doc:abc",
+        source_id="source:a",
+        source_uri="https://host/a.md",
+        source_fingerprint="etag:abc",
+        content_hash=f"sha256:{digest}",
+        title="A",
+        content=normalized,
+        media_type="text/markdown",
+        pipeline_version="pipe:v1",
+        metadata={"owner": "docs"},
+    )
+
+
+def test_chunk_ids_are_stable_for_same_content_and_repeat_runs():
+    policy = ChunkPolicy(version="paragraph:v1", max_chars=8)
+    first = chunk(document("A\n\nB"), policy)
+    second = chunk(document("A\r\n\r\nB"), policy)
+    repeated = chunk(document("A\n\nB"), policy)
+    assert first == second == repeated
+
+
+def test_policy_version_changes_ids_without_changing_boundaries():
+    doc = document("alpha\n\nbeta")
+    first = chunk(doc, ChunkPolicy(version="paragraph:v1", max_chars=6))
+    second = chunk(doc, ChunkPolicy(version="paragraph:v2", max_chars=6))
+    assert [item.content for item in first] == [item.content for item in second]
+    assert [item.chunk_id for item in first] != [item.chunk_id for item in second]
+
+
+def test_long_non_ascii_tokens_are_hard_split_by_unicode_characters():
+    chunks = chunk(document("ééééé世界"), ChunkPolicy(version="chars:v1", max_chars=3))
+    assert [item.content for item in chunks] == ["ééé", "éé世", "界"]
+    assert all(len(item.content) <= 3 for item in chunks)
+
+
+def test_chunks_have_contiguous_ordinals_hashes_and_provenance_metadata():
+    doc = document("alpha beta gamma")
+    chunks = chunk(doc, ChunkPolicy(version="words:v1", max_chars=7))
+    assert [item.ordinal for item in chunks] == list(range(len(chunks)))
+    assert len({item.chunk_id for item in chunks}) == len(chunks)
+    for item in chunks:
+        assert item.source_uri == doc.source_uri
+        assert item.document_id == doc.document_id
+        assert item.pipeline_version == doc.pipeline_version
+        assert item.metadata["chunk_policy"] == {"max_chars": 7, "version": "words:v1"}
+        assert item.metadata["document"] == {"owner": "docs"}
+        assert item.content_hash == "sha256:" + hashlib.sha256(item.content.encode()).hexdigest()
+
+
+def test_duplicate_chunk_content_cannot_collide_across_ordinals():
+    chunks = chunk(document("same\n\nsame"), ChunkPolicy(version="paragraph:v1", max_chars=8))
+    assert [item.content for item in chunks] == ["same", "same"]
+    assert chunks[0].chunk_id != chunks[1].chunk_id
+
+
+def test_empty_document_has_no_chunks_and_invalid_policy_is_rejected():
+    assert chunk(document(""), ChunkPolicy(version="v1", max_chars=4)) == []
+    with pytest.raises(ValueError):
+        ChunkPolicy(version="v1", max_chars=0)