feat(corpus): add deterministic normalization and chunking

harness/tests/test_corpus_chunk.py

@@ -0,0 +1,71 @@
+import hashlib
+
+import pytest
+
+from tht.corpus.chunk import ChunkPolicy, chunk
+from tht.corpus.models import CanonicalDocument
+
+
+def document(content: str) -> CanonicalDocument:
+    normalized = content.replace("\r\n", "\n").replace("\r", "\n")
+    digest = hashlib.sha256(normalized.encode()).hexdigest()
+    return CanonicalDocument(
+        document_id="doc:abc",
+        source_id="source:a",
+        source_uri="https://host/a.md",
+        source_fingerprint="etag:abc",
+        content_hash=f"sha256:{digest}",
+        title="A",
+        content=normalized,
+        media_type="text/markdown",
+        pipeline_version="pipe:v1",
+        metadata={"owner": "docs"},
+    )
+
+
+def test_chunk_ids_are_stable_for_same_content_and_repeat_runs():
+    policy = ChunkPolicy(version="paragraph:v1", max_chars=8)
+    first = chunk(document("A\n\nB"), policy)
+    second = chunk(document("A\r\n\r\nB"), policy)
+    repeated = chunk(document("A\n\nB"), policy)
+    assert first == second == repeated
+
+
+def test_policy_version_changes_ids_without_changing_boundaries():
+    doc = document("alpha\n\nbeta")
+    first = chunk(doc, ChunkPolicy(version="paragraph:v1", max_chars=6))
+    second = chunk(doc, ChunkPolicy(version="paragraph:v2", max_chars=6))
+    assert [item.content for item in first] == [item.content for item in second]
+    assert [item.chunk_id for item in first] != [item.chunk_id for item in second]
+
+
+def test_long_non_ascii_tokens_are_hard_split_by_unicode_characters():
+    chunks = chunk(document("ééééé世界"), ChunkPolicy(version="chars:v1", max_chars=3))
+    assert [item.content for item in chunks] == ["ééé", "éé世", "界"]
+    assert all(len(item.content) <= 3 for item in chunks)
+
+
+def test_chunks_have_contiguous_ordinals_hashes_and_provenance_metadata():
+    doc = document("alpha beta gamma")
+    chunks = chunk(doc, ChunkPolicy(version="words:v1", max_chars=7))
+    assert [item.ordinal for item in chunks] == list(range(len(chunks)))
+    assert len({item.chunk_id for item in chunks}) == len(chunks)
+    for item in chunks:
+        assert item.source_uri == doc.source_uri
+        assert item.document_id == doc.document_id
+        assert item.pipeline_version == doc.pipeline_version
+        assert item.metadata["chunk_policy"] == {"max_chars": 7, "version": "words:v1"}
+        assert item.metadata["document"] == {"owner": "docs"}
+        assert item.content_hash == "sha256:" + hashlib.sha256(item.content.encode()).hexdigest()
+
+
+def test_duplicate_chunk_content_cannot_collide_across_ordinals():
+    chunks = chunk(document("same\n\nsame"), ChunkPolicy(version="paragraph:v1", max_chars=8))
+    assert [item.content for item in chunks] == ["same", "same"]
+    assert chunks[0].chunk_id != chunks[1].chunk_id
+
+
+def test_empty_document_has_no_chunks_and_invalid_policy_is_rejected():
+    assert chunk(document(""), ChunkPolicy(version="v1", max_chars=4)) == []
+    with pytest.raises(ValueError):
+        ChunkPolicy(version="v1", max_chars=0)

harness/tests/test_corpus_normalize.py

@@ -0,0 +1,67 @@
+import hashlib
+from datetime import UTC, datetime
+
+import pytest
+
+from tht.corpus.normalize import MAX_DOCUMENT_BYTES, PermanentNormalizationError, normalize
+from tht.ports.evidence import AcquiredDocument, SourceObject
+
+
+def acquired(content: bytes, *, media_type: str = "text/markdown") -> AcquiredDocument:
+    return AcquiredDocument(
+        source=SourceObject(
+            source_id="source:guide",
+            uri="https://host/guide.md?signature=transport#part",
+            fingerprint="etag:abc",
+            modified_at=datetime(2026, 7, 12, 12, 0, tzinfo=UTC),
+            metadata={"owner": "docs"},
+        ),
+        content=content,
+        media_type=media_type,
+        metadata={"transport": "http"},
+    )
+
+
+def test_normalize_utf8_bom_newlines_unicode_and_frontmatter():
+    raw = (
+        "\ufeff---\r\ntitle: Café\r\ntags: [uno, due]\r\n---\r\n"
+        "Cafe\u0301\rBody\r\n"
+    ).encode()
+
+    document = normalize(acquired(raw, media_type="text/markdown; charset=UTF-8"), "pipe:v1")
+
+    assert document.content == "Café\nBody\n"
+    assert document.title == "Café"
+    assert document.metadata["frontmatter"] == {"tags": ("uno", "due"), "title": "Café"}
+    assert document.metadata["source"] == {"owner": "docs"}
+    assert document.metadata["acquisition"] == {"transport": "http"}
+    assert document.source_uri == "https://host/guide.md"
+    assert document.modified_at == datetime(2026, 7, 12, 12, 0, tzinfo=UTC)
+    assert document.content_hash == "sha256:" + hashlib.sha256(document.content.encode()).hexdigest()
+
+
+def test_plain_text_that_only_resembles_frontmatter_is_not_dropped():
+    document = normalize(acquired(b"---\nnot: closed\nbody"), "pipe:v1")
+    assert document.content == "---\nnot: closed\nbody"
+    assert "frontmatter" not in document.metadata
+
+
+def test_frontmatter_can_end_at_eof_without_inventing_content():
+    document = normalize(acquired(b"---\ntitle: Empty\n---"), "pipe:v1")
+    assert document.title == "Empty"
+    assert document.content == ""
+
+
+@pytest.mark.parametrize(
+    ("content", "media_type", "reason"),
+    [
+        (b"bad: \xff", "text/plain", "undecodable"),
+        (b"hello", "text/plain; charset=iso-8859-1", "unsupported_charset"),
+        (b"x" * (MAX_DOCUMENT_BYTES + 1), "text/plain", "oversized"),
+    ],
+)
+def test_rejects_invalid_input_as_typed_permanent_error(content, media_type, reason):
+    with pytest.raises(PermanentNormalizationError) as caught:
+        normalize(acquired(content, media_type=media_type), "pipe:v1")
+    assert caught.value.permanent is True
+    assert caught.value.reason == reason