fix(corpus): harden canonical chunk and frontmatter invariants

harness/tests/test_corpus_chunk.py

@@ -3,7 +3,7 @@ import hashlib
 import pytest
 
 from tht.corpus.chunk import ChunkPolicy, chunk
-from tht.corpus.models import CanonicalDocument
+from tht.corpus.models import CanonicalDocument, CorpusManifest
 
 
 def document(content: str) -> CanonicalDocument:
@@ -23,6 +23,16 @@ def document(content: str) -> CanonicalDocument:
     )
 
 
+def other_document(content: str) -> CanonicalDocument:
+    return document(content).model_copy(
+        update={
+            "document_id": "doc:def",
+            "source_id": "source:b",
+            "source_uri": "https://host/b.md",
+        }
+    )
+
+
 def test_chunk_ids_are_stable_for_same_content_and_repeat_runs():
     policy = ChunkPolicy(version="paragraph:v1", max_chars=8)
     first = chunk(document("A\n\nB"), policy)
@@ -39,12 +49,47 @@ def test_policy_version_changes_ids_without_changing_boundaries():
     assert [item.chunk_id for item in first] != [item.chunk_id for item in second]
 
 
+def test_same_policy_version_with_different_boundary_config_changes_ids():
+    doc = document("alpha beta")
+    first = chunk(doc, ChunkPolicy(version="paragraph:v1", max_chars=6))
+    second = chunk(doc, ChunkPolicy(version="paragraph:v1", max_chars=7))
+    assert first[0].chunk_id != second[0].chunk_id
+
+
+def test_identical_content_in_different_documents_cannot_collide_in_manifest():
+    policy = ChunkPolicy(version="paragraph:v1", max_chars=20)
+    first = document("same")
+    second = other_document("same")
+    chunks = [*chunk(first, policy), *chunk(second, policy)]
+    manifest = CorpusManifest(
+        pipeline_version="pipe:v1", documents=[first, second], chunks=chunks
+    )
+    assert len({item.chunk_id for item in manifest.chunks}) == 2
+
+
 def test_long_non_ascii_tokens_are_hard_split_by_unicode_characters():
     chunks = chunk(document("ééééé世界"), ChunkPolicy(version="chars:v1", max_chars=3))
     assert [item.content for item in chunks] == ["ééé", "éé世", "界"]
     assert all(len(item.content) <= 3 for item in chunks)
 
 
+@pytest.mark.parametrize(
+    "content",
+    [
+        "alpha  beta\tgamma\n\ndelta",
+        "line with markdown hard break  \nnext line\n```\na  b\n```",
+        " \t\n\n  \n",
+        "supercalifragilisticexpialidocious",
+        "é  世界\r\nnext",
+    ],
+)
+def test_chunks_preserve_every_character_and_respect_max_chars(content):
+    doc = document(content)
+    chunks = chunk(doc, ChunkPolicy(version="exact:v1", max_chars=9))
+    assert "".join(item.content for item in chunks) == doc.content
+    assert all(0 < len(item.content) <= 9 for item in chunks)
+
+
 def test_chunks_have_contiguous_ordinals_hashes_and_provenance_metadata():
     doc = document("alpha beta gamma")
     chunks = chunk(doc, ChunkPolicy(version="words:v1", max_chars=7))
@@ -54,13 +99,15 @@ def test_chunks_have_contiguous_ordinals_hashes_and_provenance_metadata():
         assert item.source_uri == doc.source_uri
         assert item.document_id == doc.document_id
         assert item.pipeline_version == doc.pipeline_version
-        assert item.metadata["chunk_policy"] == {"max_chars": 7, "version": "words:v1"}
+        assert item.metadata["chunk_policy"]["max_chars"] == 7
+        assert item.metadata["chunk_policy"]["version"] == "words:v1"
+        assert item.metadata["chunk_policy"]["fingerprint"].startswith("sha256:")
         assert item.metadata["document"] == {"owner": "docs"}
         assert item.content_hash == "sha256:" + hashlib.sha256(item.content.encode()).hexdigest()
 
 
 def test_duplicate_chunk_content_cannot_collide_across_ordinals():
-    chunks = chunk(document("same\n\nsame"), ChunkPolicy(version="paragraph:v1", max_chars=8))
+    chunks = chunk(document("samesame"), ChunkPolicy(version="paragraph:v1", max_chars=4))
     assert [item.content for item in chunks] == ["same", "same"]
     assert chunks[0].chunk_id != chunks[1].chunk_id
 

harness/tests/test_corpus_normalize.py

@@ -52,6 +52,29 @@ def test_frontmatter_can_end_at_eof_without_inventing_content():
     assert document.content == ""
 
 
+@pytest.mark.parametrize(
+    "frontmatter",
+    [
+        "title: first\ntitle: second",
+        "title: &shared value\ncopy: *shared",
+        "nested: " + "[" * 25 + "x" + "]" * 25,
+        "items: [" + ",".join("x" for _ in range(1100)) + "]",
+        "api_key: secret",
+    ],
+)
+def test_rejects_unsafe_frontmatter_as_typed_permanent_error(frontmatter):
+    raw = f"---\n{frontmatter}\n---\nbody".encode()
+    with pytest.raises(PermanentNormalizationError) as caught:
+        normalize(acquired(raw), "pipe:v1")
+    assert caught.value.reason == "invalid_frontmatter"
+
+
+def test_pipeline_policy_errors_are_not_misclassified_as_bad_frontmatter():
+    with pytest.raises(ValueError, match="pipeline_version") as caught:
+        normalize(acquired(b"---\ntitle: valid\n---\nbody"), "")
+    assert not isinstance(caught.value, PermanentNormalizationError)
+
+
 @pytest.mark.parametrize(
     ("content", "media_type", "reason"),
     [