72 lines
2.9 KiB
Python
72 lines
2.9 KiB
Python
import hashlib
|
|
|
|
import pytest
|
|
|
|
from tht.corpus.chunk import ChunkPolicy, chunk
|
|
from tht.corpus.models import CanonicalDocument
|
|
|
|
|
|
def document(content: str) -> CanonicalDocument:
|
|
normalized = content.replace("\r\n", "\n").replace("\r", "\n")
|
|
digest = hashlib.sha256(normalized.encode()).hexdigest()
|
|
return CanonicalDocument(
|
|
document_id="doc:abc",
|
|
source_id="source:a",
|
|
source_uri="https://host/a.md",
|
|
source_fingerprint="etag:abc",
|
|
content_hash=f"sha256:{digest}",
|
|
title="A",
|
|
content=normalized,
|
|
media_type="text/markdown",
|
|
pipeline_version="pipe:v1",
|
|
metadata={"owner": "docs"},
|
|
)
|
|
|
|
|
|
def test_chunk_ids_are_stable_for_same_content_and_repeat_runs():
|
|
policy = ChunkPolicy(version="paragraph:v1", max_chars=8)
|
|
first = chunk(document("A\n\nB"), policy)
|
|
second = chunk(document("A\r\n\r\nB"), policy)
|
|
repeated = chunk(document("A\n\nB"), policy)
|
|
assert first == second == repeated
|
|
|
|
|
|
def test_policy_version_changes_ids_without_changing_boundaries():
|
|
doc = document("alpha\n\nbeta")
|
|
first = chunk(doc, ChunkPolicy(version="paragraph:v1", max_chars=6))
|
|
second = chunk(doc, ChunkPolicy(version="paragraph:v2", max_chars=6))
|
|
assert [item.content for item in first] == [item.content for item in second]
|
|
assert [item.chunk_id for item in first] != [item.chunk_id for item in second]
|
|
|
|
|
|
def test_long_non_ascii_tokens_are_hard_split_by_unicode_characters():
|
|
chunks = chunk(document("ééééé世界"), ChunkPolicy(version="chars:v1", max_chars=3))
|
|
assert [item.content for item in chunks] == ["ééé", "éé世", "界"]
|
|
assert all(len(item.content) <= 3 for item in chunks)
|
|
|
|
|
|
def test_chunks_have_contiguous_ordinals_hashes_and_provenance_metadata():
|
|
doc = document("alpha beta gamma")
|
|
chunks = chunk(doc, ChunkPolicy(version="words:v1", max_chars=7))
|
|
assert [item.ordinal for item in chunks] == list(range(len(chunks)))
|
|
assert len({item.chunk_id for item in chunks}) == len(chunks)
|
|
for item in chunks:
|
|
assert item.source_uri == doc.source_uri
|
|
assert item.document_id == doc.document_id
|
|
assert item.pipeline_version == doc.pipeline_version
|
|
assert item.metadata["chunk_policy"] == {"max_chars": 7, "version": "words:v1"}
|
|
assert item.metadata["document"] == {"owner": "docs"}
|
|
assert item.content_hash == "sha256:" + hashlib.sha256(item.content.encode()).hexdigest()
|
|
|
|
|
|
def test_duplicate_chunk_content_cannot_collide_across_ordinals():
|
|
chunks = chunk(document("same\n\nsame"), ChunkPolicy(version="paragraph:v1", max_chars=8))
|
|
assert [item.content for item in chunks] == ["same", "same"]
|
|
assert chunks[0].chunk_id != chunks[1].chunk_id
|
|
|
|
|
|
def test_empty_document_has_no_chunks_and_invalid_policy_is_rejected():
|
|
assert chunk(document(""), ChunkPolicy(version="v1", max_chars=4)) == []
|
|
with pytest.raises(ValueError):
|
|
ChunkPolicy(version="v1", max_chars=0)
|