85 lines
2.8 KiB
Python
85 lines
2.8 KiB
Python
"""Versioned deterministic chunking for canonical corpus documents."""
|
|
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from dataclasses import asdict, dataclass
|
|
|
|
from tht.corpus.models import CanonicalChunk, CanonicalDocument
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ChunkPolicy:
|
|
version: str
|
|
max_chars: int
|
|
|
|
def __post_init__(self) -> None:
|
|
if not self.version:
|
|
raise ValueError("chunk policy version must not be empty")
|
|
if self.max_chars <= 0:
|
|
raise ValueError("max_chars must be greater than zero")
|
|
|
|
|
|
def _hash(text: str) -> str:
|
|
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _contents(content: str, maximum: int) -> list[str]:
|
|
result: list[str] = []
|
|
start = 0
|
|
while start < len(content):
|
|
end = min(start + maximum, len(content))
|
|
if end < len(content):
|
|
boundaries = list(re.finditer(r"\s+", content[start:end]))
|
|
if boundaries:
|
|
end = start + boundaries[-1].end()
|
|
result.append(content[start:end])
|
|
start = end
|
|
return result
|
|
|
|
|
|
def _policy_fingerprint(policy: ChunkPolicy) -> str:
|
|
serialized = json.dumps(asdict(policy), ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
return f"sha256:{_hash(serialized)}"
|
|
|
|
|
|
def chunk(document: CanonicalDocument, policy: ChunkPolicy) -> list[CanonicalChunk]:
|
|
"""Split canonical text with stable character-count boundaries and identifiers."""
|
|
chunks: list[CanonicalChunk] = []
|
|
policy_fingerprint = _policy_fingerprint(policy)
|
|
for ordinal, content in enumerate(_contents(document.content, policy.max_chars)):
|
|
chunk_hash = f"sha256:{_hash(content)}"
|
|
identifier = _hash(
|
|
":".join(
|
|
(
|
|
document.document_id,
|
|
document.content_hash,
|
|
policy_fingerprint,
|
|
str(ordinal),
|
|
chunk_hash,
|
|
)
|
|
)
|
|
)
|
|
chunks.append(
|
|
CanonicalChunk(
|
|
chunk_id=f"chunk:{identifier}",
|
|
document_id=document.document_id,
|
|
ordinal=ordinal,
|
|
content=content,
|
|
content_hash=chunk_hash,
|
|
source_uri=document.source_uri,
|
|
pipeline_version=document.pipeline_version,
|
|
metadata={
|
|
"chunk_policy": {
|
|
"version": policy.version,
|
|
"max_chars": policy.max_chars,
|
|
"fingerprint": policy_fingerprint,
|
|
},
|
|
"document": document.model_dump(mode="json")["metadata"],
|
|
"source_fingerprint": document.source_fingerprint,
|
|
"title": document.title,
|
|
},
|
|
)
|
|
)
|
|
return chunks
|