"""Evidence-owned deterministic chunking for canonical corpus documents.""" import hashlib import json import re from dataclasses import asdict, dataclass from tht.evidence.corpus.models import CanonicalChunk, CanonicalDocument @dataclass(frozen=True, slots=True) class ChunkPolicy: version: str max_chars: int def __post_init__(self) -> None: if not self.version: raise ValueError("chunk policy version must not be empty") if self.max_chars <= 0: raise ValueError("max_chars must be greater than zero") def _hash(text: str) -> str: return hashlib.sha256(text.encode("utf-8")).hexdigest() def _contents(content: str, maximum: int) -> list[str]: result: list[str] = [] start = 0 while start < len(content): end = min(start + maximum, len(content)) if end < len(content): boundaries = list(re.finditer(r"\s+", content[start:end])) if boundaries: end = start + boundaries[-1].end() result.append(content[start:end]) start = end return result def _policy_fingerprint(policy: ChunkPolicy) -> str: serialized = json.dumps(asdict(policy), ensure_ascii=False, sort_keys=True, separators=(",", ":")) return f"sha256:{_hash(serialized)}" def chunk(document: CanonicalDocument, policy: ChunkPolicy) -> list[CanonicalChunk]: """Split canonical text with stable character-count boundaries and identifiers.""" chunks: list[CanonicalChunk] = [] policy_fingerprint = _policy_fingerprint(policy) for ordinal, content in enumerate(_contents(document.content, policy.max_chars)): chunk_hash = f"sha256:{_hash(content)}" identifier = _hash( ":".join( ( document.document_id, document.content_hash, policy_fingerprint, str(ordinal), chunk_hash, ) ) ) chunks.append( CanonicalChunk( chunk_id=f"chunk:{identifier}", document_id=document.document_id, ordinal=ordinal, content=content, content_hash=chunk_hash, source_uri=document.source_uri, pipeline_version=document.pipeline_version, metadata={ "chunk_policy": { "version": policy.version, "max_chars": policy.max_chars, "fingerprint": policy_fingerprint, }, "document": document.model_dump(mode="json")["metadata"], "source_fingerprint": document.source_fingerprint, "title": document.title, }, ) ) return chunks