refactor(evidence): remove legacy Python layout (#31)

harness/tht/evidence/corpus/chunk.py

@@ -0,0 +1,84 @@
+"""Evidence-owned deterministic chunking for canonical corpus documents."""
+
+import hashlib
+import json
+import re
+from dataclasses import asdict, dataclass
+
+from tht.evidence.corpus.models import CanonicalChunk, CanonicalDocument
+
+
+@dataclass(frozen=True, slots=True)
+class ChunkPolicy:
+    version: str
+    max_chars: int
+
+    def __post_init__(self) -> None:
+        if not self.version:
+            raise ValueError("chunk policy version must not be empty")
+        if self.max_chars <= 0:
+            raise ValueError("max_chars must be greater than zero")
+
+
+def _hash(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def _contents(content: str, maximum: int) -> list[str]:
+    result: list[str] = []
+    start = 0
+    while start < len(content):
+        end = min(start + maximum, len(content))
+        if end < len(content):
+            boundaries = list(re.finditer(r"\s+", content[start:end]))
+            if boundaries:
+                end = start + boundaries[-1].end()
+        result.append(content[start:end])
+        start = end
+    return result
+
+
+def _policy_fingerprint(policy: ChunkPolicy) -> str:
+    serialized = json.dumps(asdict(policy), ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+    return f"sha256:{_hash(serialized)}"
+
+
+def chunk(document: CanonicalDocument, policy: ChunkPolicy) -> list[CanonicalChunk]:
+    """Split canonical text with stable character-count boundaries and identifiers."""
+    chunks: list[CanonicalChunk] = []
+    policy_fingerprint = _policy_fingerprint(policy)
+    for ordinal, content in enumerate(_contents(document.content, policy.max_chars)):
+        chunk_hash = f"sha256:{_hash(content)}"
+        identifier = _hash(
+            ":".join(
+                (
+                    document.document_id,
+                    document.content_hash,
+                    policy_fingerprint,
+                    str(ordinal),
+                    chunk_hash,
+                )
+            )
+        )
+        chunks.append(
+            CanonicalChunk(
+                chunk_id=f"chunk:{identifier}",
+                document_id=document.document_id,
+                ordinal=ordinal,
+                content=content,
+                content_hash=chunk_hash,
+                source_uri=document.source_uri,
+                pipeline_version=document.pipeline_version,
+                metadata={
+                    "chunk_policy": {
+                        "version": policy.version,
+                        "max_chars": policy.max_chars,
+                        "fingerprint": policy_fingerprint,
+                    },
+                    "document": document.model_dump(mode="json")["metadata"],
+                    "source_fingerprint": document.source_fingerprint,
+                    "title": document.title,
+                },
+            )
+        )
+    return chunks