fix(corpus): harden canonical chunk and frontmatter invariants
This commit is contained in:
+31
-36
@@ -1,8 +1,9 @@
|
||||
"""Versioned deterministic chunking for canonical corpus documents."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import asdict, dataclass
|
||||
|
||||
from tht.corpus.models import CanonicalChunk, CanonicalDocument
|
||||
|
||||
@@ -23,62 +24,56 @@ def _hash(text: str) -> str:
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _hard_split(text: str, maximum: int) -> list[str]:
|
||||
return [text[start : start + maximum] for start in range(0, len(text), maximum)]
|
||||
|
||||
|
||||
def _split_block(block: str, maximum: int) -> list[str]:
|
||||
if len(block) <= maximum:
|
||||
return [block]
|
||||
tokens = re.findall(r"\S+", block)
|
||||
chunks: list[str] = []
|
||||
current = ""
|
||||
for token in tokens:
|
||||
if len(token) > maximum:
|
||||
if current:
|
||||
chunks.append(current)
|
||||
current = ""
|
||||
chunks.extend(_hard_split(token, maximum))
|
||||
continue
|
||||
candidate = f"{current} {token}" if current else token
|
||||
if len(candidate) <= maximum:
|
||||
current = candidate
|
||||
else:
|
||||
chunks.append(current)
|
||||
current = token
|
||||
if current:
|
||||
chunks.append(current)
|
||||
return chunks
|
||||
|
||||
|
||||
def _contents(content: str, maximum: int) -> list[str]:
|
||||
if not content:
|
||||
return []
|
||||
result: list[str] = []
|
||||
for block in re.split(r"\n{2,}", content):
|
||||
if block:
|
||||
result.extend(_split_block(block, maximum))
|
||||
start = 0
|
||||
while start < len(content):
|
||||
end = min(start + maximum, len(content))
|
||||
if end < len(content):
|
||||
boundaries = list(re.finditer(r"\s+", content[start:end]))
|
||||
if boundaries:
|
||||
end = start + boundaries[-1].end()
|
||||
result.append(content[start:end])
|
||||
start = end
|
||||
return result
|
||||
|
||||
|
||||
def _policy_fingerprint(policy: ChunkPolicy) -> str:
|
||||
serialized = json.dumps(asdict(policy), ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
return f"sha256:{_hash(serialized)}"
|
||||
|
||||
|
||||
def chunk(document: CanonicalDocument, policy: ChunkPolicy) -> list[CanonicalChunk]:
|
||||
"""Split canonical text with stable character-count boundaries and identifiers."""
|
||||
chunks: list[CanonicalChunk] = []
|
||||
policy_fingerprint = _policy_fingerprint(policy)
|
||||
for ordinal, content in enumerate(_contents(document.content, policy.max_chars)):
|
||||
identifier = _hash(f"{document.content_hash}:{ordinal}:{policy.version}")
|
||||
chunk_hash = f"sha256:{_hash(content)}"
|
||||
identifier = _hash(
|
||||
":".join(
|
||||
(
|
||||
document.document_id,
|
||||
document.content_hash,
|
||||
policy_fingerprint,
|
||||
str(ordinal),
|
||||
chunk_hash,
|
||||
)
|
||||
)
|
||||
)
|
||||
chunks.append(
|
||||
CanonicalChunk(
|
||||
chunk_id=f"chunk:{identifier}",
|
||||
document_id=document.document_id,
|
||||
ordinal=ordinal,
|
||||
content=content,
|
||||
content_hash=f"sha256:{_hash(content)}",
|
||||
content_hash=chunk_hash,
|
||||
source_uri=document.source_uri,
|
||||
pipeline_version=document.pipeline_version,
|
||||
metadata={
|
||||
"chunk_policy": {
|
||||
"version": policy.version,
|
||||
"max_chars": policy.max_chars,
|
||||
"fingerprint": policy_fingerprint,
|
||||
},
|
||||
"document": document.model_dump(mode="json")["metadata"],
|
||||
"source_fingerprint": document.source_fingerprint,
|
||||
|
||||
@@ -7,6 +7,8 @@ from collections.abc import Mapping
|
||||
|
||||
import yaml
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
from yaml.events import AliasEvent
|
||||
from yaml.nodes import MappingNode
|
||||
|
||||
from tht.corpus.models import CanonicalDocument
|
||||
from tht.ports.evidence import AcquiredDocument, canonical_provenance_uri
|
||||
@@ -16,6 +18,47 @@ MAX_DOCUMENT_BYTES = 10 * 1024 * 1024
|
||||
_CHARSET = re.compile(r"(?:^|;)\s*charset\s*=\s*[\"']?([^;\s\"']+)", re.IGNORECASE)
|
||||
_FRONTMATTER = re.compile(r"\A---\n(.*?)\n---(?:\n|\Z)", re.DOTALL)
|
||||
_JSON_OBJECT = TypeAdapter(dict[str, JsonValue])
|
||||
_MAX_FRONTMATTER_DEPTH = 20
|
||||
_MAX_FRONTMATTER_NODES = 1000
|
||||
|
||||
|
||||
class _FrontmatterLoader(yaml.SafeLoader):
|
||||
"""SafeLoader with bounded structure and no YAML graph features."""
|
||||
|
||||
def __init__(self, stream) -> None:
|
||||
super().__init__(stream)
|
||||
self._depth = 0
|
||||
self._nodes = 0
|
||||
|
||||
def compose_node(self, parent, index):
|
||||
event = self.peek_event()
|
||||
if isinstance(event, AliasEvent) or getattr(event, "anchor", None) is not None:
|
||||
raise yaml.constructor.ConstructorError(None, None, "aliases are not allowed")
|
||||
self._depth += 1
|
||||
self._nodes += 1
|
||||
if self._depth > _MAX_FRONTMATTER_DEPTH or self._nodes > _MAX_FRONTMATTER_NODES:
|
||||
raise yaml.constructor.ConstructorError(None, None, "frontmatter is too complex")
|
||||
try:
|
||||
return super().compose_node(parent, index)
|
||||
finally:
|
||||
self._depth -= 1
|
||||
|
||||
def construct_mapping(self, node, deep=False):
|
||||
if not isinstance(node, MappingNode):
|
||||
return super().construct_mapping(node, deep=deep)
|
||||
seen: set[object] = set()
|
||||
for key_node, _ in node.value:
|
||||
key = self.construct_object(key_node, deep=deep)
|
||||
try:
|
||||
duplicate = key in seen
|
||||
seen.add(key)
|
||||
except TypeError as error:
|
||||
raise yaml.constructor.ConstructorError(
|
||||
None, None, "mapping keys must be scalar"
|
||||
) from error
|
||||
if duplicate:
|
||||
raise yaml.constructor.ConstructorError(None, None, "duplicate mapping key")
|
||||
return super().construct_mapping(node, deep=deep)
|
||||
|
||||
|
||||
class PermanentNormalizationError(ValueError):
|
||||
@@ -55,7 +98,7 @@ def _frontmatter(text: str) -> tuple[dict[str, JsonValue], str]:
|
||||
if match is None:
|
||||
return {}, text
|
||||
try:
|
||||
loaded = yaml.safe_load(match.group(1))
|
||||
loaded = yaml.load(match.group(1), Loader=_FrontmatterLoader)
|
||||
if loaded is None:
|
||||
loaded = {}
|
||||
if not isinstance(loaded, Mapping):
|
||||
@@ -84,16 +127,21 @@ def normalize(acquired: AcquiredDocument, pipeline_version: str) -> CanonicalDoc
|
||||
if frontmatter:
|
||||
metadata["frontmatter"] = frontmatter
|
||||
|
||||
return CanonicalDocument(
|
||||
document_id=f"doc:{_sha256(identity)}",
|
||||
source_id=acquired.source.source_id,
|
||||
source_uri=source_uri,
|
||||
source_fingerprint=acquired.source.fingerprint,
|
||||
content_hash=f"sha256:{_sha256(content)}",
|
||||
title=str(frontmatter.get("title", "")),
|
||||
content=content,
|
||||
media_type=media_type,
|
||||
modified_at=acquired.source.modified_at,
|
||||
pipeline_version=pipeline_version,
|
||||
metadata=metadata,
|
||||
)
|
||||
try:
|
||||
return CanonicalDocument(
|
||||
document_id=f"doc:{_sha256(identity)}",
|
||||
source_id=acquired.source.source_id,
|
||||
source_uri=source_uri,
|
||||
source_fingerprint=acquired.source.fingerprint,
|
||||
content_hash=f"sha256:{_sha256(content)}",
|
||||
title=str(frontmatter.get("title", "")),
|
||||
content=content,
|
||||
media_type=media_type,
|
||||
modified_at=acquired.source.modified_at,
|
||||
pipeline_version=pipeline_version,
|
||||
metadata=metadata,
|
||||
)
|
||||
except ValidationError as error:
|
||||
if frontmatter:
|
||||
raise PermanentNormalizationError("invalid_frontmatter") from error
|
||||
raise
|
||||
|
||||
Reference in New Issue
Block a user