148 lines
5.5 KiB
Python
148 lines
5.5 KiB
Python
"""Pure, deterministic conversion of acquired bytes into canonical text."""
|
|
|
|
import hashlib
|
|
import re
|
|
import unicodedata
|
|
from collections.abc import Mapping
|
|
|
|
import yaml
|
|
from pydantic import JsonValue, TypeAdapter, ValidationError
|
|
from yaml.events import AliasEvent
|
|
from yaml.nodes import MappingNode
|
|
|
|
from tht.corpus.models import CanonicalDocument
|
|
from tht.evidence.contracts import AcquiredDocument, canonical_provenance_uri
|
|
|
|
|
|
MAX_DOCUMENT_BYTES = 10 * 1024 * 1024
|
|
_CHARSET = re.compile(r"(?:^|;)\s*charset\s*=\s*[\"']?([^;\s\"']+)", re.IGNORECASE)
|
|
_FRONTMATTER = re.compile(r"\A---\n(.*?)\n---(?:\n|\Z)", re.DOTALL)
|
|
_JSON_OBJECT = TypeAdapter(dict[str, JsonValue])
|
|
_MAX_FRONTMATTER_DEPTH = 20
|
|
_MAX_FRONTMATTER_NODES = 1000
|
|
|
|
|
|
class _FrontmatterLoader(yaml.SafeLoader):
|
|
"""SafeLoader with bounded structure and no YAML graph features."""
|
|
|
|
def __init__(self, stream) -> None:
|
|
super().__init__(stream)
|
|
self._depth = 0
|
|
self._nodes = 0
|
|
|
|
def compose_node(self, parent, index):
|
|
event = self.peek_event()
|
|
if isinstance(event, AliasEvent) or getattr(event, "anchor", None) is not None:
|
|
raise yaml.constructor.ConstructorError(None, None, "aliases are not allowed")
|
|
self._depth += 1
|
|
self._nodes += 1
|
|
if self._depth > _MAX_FRONTMATTER_DEPTH or self._nodes > _MAX_FRONTMATTER_NODES:
|
|
raise yaml.constructor.ConstructorError(None, None, "frontmatter is too complex")
|
|
try:
|
|
return super().compose_node(parent, index)
|
|
finally:
|
|
self._depth -= 1
|
|
|
|
def construct_mapping(self, node, deep=False):
|
|
if not isinstance(node, MappingNode):
|
|
return super().construct_mapping(node, deep=deep)
|
|
seen: set[object] = set()
|
|
for key_node, _ in node.value:
|
|
key = self.construct_object(key_node, deep=deep)
|
|
try:
|
|
duplicate = key in seen
|
|
seen.add(key)
|
|
except TypeError as error:
|
|
raise yaml.constructor.ConstructorError(
|
|
None, None, "mapping keys must be scalar"
|
|
) from error
|
|
if duplicate:
|
|
raise yaml.constructor.ConstructorError(None, None, "duplicate mapping key")
|
|
return super().construct_mapping(node, deep=deep)
|
|
|
|
|
|
class PermanentNormalizationError(ValueError):
|
|
"""A deterministic input failure which retrying cannot repair."""
|
|
|
|
def __init__(self, reason: str) -> None:
|
|
super().__init__(f"document normalization failed: {reason}")
|
|
self.reason = reason
|
|
self.permanent = True
|
|
|
|
|
|
def _sha256(value: str) -> str:
|
|
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _decode(acquired: AcquiredDocument) -> str:
|
|
if len(acquired.content) > MAX_DOCUMENT_BYTES:
|
|
raise PermanentNormalizationError("oversized")
|
|
|
|
media_type = acquired.media_type or "text/plain"
|
|
charset = _CHARSET.search(media_type)
|
|
if charset and charset.group(1).lower().replace("_", "-") not in {
|
|
"utf-8",
|
|
"utf8",
|
|
"us-ascii",
|
|
"ascii",
|
|
}:
|
|
raise PermanentNormalizationError("unsupported_charset")
|
|
try:
|
|
return acquired.content.decode("utf-8-sig", errors="strict")
|
|
except UnicodeDecodeError as error:
|
|
raise PermanentNormalizationError("undecodable") from error
|
|
|
|
|
|
def _frontmatter(text: str) -> tuple[dict[str, JsonValue], str]:
|
|
match = _FRONTMATTER.match(text)
|
|
if match is None:
|
|
return {}, text
|
|
try:
|
|
loaded = yaml.load(match.group(1), Loader=_FrontmatterLoader)
|
|
if loaded is None:
|
|
loaded = {}
|
|
if not isinstance(loaded, Mapping):
|
|
raise TypeError("frontmatter is not a mapping")
|
|
metadata = _JSON_OBJECT.validate_python(dict(loaded))
|
|
except (TypeError, UnicodeError, ValidationError, yaml.YAMLError) as error:
|
|
raise PermanentNormalizationError("invalid_frontmatter") from error
|
|
return metadata, text[match.end() :]
|
|
|
|
|
|
def normalize(acquired: AcquiredDocument, pipeline_version: str) -> CanonicalDocument:
|
|
"""Normalize one transport result without I/O or implicit data loss."""
|
|
if not pipeline_version:
|
|
raise ValueError("pipeline_version must not be empty")
|
|
|
|
decoded = _decode(acquired)
|
|
canonical = unicodedata.normalize("NFC", decoded.replace("\r\n", "\n").replace("\r", "\n"))
|
|
frontmatter, content = _frontmatter(canonical)
|
|
source_uri = canonical_provenance_uri(acquired.source.uri)
|
|
identity = f"{acquired.source.source_id}\n{source_uri}"
|
|
media_type = (acquired.media_type or "text/plain").split(";", 1)[0].strip().lower()
|
|
metadata: dict[str, JsonValue] = {
|
|
"source": acquired.source.model_dump(mode="json")["metadata"],
|
|
"acquisition": acquired.model_dump(mode="json")["metadata"],
|
|
}
|
|
if frontmatter:
|
|
metadata["frontmatter"] = frontmatter
|
|
|
|
try:
|
|
return CanonicalDocument(
|
|
document_id=f"doc:{_sha256(identity)}",
|
|
source_id=acquired.source.source_id,
|
|
source_uri=source_uri,
|
|
source_fingerprint=acquired.source.fingerprint,
|
|
content_hash=f"sha256:{_sha256(content)}",
|
|
title=str(frontmatter.get("title", "")),
|
|
content=content,
|
|
media_type=media_type,
|
|
modified_at=acquired.source.modified_at,
|
|
pipeline_version=pipeline_version,
|
|
metadata=metadata,
|
|
)
|
|
except ValidationError as error:
|
|
if frontmatter:
|
|
raise PermanentNormalizationError("invalid_frontmatter") from error
|
|
raise
|