fix(evidence): harden canonical corpus contracts
This commit is contained in:
@@ -1,64 +1,135 @@
|
||||
"""Immutable records emitted by the Evidence preprocessing pipeline."""
|
||||
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator
|
||||
|
||||
from tht.ports.evidence import _reject_credentials
|
||||
from tht.ports.evidence import (
|
||||
normalize_aware_datetime,
|
||||
validate_canonical_uri,
|
||||
validate_namespaced_value,
|
||||
validate_safe_metadata,
|
||||
)
|
||||
|
||||
|
||||
_NAMESPACED_ID = re.compile(r"^[a-z][a-z0-9_-]*:[A-Za-z0-9._:-]+$")
|
||||
_SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$")
|
||||
|
||||
|
||||
def _validate_namespaced_id(value: str) -> str:
|
||||
if not _NAMESPACED_ID.fullmatch(value):
|
||||
raise ValueError("identifier must be namespaced as '<kind>:<stable-value>'")
|
||||
return value
|
||||
|
||||
|
||||
def _validate_hash(value: str) -> str:
|
||||
if not _SHA256.fullmatch(value):
|
||||
raise ValueError("content hash must be 'sha256:' followed by 64 lowercase hex digits")
|
||||
return value
|
||||
|
||||
|
||||
class _CanonicalValue(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
model_config = ConfigDict(
|
||||
frozen=True, extra="forbid", validate_default=True, revalidate_instances="always"
|
||||
)
|
||||
|
||||
|
||||
class _WithMetadata(_CanonicalValue):
|
||||
metadata: dict[str, JsonValue] = Field(default_factory=dict)
|
||||
|
||||
@field_validator("metadata")
|
||||
@classmethod
|
||||
def metadata_has_no_credentials(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]:
|
||||
_reject_credentials(value)
|
||||
return value
|
||||
_frozen_metadata = field_validator("metadata")(validate_safe_metadata)
|
||||
|
||||
|
||||
class CanonicalDocument(_WithMetadata):
|
||||
document_id: str = Field(min_length=1)
|
||||
source_id: str = Field(min_length=1)
|
||||
source_uri: str = Field(min_length=1)
|
||||
document_id: str
|
||||
source_id: str
|
||||
source_uri: str
|
||||
source_fingerprint: str = Field(min_length=1)
|
||||
content_hash: str = Field(min_length=1)
|
||||
content_hash: str
|
||||
title: str = ""
|
||||
content: str
|
||||
media_type: str = "text/plain"
|
||||
modified_at: datetime | None = None
|
||||
pipeline_version: str = Field(min_length=1)
|
||||
|
||||
_document_id = field_validator("document_id")(_validate_namespaced_id)
|
||||
_source_id = field_validator("source_id")(_validate_namespaced_id)
|
||||
_source_uri = field_validator("source_uri")(validate_canonical_uri)
|
||||
_source_fingerprint = field_validator("source_fingerprint")(validate_namespaced_value)
|
||||
_content_hash = field_validator("content_hash")(_validate_hash)
|
||||
_modified_at = field_validator("modified_at")(normalize_aware_datetime)
|
||||
|
||||
|
||||
class CanonicalChunk(_WithMetadata):
|
||||
chunk_id: str = Field(min_length=1)
|
||||
document_id: str = Field(min_length=1)
|
||||
chunk_id: str
|
||||
document_id: str
|
||||
ordinal: int = Field(ge=0)
|
||||
content: str
|
||||
content_hash: str = Field(min_length=1)
|
||||
source_uri: str = Field(min_length=1)
|
||||
content_hash: str
|
||||
source_uri: str
|
||||
pipeline_version: str = Field(min_length=1)
|
||||
|
||||
_chunk_id = field_validator("chunk_id")(_validate_namespaced_id)
|
||||
_document_id = field_validator("document_id")(_validate_namespaced_id)
|
||||
_content_hash = field_validator("content_hash")(_validate_hash)
|
||||
_source_uri = field_validator("source_uri")(validate_canonical_uri)
|
||||
|
||||
|
||||
class CorpusManifest(_WithMetadata):
|
||||
"""Description of one publishable canonical/vector generation."""
|
||||
"""Description of one internally consistent publishable generation."""
|
||||
|
||||
schema_version: int = Field(default=1, ge=1)
|
||||
manifest_id: str | None = None
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
pipeline_version: str = Field(default="1", min_length=1)
|
||||
pipeline_version: str = Field(default="evidence-v1", min_length=1)
|
||||
embedding_model: str | None = None
|
||||
embedding_dimensions: int | None = Field(default=None, gt=0)
|
||||
vector_generation: str | None = None
|
||||
documents: list[CanonicalDocument] = Field(default_factory=list)
|
||||
chunks: list[CanonicalChunk] = Field(default_factory=list)
|
||||
documents: tuple[CanonicalDocument, ...] = Field(default_factory=tuple)
|
||||
chunks: tuple[CanonicalChunk, ...] = Field(default_factory=tuple)
|
||||
|
||||
_manifest_id = field_validator("manifest_id")(
|
||||
lambda value: _validate_namespaced_id(value) if value is not None else None
|
||||
)
|
||||
_vector_generation = field_validator("vector_generation")(
|
||||
lambda value: _validate_namespaced_id(value) if value is not None else None
|
||||
)
|
||||
_created_at = field_validator("created_at")(normalize_aware_datetime)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def embedding_fields_are_complete(self) -> "CorpusManifest":
|
||||
def validate_generation(self) -> "CorpusManifest":
|
||||
if (self.embedding_model is None) != (self.embedding_dimensions is None):
|
||||
raise ValueError("embedding_model and embedding_dimensions must be set together")
|
||||
if self.vector_generation is not None and self.embedding_model is None:
|
||||
raise ValueError("vector_generation requires embedding model and dimension compatibility")
|
||||
|
||||
document_ids = [document.document_id for document in self.documents]
|
||||
source_ids = [document.source_id for document in self.documents]
|
||||
chunk_ids = [chunk.chunk_id for chunk in self.chunks]
|
||||
self._require_unique("document_id", document_ids)
|
||||
self._require_unique("source_id", source_ids)
|
||||
self._require_unique("chunk_id", chunk_ids)
|
||||
|
||||
documents = {document.document_id: document for document in self.documents}
|
||||
ordinals: dict[str, list[int]] = {}
|
||||
for document in self.documents:
|
||||
if document.pipeline_version != self.pipeline_version:
|
||||
raise ValueError("document pipeline_version must match manifest pipeline_version")
|
||||
for chunk in self.chunks:
|
||||
document = documents.get(chunk.document_id)
|
||||
if document is None:
|
||||
raise ValueError(f"chunk references unknown document: {chunk.document_id}")
|
||||
if chunk.pipeline_version != self.pipeline_version:
|
||||
raise ValueError("chunk pipeline_version must match manifest pipeline_version")
|
||||
if chunk.source_uri != document.source_uri:
|
||||
raise ValueError("chunk source_uri must match its document provenance")
|
||||
ordinals.setdefault(chunk.document_id, []).append(chunk.ordinal)
|
||||
for document_id, values in ordinals.items():
|
||||
if sorted(values) != list(range(len(values))):
|
||||
raise ValueError(f"chunk ordinals must be unique and contiguous for {document_id}")
|
||||
return self
|
||||
|
||||
@staticmethod
|
||||
def _require_unique(field: str, values: list[str]) -> None:
|
||||
if len(values) != len(set(values)):
|
||||
raise ValueError(f"{field} values must be unique")
|
||||
|
||||
+134
-28
@@ -1,42 +1,125 @@
|
||||
"""Port for discovering and acquiring Evidence source objects.
|
||||
"""Credential-free port for discovering and acquiring Evidence objects."""
|
||||
|
||||
Source adapters own transport details and credentials. The values crossing this
|
||||
boundary are deliberately credential-free so they can safely become provenance.
|
||||
"""
|
||||
import re
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from datetime import UTC, datetime
|
||||
from enum import Enum
|
||||
from typing import Protocol, runtime_checkable
|
||||
from urllib.parse import parse_qsl, urlsplit
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Iterable, Protocol, runtime_checkable
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator
|
||||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, field_validator
|
||||
|
||||
|
||||
_SECRET_KEYS = {
|
||||
"api_key",
|
||||
class FrozenDict(dict):
|
||||
"""A JSON-serializable dict whose mutation operations are disabled."""
|
||||
|
||||
def _immutable(self, *args, **kwargs):
|
||||
raise TypeError("frozen JSON metadata cannot be mutated")
|
||||
|
||||
__delitem__ = _immutable
|
||||
__ior__ = _immutable
|
||||
__setitem__ = _immutable
|
||||
clear = _immutable
|
||||
pop = _immutable
|
||||
popitem = _immutable
|
||||
setdefault = _immutable
|
||||
update = _immutable
|
||||
|
||||
|
||||
_CAMEL_BOUNDARY = re.compile(r"(?<=[a-z0-9])(?=[A-Z])")
|
||||
_SEPARATORS = re.compile(r"[^a-z0-9]+")
|
||||
_NAMESPACED_VALUE = re.compile(r"^[a-z][a-z0-9_-]*:[A-Za-z0-9._:-]+$")
|
||||
_CREDENTIAL_KEYS = {
|
||||
"apikey",
|
||||
"authorization",
|
||||
"authtoken",
|
||||
"bearertoken",
|
||||
"clientsecret",
|
||||
"credential",
|
||||
"credentials",
|
||||
"password",
|
||||
"secret",
|
||||
"token",
|
||||
"passwd",
|
||||
"privatekey",
|
||||
"refreshtoken",
|
||||
"sessioncookie",
|
||||
"xapikey",
|
||||
"accesstoken",
|
||||
}
|
||||
_JSON_METADATA = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
def _reject_credentials(value: JsonValue, path: str = "metadata") -> JsonValue:
|
||||
if isinstance(value, dict):
|
||||
def _normalize_key(key: str) -> str:
|
||||
return _SEPARATORS.sub("", _CAMEL_BOUNDARY.sub("_", key).lower())
|
||||
|
||||
|
||||
def _is_credential_key(key: str) -> bool:
|
||||
return _normalize_key(key) in _CREDENTIAL_KEYS
|
||||
|
||||
|
||||
def _reject_credentials(value, path: str = "metadata") -> None:
|
||||
if isinstance(value, Mapping):
|
||||
for key, child in value.items():
|
||||
normalized = key.lower().replace("-", "_")
|
||||
if normalized in _SECRET_KEYS or normalized.endswith(("_password", "_secret", "_token")):
|
||||
if _is_credential_key(str(key)):
|
||||
raise ValueError(f"credential-like metadata key is not allowed: {path}.{key}")
|
||||
_reject_credentials(child, f"{path}.{key}")
|
||||
elif isinstance(value, list):
|
||||
elif isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
|
||||
for index, child in enumerate(value):
|
||||
_reject_credentials(child, f"{path}[{index}]")
|
||||
|
||||
|
||||
def freeze_json(value):
|
||||
"""Recursively freeze a Pydantic-validated JSON value without changing its JSON shape."""
|
||||
if isinstance(value, Mapping):
|
||||
return FrozenDict({str(key): freeze_json(child) for key, child in value.items()})
|
||||
if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
|
||||
return tuple(freeze_json(child) for child in value)
|
||||
return value
|
||||
|
||||
|
||||
def validate_safe_metadata(value: dict[str, JsonValue]) -> FrozenDict:
|
||||
_reject_credentials(value)
|
||||
return freeze_json(value)
|
||||
|
||||
|
||||
def validate_canonical_uri(value: str) -> str:
|
||||
try:
|
||||
parsed = urlsplit(value)
|
||||
_ = parsed.port
|
||||
except ValueError as error:
|
||||
raise ValueError("invalid canonical URI") from error
|
||||
if not parsed.scheme:
|
||||
raise ValueError("canonical URI must include a scheme")
|
||||
if parsed.username is not None or parsed.password is not None:
|
||||
raise ValueError("canonical URI must not contain credentials in userinfo")
|
||||
for key, _ in parse_qsl(parsed.query, keep_blank_values=True):
|
||||
if _is_credential_key(key):
|
||||
raise ValueError("canonical URI must not contain credentials in query parameters")
|
||||
return value
|
||||
|
||||
|
||||
def normalize_aware_datetime(value: datetime | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None or value.utcoffset() is None:
|
||||
raise ValueError("datetime must be timezone-aware")
|
||||
return value.astimezone(UTC)
|
||||
|
||||
|
||||
def validate_namespaced_value(value: str) -> str:
|
||||
if not _NAMESPACED_VALUE.fullmatch(value):
|
||||
raise ValueError("value must be namespaced as '<kind>:<stable-value>'")
|
||||
return value
|
||||
|
||||
|
||||
class _EvidenceValue(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
model_config = ConfigDict(
|
||||
frozen=True,
|
||||
extra="forbid",
|
||||
revalidate_instances="always",
|
||||
validate_default=True,
|
||||
ser_json_bytes="base64",
|
||||
val_json_bytes="base64",
|
||||
)
|
||||
|
||||
|
||||
class SourceObject(_EvidenceValue):
|
||||
@@ -46,25 +129,48 @@ class SourceObject(_EvidenceValue):
|
||||
modified_at: datetime | None = None
|
||||
metadata: dict[str, JsonValue] = Field(default_factory=dict)
|
||||
|
||||
@field_validator("metadata")
|
||||
@classmethod
|
||||
def metadata_has_no_credentials(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]:
|
||||
_reject_credentials(value)
|
||||
return value
|
||||
_source_id = field_validator("source_id")(validate_namespaced_value)
|
||||
_fingerprint = field_validator("fingerprint")(validate_namespaced_value)
|
||||
_safe_uri = field_validator("uri")(validate_canonical_uri)
|
||||
_aware_modified_at = field_validator("modified_at")(normalize_aware_datetime)
|
||||
_frozen_metadata = field_validator("metadata")(validate_safe_metadata)
|
||||
|
||||
|
||||
class AcquiredDocument(_EvidenceValue):
|
||||
"""Transport result; bytes use explicit base64 encoding in JSON mode."""
|
||||
|
||||
source: SourceObject
|
||||
content: bytes
|
||||
media_type: str | None = None
|
||||
acquired_at: datetime | None = None
|
||||
metadata: dict[str, JsonValue] = Field(default_factory=dict)
|
||||
|
||||
@field_validator("metadata")
|
||||
@classmethod
|
||||
def metadata_has_no_credentials(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]:
|
||||
_reject_credentials(value)
|
||||
return value
|
||||
_aware_acquired_at = field_validator("acquired_at")(normalize_aware_datetime)
|
||||
_frozen_metadata = field_validator("metadata")(validate_safe_metadata)
|
||||
|
||||
|
||||
class EvidenceSourceErrorCategory(str, Enum):
|
||||
TRANSIENT = "transient"
|
||||
PERMANENT = "permanent"
|
||||
|
||||
|
||||
class EvidenceSourceError(Exception):
|
||||
"""Classified source failure with credential-free structured diagnostics."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
*,
|
||||
category: EvidenceSourceErrorCategory,
|
||||
details: dict[str, JsonValue] | None = None,
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.category = EvidenceSourceErrorCategory(category)
|
||||
self.details = validate_safe_metadata(_JSON_METADATA.validate_python(details or {}))
|
||||
|
||||
@property
|
||||
def retryable(self) -> bool:
|
||||
return self.category is EvidenceSourceErrorCategory.TRANSIENT
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
|
||||
Reference in New Issue
Block a user