"""Immutable records emitted by the Evidence preprocessing pipeline.""" import re from datetime import UTC, datetime from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator from tht.ports.evidence import ( normalize_aware_datetime, validate_canonical_uri, validate_namespaced_value, validate_safe_metadata, ) _NAMESPACED_ID = re.compile(r"^[a-z][a-z0-9_-]*:[A-Za-z0-9._:-]+$") _SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$") def _validate_namespaced_id(value: str) -> str: if not _NAMESPACED_ID.fullmatch(value): raise ValueError("identifier must be namespaced as ':'") return value def _validate_hash(value: str) -> str: if not _SHA256.fullmatch(value): raise ValueError("content hash must be 'sha256:' followed by 64 lowercase hex digits") return value class _CanonicalValue(BaseModel): model_config = ConfigDict( frozen=True, extra="forbid", validate_default=True, revalidate_instances="always" ) class _WithMetadata(_CanonicalValue): metadata: dict[str, JsonValue] = Field(default_factory=dict) _frozen_metadata = field_validator("metadata")(validate_safe_metadata) class CanonicalDocument(_WithMetadata): document_id: str source_id: str source_uri: str source_fingerprint: str = Field(min_length=1) content_hash: str title: str = "" content: str media_type: str = "text/plain" modified_at: datetime | None = None pipeline_version: str = Field(min_length=1) _document_id = field_validator("document_id")(_validate_namespaced_id) _source_id = field_validator("source_id")(_validate_namespaced_id) _source_uri = field_validator("source_uri")(validate_canonical_uri) _source_fingerprint = field_validator("source_fingerprint")(validate_namespaced_value) _content_hash = field_validator("content_hash")(_validate_hash) _modified_at = field_validator("modified_at")(normalize_aware_datetime) class CanonicalChunk(_WithMetadata): chunk_id: str document_id: str ordinal: int = Field(ge=0) content: str content_hash: str source_uri: str pipeline_version: str = Field(min_length=1) _chunk_id = field_validator("chunk_id")(_validate_namespaced_id) _document_id = field_validator("document_id")(_validate_namespaced_id) _content_hash = field_validator("content_hash")(_validate_hash) _source_uri = field_validator("source_uri")(validate_canonical_uri) class CorpusManifest(_WithMetadata): """Description of one internally consistent publishable generation.""" schema_version: int = Field(default=1, ge=1) manifest_id: str | None = None created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) pipeline_version: str = Field(default="evidence-v1", min_length=1) embedding_model: str | None = None embedding_dimensions: int | None = Field(default=None, gt=0) vector_generation: str | None = None documents: tuple[CanonicalDocument, ...] = Field(default_factory=tuple) chunks: tuple[CanonicalChunk, ...] = Field(default_factory=tuple) _manifest_id = field_validator("manifest_id")( lambda value: _validate_namespaced_id(value) if value is not None else None ) _vector_generation = field_validator("vector_generation")( lambda value: _validate_namespaced_id(value) if value is not None else None ) _created_at = field_validator("created_at")(normalize_aware_datetime) @model_validator(mode="after") def validate_generation(self) -> "CorpusManifest": if (self.embedding_model is None) != (self.embedding_dimensions is None): raise ValueError("embedding_model and embedding_dimensions must be set together") if self.vector_generation is not None and self.embedding_model is None: raise ValueError("vector_generation requires embedding model and dimension compatibility") document_ids = [document.document_id for document in self.documents] source_ids = [document.source_id for document in self.documents] chunk_ids = [chunk.chunk_id for chunk in self.chunks] self._require_unique("document_id", document_ids) self._require_unique("source_id", source_ids) self._require_unique("chunk_id", chunk_ids) documents = {document.document_id: document for document in self.documents} ordinals: dict[str, list[int]] = {} for document in self.documents: if document.pipeline_version != self.pipeline_version: raise ValueError("document pipeline_version must match manifest pipeline_version") for chunk in self.chunks: document = documents.get(chunk.document_id) if document is None: raise ValueError(f"chunk references unknown document: {chunk.document_id}") if chunk.pipeline_version != self.pipeline_version: raise ValueError("chunk pipeline_version must match manifest pipeline_version") if chunk.source_uri != document.source_uri: raise ValueError("chunk source_uri must match its document provenance") ordinals.setdefault(chunk.document_id, []).append(chunk.ordinal) for document_id, values in ordinals.items(): if sorted(values) != list(range(len(values))): raise ValueError(f"chunk ordinals must be unique and contiguous for {document_id}") return self @staticmethod def _require_unique(field: str, values: list[str]) -> None: if len(values) != len(set(values)): raise ValueError(f"{field} values must be unique")