65 lines
2.2 KiB
Python
65 lines
2.2 KiB
Python
"""Immutable records emitted by the Evidence preprocessing pipeline."""
|
|
|
|
from datetime import UTC, datetime
|
|
|
|
from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator
|
|
|
|
from tht.ports.evidence import _reject_credentials
|
|
|
|
|
|
class _CanonicalValue(BaseModel):
|
|
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
|
|
|
|
class _WithMetadata(_CanonicalValue):
|
|
metadata: dict[str, JsonValue] = Field(default_factory=dict)
|
|
|
|
@field_validator("metadata")
|
|
@classmethod
|
|
def metadata_has_no_credentials(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]:
|
|
_reject_credentials(value)
|
|
return value
|
|
|
|
|
|
class CanonicalDocument(_WithMetadata):
|
|
document_id: str = Field(min_length=1)
|
|
source_id: str = Field(min_length=1)
|
|
source_uri: str = Field(min_length=1)
|
|
source_fingerprint: str = Field(min_length=1)
|
|
content_hash: str = Field(min_length=1)
|
|
title: str = ""
|
|
content: str
|
|
media_type: str = "text/plain"
|
|
modified_at: datetime | None = None
|
|
pipeline_version: str = Field(min_length=1)
|
|
|
|
|
|
class CanonicalChunk(_WithMetadata):
|
|
chunk_id: str = Field(min_length=1)
|
|
document_id: str = Field(min_length=1)
|
|
ordinal: int = Field(ge=0)
|
|
content: str
|
|
content_hash: str = Field(min_length=1)
|
|
source_uri: str = Field(min_length=1)
|
|
pipeline_version: str = Field(min_length=1)
|
|
|
|
|
|
class CorpusManifest(_WithMetadata):
|
|
"""Description of one publishable canonical/vector generation."""
|
|
|
|
schema_version: int = Field(default=1, ge=1)
|
|
manifest_id: str | None = None
|
|
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
|
pipeline_version: str = Field(default="1", min_length=1)
|
|
embedding_model: str | None = None
|
|
embedding_dimensions: int | None = Field(default=None, gt=0)
|
|
vector_generation: str | None = None
|
|
documents: list[CanonicalDocument] = Field(default_factory=list)
|
|
chunks: list[CanonicalChunk] = Field(default_factory=list)
|
|
|
|
@model_validator(mode="after")
|
|
def embedding_fields_are_complete(self) -> "CorpusManifest":
|
|
if (self.embedding_model is None) != (self.embedding_dimensions is None):
|
|
raise ValueError("embedding_model and embedding_dimensions must be set together")
|
|
return self
|