feat(evidence): define source and corpus contracts

harness/tht/corpus/__init__.py

@@ -0,0 +1 @@
+"""Canonical, transport-independent Evidence corpus."""

harness/tht/corpus/models.py

@@ -0,0 +1,64 @@
+"""Immutable records emitted by the Evidence preprocessing pipeline."""
+
+from datetime import UTC, datetime
+
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator
+
+from tht.ports.evidence import _reject_credentials
+
+
+class _CanonicalValue(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+
+class _WithMetadata(_CanonicalValue):
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+    @field_validator("metadata")
+    @classmethod
+    def metadata_has_no_credentials(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]:
+        _reject_credentials(value)
+        return value
+
+
+class CanonicalDocument(_WithMetadata):
+    document_id: str = Field(min_length=1)
+    source_id: str = Field(min_length=1)
+    source_uri: str = Field(min_length=1)
+    source_fingerprint: str = Field(min_length=1)
+    content_hash: str = Field(min_length=1)
+    title: str = ""
+    content: str
+    media_type: str = "text/plain"
+    modified_at: datetime | None = None
+    pipeline_version: str = Field(min_length=1)
+
+
+class CanonicalChunk(_WithMetadata):
+    chunk_id: str = Field(min_length=1)
+    document_id: str = Field(min_length=1)
+    ordinal: int = Field(ge=0)
+    content: str
+    content_hash: str = Field(min_length=1)
+    source_uri: str = Field(min_length=1)
+    pipeline_version: str = Field(min_length=1)
+
+
+class CorpusManifest(_WithMetadata):
+    """Description of one publishable canonical/vector generation."""
+
+    schema_version: int = Field(default=1, ge=1)
+    manifest_id: str | None = None
+    created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+    pipeline_version: str = Field(default="1", min_length=1)
+    embedding_model: str | None = None
+    embedding_dimensions: int | None = Field(default=None, gt=0)
+    vector_generation: str | None = None
+    documents: list[CanonicalDocument] = Field(default_factory=list)
+    chunks: list[CanonicalChunk] = Field(default_factory=list)
+
+    @model_validator(mode="after")
+    def embedding_fields_are_complete(self) -> "CorpusManifest":
+        if (self.embedding_model is None) != (self.embedding_dimensions is None):
+            raise ValueError("embedding_model and embedding_dimensions must be set together")
+        return self

harness/tht/ports/evidence.py

@@ -0,0 +1,74 @@
+"""Port for discovering and acquiring Evidence source objects.
+
+Source adapters own transport details and credentials.  The values crossing this
+boundary are deliberately credential-free so they can safely become provenance.
+"""
+
+from datetime import datetime
+from typing import Iterable, Protocol, runtime_checkable
+
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator
+
+
+_SECRET_KEYS = {
+    "api_key",
+    "apikey",
+    "authorization",
+    "credential",
+    "credentials",
+    "password",
+    "secret",
+    "token",
+}
+
+
+def _reject_credentials(value: JsonValue, path: str = "metadata") -> JsonValue:
+    if isinstance(value, dict):
+        for key, child in value.items():
+            normalized = key.lower().replace("-", "_")
+            if normalized in _SECRET_KEYS or normalized.endswith(("_password", "_secret", "_token")):
+                raise ValueError(f"credential-like metadata key is not allowed: {path}.{key}")
+            _reject_credentials(child, f"{path}.{key}")
+    elif isinstance(value, list):
+        for index, child in enumerate(value):
+            _reject_credentials(child, f"{path}[{index}]")
+    return value
+
+
+class _EvidenceValue(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+
+class SourceObject(_EvidenceValue):
+    source_id: str = Field(min_length=1)
+    uri: str = Field(min_length=1)
+    fingerprint: str = Field(min_length=1)
+    modified_at: datetime | None = None
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+    @field_validator("metadata")
+    @classmethod
+    def metadata_has_no_credentials(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]:
+        _reject_credentials(value)
+        return value
+
+
+class AcquiredDocument(_EvidenceValue):
+    source: SourceObject
+    content: bytes
+    media_type: str | None = None
+    acquired_at: datetime | None = None
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+    @field_validator("metadata")
+    @classmethod
+    def metadata_has_no_credentials(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]:
+        _reject_credentials(value)
+        return value
+
+
+@runtime_checkable
+class EvidenceSource(Protocol):
+    def discover(self) -> Iterable[SourceObject]: ...
+
+    def acquire(self, item: SourceObject) -> AcquiredDocument: ...