feat(evidence): define source and corpus contracts
This commit is contained in:
@@ -0,0 +1,106 @@
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from tht.corpus.models import CanonicalChunk, CanonicalDocument, CorpusManifest
|
||||
|
||||
|
||||
def document(source_uri: str = "https://host/a.md") -> CanonicalDocument:
|
||||
return CanonicalDocument(
|
||||
document_id="doc:abc",
|
||||
source_id="source:a",
|
||||
source_uri=source_uri,
|
||||
source_fingerprint="etag:abc",
|
||||
content_hash="sha256:def",
|
||||
title="A",
|
||||
content="# A",
|
||||
media_type="text/markdown",
|
||||
pipeline_version="normalize-v1",
|
||||
)
|
||||
|
||||
|
||||
def chunk() -> CanonicalChunk:
|
||||
return CanonicalChunk(
|
||||
chunk_id="chunk:abc:0",
|
||||
document_id="doc:abc",
|
||||
ordinal=0,
|
||||
content="# A",
|
||||
content_hash="sha256:def",
|
||||
source_uri="https://host/a.md",
|
||||
pipeline_version="chunk-v1",
|
||||
)
|
||||
|
||||
|
||||
def test_manifest_contains_provenance_without_credentials():
|
||||
manifest = CorpusManifest(
|
||||
manifest_id="manifest:abc",
|
||||
created_at=datetime(2026, 7, 12, tzinfo=UTC),
|
||||
pipeline_version="evidence-v1",
|
||||
embedding_model="nomic-embed-text",
|
||||
embedding_dimensions=768,
|
||||
documents=[document()],
|
||||
chunks=[chunk()],
|
||||
)
|
||||
|
||||
payload = manifest.model_dump_json()
|
||||
|
||||
assert "https://host/a.md" in payload
|
||||
assert "etag:abc" in payload
|
||||
assert "evidence-v1" in payload
|
||||
assert "nomic-embed-text" in payload
|
||||
assert "api_key" not in payload
|
||||
|
||||
|
||||
def test_manifest_can_be_assembled_before_publish_identifiers_are_assigned():
|
||||
manifest = CorpusManifest(documents=[document()])
|
||||
|
||||
assert manifest.documents[0].source_uri == "https://host/a.md"
|
||||
assert manifest.manifest_id is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", [document(), chunk()])
|
||||
def test_canonical_records_are_frozen(model):
|
||||
with pytest.raises(ValidationError):
|
||||
model.pipeline_version = "changed" # type: ignore[misc]
|
||||
|
||||
|
||||
def test_manifest_collections_have_independent_defaults():
|
||||
first = CorpusManifest(
|
||||
manifest_id="one", created_at=datetime.now(UTC), pipeline_version="v1"
|
||||
)
|
||||
second = CorpusManifest(
|
||||
manifest_id="two", created_at=datetime.now(UTC), pipeline_version="v1"
|
||||
)
|
||||
|
||||
first.documents.append(document())
|
||||
|
||||
assert second.documents == []
|
||||
|
||||
|
||||
def test_manifest_validates_embedding_compatibility_fields():
|
||||
with pytest.raises(ValidationError):
|
||||
CorpusManifest(
|
||||
manifest_id="bad",
|
||||
created_at=datetime.now(UTC),
|
||||
pipeline_version="v1",
|
||||
embedding_dimensions=0,
|
||||
)
|
||||
|
||||
|
||||
def test_canonical_metadata_rejects_secrets_and_non_json_values():
|
||||
with pytest.raises(ValidationError, match="credential-like"):
|
||||
CanonicalDocument.model_validate(
|
||||
{**document().model_dump(), "metadata": {"password": "secret"}}
|
||||
)
|
||||
with pytest.raises(ValidationError):
|
||||
CanonicalChunk(
|
||||
chunk_id="c",
|
||||
document_id="d",
|
||||
ordinal=0,
|
||||
content="x",
|
||||
content_hash="sha256:x",
|
||||
source_uri="file:///x",
|
||||
pipeline_version="v1",
|
||||
metadata={"bad": object()},
|
||||
)
|
||||
@@ -0,0 +1,69 @@
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from tht.ports.evidence import AcquiredDocument, EvidenceSource, SourceObject
|
||||
|
||||
|
||||
class StubSource:
|
||||
def discover(self):
|
||||
return iter(
|
||||
[
|
||||
SourceObject(
|
||||
source_id="handbook",
|
||||
uri="https://host/handbook.md",
|
||||
fingerprint="sha256:abc",
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
def acquire(self, item: SourceObject) -> AcquiredDocument:
|
||||
return AcquiredDocument(
|
||||
source=item,
|
||||
content=b"# Handbook",
|
||||
acquired_at=datetime(2026, 7, 12, tzinfo=UTC),
|
||||
media_type="text/markdown",
|
||||
)
|
||||
|
||||
|
||||
def test_runtime_checkable_source_protocol():
|
||||
source = StubSource()
|
||||
|
||||
assert isinstance(source, EvidenceSource)
|
||||
assert source.acquire(next(source.discover())).content == b"# Handbook"
|
||||
|
||||
|
||||
def test_source_objects_are_frozen_and_metadata_defaults_are_independent():
|
||||
first = SourceObject(source_id="a", uri="file:///a", fingerprint="sha256:a")
|
||||
second = SourceObject(source_id="b", uri="file:///b", fingerprint="sha256:b")
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
first.uri = "file:///changed" # type: ignore[misc]
|
||||
first.metadata["owner"] = "team-a"
|
||||
assert second.metadata == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("metadata", [{"api_key": "secret"}, {"auth": {"token": "secret"}}])
|
||||
def test_source_metadata_rejects_credentials(metadata):
|
||||
with pytest.raises(ValidationError, match="credential-like"):
|
||||
SourceObject(
|
||||
source_id="a", uri="https://host/a", fingerprint="etag:abc", metadata=metadata
|
||||
)
|
||||
|
||||
|
||||
def test_source_metadata_must_be_json_safe():
|
||||
with pytest.raises(ValidationError):
|
||||
SourceObject(
|
||||
source_id="a",
|
||||
uri="file:///a",
|
||||
fingerprint="sha256:a",
|
||||
metadata={"path": object()},
|
||||
)
|
||||
|
||||
|
||||
def test_acquired_document_does_not_accept_credentials_as_extra_fields():
|
||||
item = SourceObject(source_id="a", uri="https://host/a", fingerprint="etag:abc")
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
AcquiredDocument(source=item, content=b"a", api_key="secret")
|
||||
@@ -0,0 +1 @@
|
||||
"""Canonical, transport-independent Evidence corpus."""
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Immutable records emitted by the Evidence preprocessing pipeline."""
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator
|
||||
|
||||
from tht.ports.evidence import _reject_credentials
|
||||
|
||||
|
||||
class _CanonicalValue(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
|
||||
class _WithMetadata(_CanonicalValue):
|
||||
metadata: dict[str, JsonValue] = Field(default_factory=dict)
|
||||
|
||||
@field_validator("metadata")
|
||||
@classmethod
|
||||
def metadata_has_no_credentials(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]:
|
||||
_reject_credentials(value)
|
||||
return value
|
||||
|
||||
|
||||
class CanonicalDocument(_WithMetadata):
|
||||
document_id: str = Field(min_length=1)
|
||||
source_id: str = Field(min_length=1)
|
||||
source_uri: str = Field(min_length=1)
|
||||
source_fingerprint: str = Field(min_length=1)
|
||||
content_hash: str = Field(min_length=1)
|
||||
title: str = ""
|
||||
content: str
|
||||
media_type: str = "text/plain"
|
||||
modified_at: datetime | None = None
|
||||
pipeline_version: str = Field(min_length=1)
|
||||
|
||||
|
||||
class CanonicalChunk(_WithMetadata):
|
||||
chunk_id: str = Field(min_length=1)
|
||||
document_id: str = Field(min_length=1)
|
||||
ordinal: int = Field(ge=0)
|
||||
content: str
|
||||
content_hash: str = Field(min_length=1)
|
||||
source_uri: str = Field(min_length=1)
|
||||
pipeline_version: str = Field(min_length=1)
|
||||
|
||||
|
||||
class CorpusManifest(_WithMetadata):
|
||||
"""Description of one publishable canonical/vector generation."""
|
||||
|
||||
schema_version: int = Field(default=1, ge=1)
|
||||
manifest_id: str | None = None
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
pipeline_version: str = Field(default="1", min_length=1)
|
||||
embedding_model: str | None = None
|
||||
embedding_dimensions: int | None = Field(default=None, gt=0)
|
||||
vector_generation: str | None = None
|
||||
documents: list[CanonicalDocument] = Field(default_factory=list)
|
||||
chunks: list[CanonicalChunk] = Field(default_factory=list)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def embedding_fields_are_complete(self) -> "CorpusManifest":
|
||||
if (self.embedding_model is None) != (self.embedding_dimensions is None):
|
||||
raise ValueError("embedding_model and embedding_dimensions must be set together")
|
||||
return self
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Port for discovering and acquiring Evidence source objects.
|
||||
|
||||
Source adapters own transport details and credentials. The values crossing this
|
||||
boundary are deliberately credential-free so they can safely become provenance.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Iterable, Protocol, runtime_checkable
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator
|
||||
|
||||
|
||||
_SECRET_KEYS = {
|
||||
"api_key",
|
||||
"apikey",
|
||||
"authorization",
|
||||
"credential",
|
||||
"credentials",
|
||||
"password",
|
||||
"secret",
|
||||
"token",
|
||||
}
|
||||
|
||||
|
||||
def _reject_credentials(value: JsonValue, path: str = "metadata") -> JsonValue:
|
||||
if isinstance(value, dict):
|
||||
for key, child in value.items():
|
||||
normalized = key.lower().replace("-", "_")
|
||||
if normalized in _SECRET_KEYS or normalized.endswith(("_password", "_secret", "_token")):
|
||||
raise ValueError(f"credential-like metadata key is not allowed: {path}.{key}")
|
||||
_reject_credentials(child, f"{path}.{key}")
|
||||
elif isinstance(value, list):
|
||||
for index, child in enumerate(value):
|
||||
_reject_credentials(child, f"{path}[{index}]")
|
||||
return value
|
||||
|
||||
|
||||
class _EvidenceValue(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
|
||||
class SourceObject(_EvidenceValue):
|
||||
source_id: str = Field(min_length=1)
|
||||
uri: str = Field(min_length=1)
|
||||
fingerprint: str = Field(min_length=1)
|
||||
modified_at: datetime | None = None
|
||||
metadata: dict[str, JsonValue] = Field(default_factory=dict)
|
||||
|
||||
@field_validator("metadata")
|
||||
@classmethod
|
||||
def metadata_has_no_credentials(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]:
|
||||
_reject_credentials(value)
|
||||
return value
|
||||
|
||||
|
||||
class AcquiredDocument(_EvidenceValue):
|
||||
source: SourceObject
|
||||
content: bytes
|
||||
media_type: str | None = None
|
||||
acquired_at: datetime | None = None
|
||||
metadata: dict[str, JsonValue] = Field(default_factory=dict)
|
||||
|
||||
@field_validator("metadata")
|
||||
@classmethod
|
||||
def metadata_has_no_credentials(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]:
|
||||
_reject_credentials(value)
|
||||
return value
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class EvidenceSource(Protocol):
|
||||
def discover(self) -> Iterable[SourceObject]: ...
|
||||
|
||||
def acquire(self, item: SourceObject) -> AcquiredDocument: ...
|
||||
Reference in New Issue
Block a user