From d702aad93d6c64b4ff0ada1b493b73339d1ceff1 Mon Sep 17 00:00:00 2001
From: mptyl <mp@tylconsulting.it>
Date: Sun, 12 Jul 2026 03:02:10 +0200
Subject: [PATCH] feat(evidence): define source and corpus contracts

---
 .superpowers/sdd/evidence-task-1-report.md   |  43 ++++++++
 harness/tests/test_corpus_models.py          | 106 +++++++++++++++++++
 harness/tests/test_evidence_port_contract.py |  69 ++++++++++++
 harness/tht/corpus/__init__.py               |   1 +
 harness/tht/corpus/models.py                 |  64 +++++++++++
 harness/tht/ports/evidence.py                |  74 +++++++++++++
 6 files changed, 357 insertions(+)
 create mode 100644 .superpowers/sdd/evidence-task-1-report.md
 create mode 100644 harness/tests/test_corpus_models.py
 create mode 100644 harness/tests/test_evidence_port_contract.py
 create mode 100644 harness/tht/corpus/__init__.py
 create mode 100644 harness/tht/corpus/models.py
 create mode 100644 harness/tht/ports/evidence.py

diff --git a/.superpowers/sdd/evidence-task-1-report.md b/.superpowers/sdd/evidence-task-1-report.md
new file mode 100644
index 00000000..5d5d8eab
--- /dev/null
+++ b/.superpowers/sdd/evidence-task-1-report.md
@@ -0,0 +1,43 @@
+# Evidence / Preprocessing Task 1 Report
+
+## Outcome
+
+Implemented the additive Evidence source port and canonical corpus records. Existing evidence,
+search, vector, and session runtime code is unchanged.
+
+## Contract
+
+- `EvidenceSource` is a runtime-checkable protocol with `discover` and `acquire` operations.
+- `SourceObject` and `AcquiredDocument` are frozen, reject extra fields, use independent metadata
+  defaults, and restrict metadata to Pydantic `JsonValue` values.
+- `CanonicalDocument`, `CanonicalChunk`, and `CorpusManifest` are frozen and reject extra fields.
+- Provenance includes stable source IDs, canonical URIs, fingerprints, modification time, and
+  content hashes.
+- Pipeline versions are recorded on documents, chunks, and manifests. Manifests also carry schema
+  version, optional publish ID/vector generation, and paired embedding model/dimension fields.
+- Credential-like metadata keys are rejected recursively. Credentials are not model fields and
+  therefore cannot enter serialized canonical artifacts through extras.
+
+## TDD evidence
+
+The initial focused run failed during collection because `tht.ports.evidence` and `tht.corpus`
+did not exist. After implementation, the focused suite passed.
+
+## Verification
+
+- Focused models/protocol tests: 13 passed.
+- Harness excluding Docker-backed L0 and the network-dependent wheel packaging test: 444 passed,
+  5 deselected.
+- Focused Ruff: passed.
+- Full-repository Ruff remains blocked by 34 pre-existing findings outside the task files.
+- An unrestricted `pytest -q` attempt reached 453 passed and 5 deselected, but reported 47 Docker
+  setup errors plus 4 Docker parity failures because the sandbox cannot access the Docker socket;
+  the wheel packaging test also failed because its isolated `uv build` needs unavailable network.
+
+## Concerns / follow-up
+
+- Pydantic's `frozen=True` prevents model field reassignment but does not recursively freeze list
+  and dict contents. `default_factory` prevents shared mutable defaults. Later pipeline stages should
+  treat these value objects as immutable and construct replacements rather than mutate collections.
+- The adapter and normalization tasks should preserve the credential-free boundary by passing only
+  these records beyond acquisition.
diff --git a/harness/tests/test_corpus_models.py b/harness/tests/test_corpus_models.py
new file mode 100644
index 00000000..adb19a8a
--- /dev/null
+++ b/harness/tests/test_corpus_models.py
@@ -0,0 +1,106 @@
+from datetime import UTC, datetime
+
+import pytest
+from pydantic import ValidationError
+
+from tht.corpus.models import CanonicalChunk, CanonicalDocument, CorpusManifest
+
+
+def document(source_uri: str = "https://host/a.md") -> CanonicalDocument:
+    return CanonicalDocument(
+        document_id="doc:abc",
+        source_id="source:a",
+        source_uri=source_uri,
+        source_fingerprint="etag:abc",
+        content_hash="sha256:def",
+        title="A",
+        content="# A",
+        media_type="text/markdown",
+        pipeline_version="normalize-v1",
+    )
+
+
+def chunk() -> CanonicalChunk:
+    return CanonicalChunk(
+        chunk_id="chunk:abc:0",
+        document_id="doc:abc",
+        ordinal=0,
+        content="# A",
+        content_hash="sha256:def",
+        source_uri="https://host/a.md",
+        pipeline_version="chunk-v1",
+    )
+
+
+def test_manifest_contains_provenance_without_credentials():
+    manifest = CorpusManifest(
+        manifest_id="manifest:abc",
+        created_at=datetime(2026, 7, 12, tzinfo=UTC),
+        pipeline_version="evidence-v1",
+        embedding_model="nomic-embed-text",
+        embedding_dimensions=768,
+        documents=[document()],
+        chunks=[chunk()],
+    )
+
+    payload = manifest.model_dump_json()
+
+    assert "https://host/a.md" in payload
+    assert "etag:abc" in payload
+    assert "evidence-v1" in payload
+    assert "nomic-embed-text" in payload
+    assert "api_key" not in payload
+
+
+def test_manifest_can_be_assembled_before_publish_identifiers_are_assigned():
+    manifest = CorpusManifest(documents=[document()])
+
+    assert manifest.documents[0].source_uri == "https://host/a.md"
+    assert manifest.manifest_id is None
+
+
+@pytest.mark.parametrize("model", [document(), chunk()])
+def test_canonical_records_are_frozen(model):
+    with pytest.raises(ValidationError):
+        model.pipeline_version = "changed"  # type: ignore[misc]
+
+
+def test_manifest_collections_have_independent_defaults():
+    first = CorpusManifest(
+        manifest_id="one", created_at=datetime.now(UTC), pipeline_version="v1"
+    )
+    second = CorpusManifest(
+        manifest_id="two", created_at=datetime.now(UTC), pipeline_version="v1"
+    )
+
+    first.documents.append(document())
+
+    assert second.documents == []
+
+
+def test_manifest_validates_embedding_compatibility_fields():
+    with pytest.raises(ValidationError):
+        CorpusManifest(
+            manifest_id="bad",
+            created_at=datetime.now(UTC),
+            pipeline_version="v1",
+            embedding_dimensions=0,
+        )
+
+
+def test_canonical_metadata_rejects_secrets_and_non_json_values():
+    with pytest.raises(ValidationError, match="credential-like"):
+        CanonicalDocument.model_validate(
+            {**document().model_dump(), "metadata": {"password": "secret"}}
+        )
+    with pytest.raises(ValidationError):
+        CanonicalChunk(
+            chunk_id="c",
+            document_id="d",
+            ordinal=0,
+            content="x",
+            content_hash="sha256:x",
+            source_uri="file:///x",
+            pipeline_version="v1",
+            metadata={"bad": object()},
+        )
diff --git a/harness/tests/test_evidence_port_contract.py b/harness/tests/test_evidence_port_contract.py
new file mode 100644
index 00000000..3dd9f159
--- /dev/null
+++ b/harness/tests/test_evidence_port_contract.py
@@ -0,0 +1,69 @@
+from datetime import UTC, datetime
+
+import pytest
+from pydantic import ValidationError
+
+from tht.ports.evidence import AcquiredDocument, EvidenceSource, SourceObject
+
+
+class StubSource:
+    def discover(self):
+        return iter(
+            [
+                SourceObject(
+                    source_id="handbook",
+                    uri="https://host/handbook.md",
+                    fingerprint="sha256:abc",
+                )
+            ]
+        )
+
+    def acquire(self, item: SourceObject) -> AcquiredDocument:
+        return AcquiredDocument(
+            source=item,
+            content=b"# Handbook",
+            acquired_at=datetime(2026, 7, 12, tzinfo=UTC),
+            media_type="text/markdown",
+        )
+
+
+def test_runtime_checkable_source_protocol():
+    source = StubSource()
+
+    assert isinstance(source, EvidenceSource)
+    assert source.acquire(next(source.discover())).content == b"# Handbook"
+
+
+def test_source_objects_are_frozen_and_metadata_defaults_are_independent():
+    first = SourceObject(source_id="a", uri="file:///a", fingerprint="sha256:a")
+    second = SourceObject(source_id="b", uri="file:///b", fingerprint="sha256:b")
+
+    with pytest.raises(ValidationError):
+        first.uri = "file:///changed"  # type: ignore[misc]
+    first.metadata["owner"] = "team-a"
+    assert second.metadata == {}
+
+
+@pytest.mark.parametrize("metadata", [{"api_key": "secret"}, {"auth": {"token": "secret"}}])
+def test_source_metadata_rejects_credentials(metadata):
+    with pytest.raises(ValidationError, match="credential-like"):
+        SourceObject(
+            source_id="a", uri="https://host/a", fingerprint="etag:abc", metadata=metadata
+        )
+
+
+def test_source_metadata_must_be_json_safe():
+    with pytest.raises(ValidationError):
+        SourceObject(
+            source_id="a",
+            uri="file:///a",
+            fingerprint="sha256:a",
+            metadata={"path": object()},
+        )
+
+
+def test_acquired_document_does_not_accept_credentials_as_extra_fields():
+    item = SourceObject(source_id="a", uri="https://host/a", fingerprint="etag:abc")
+
+    with pytest.raises(ValidationError):
+        AcquiredDocument(source=item, content=b"a", api_key="secret")
diff --git a/harness/tht/corpus/__init__.py b/harness/tht/corpus/__init__.py
new file mode 100644
index 00000000..34ac3790
--- /dev/null
+++ b/harness/tht/corpus/__init__.py
@@ -0,0 +1 @@
+"""Canonical, transport-independent Evidence corpus."""
diff --git a/harness/tht/corpus/models.py b/harness/tht/corpus/models.py
new file mode 100644
index 00000000..10dfafec
--- /dev/null
+++ b/harness/tht/corpus/models.py
@@ -0,0 +1,64 @@
+"""Immutable records emitted by the Evidence preprocessing pipeline."""
+
+from datetime import UTC, datetime
+
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator
+
+from tht.ports.evidence import _reject_credentials
+
+
+class _CanonicalValue(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+
+class _WithMetadata(_CanonicalValue):
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+    @field_validator("metadata")
+    @classmethod
+    def metadata_has_no_credentials(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]:
+        _reject_credentials(value)
+        return value
+
+
+class CanonicalDocument(_WithMetadata):
+    document_id: str = Field(min_length=1)
+    source_id: str = Field(min_length=1)
+    source_uri: str = Field(min_length=1)
+    source_fingerprint: str = Field(min_length=1)
+    content_hash: str = Field(min_length=1)
+    title: str = ""
+    content: str
+    media_type: str = "text/plain"
+    modified_at: datetime | None = None
+    pipeline_version: str = Field(min_length=1)
+
+
+class CanonicalChunk(_WithMetadata):
+    chunk_id: str = Field(min_length=1)
+    document_id: str = Field(min_length=1)
+    ordinal: int = Field(ge=0)
+    content: str
+    content_hash: str = Field(min_length=1)
+    source_uri: str = Field(min_length=1)
+    pipeline_version: str = Field(min_length=1)
+
+
+class CorpusManifest(_WithMetadata):
+    """Description of one publishable canonical/vector generation."""
+
+    schema_version: int = Field(default=1, ge=1)
+    manifest_id: str | None = None
+    created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+    pipeline_version: str = Field(default="1", min_length=1)
+    embedding_model: str | None = None
+    embedding_dimensions: int | None = Field(default=None, gt=0)
+    vector_generation: str | None = None
+    documents: list[CanonicalDocument] = Field(default_factory=list)
+    chunks: list[CanonicalChunk] = Field(default_factory=list)
+
+    @model_validator(mode="after")
+    def embedding_fields_are_complete(self) -> "CorpusManifest":
+        if (self.embedding_model is None) != (self.embedding_dimensions is None):
+            raise ValueError("embedding_model and embedding_dimensions must be set together")
+        return self
diff --git a/harness/tht/ports/evidence.py b/harness/tht/ports/evidence.py
new file mode 100644
index 00000000..cd52f3d9
--- /dev/null
+++ b/harness/tht/ports/evidence.py
@@ -0,0 +1,74 @@
+"""Port for discovering and acquiring Evidence source objects.
+
+Source adapters own transport details and credentials.  The values crossing this
+boundary are deliberately credential-free so they can safely become provenance.
+"""
+
+from datetime import datetime
+from typing import Iterable, Protocol, runtime_checkable
+
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator
+
+
+_SECRET_KEYS = {
+    "api_key",
+    "apikey",
+    "authorization",
+    "credential",
+    "credentials",
+    "password",
+    "secret",
+    "token",
+}
+
+
+def _reject_credentials(value: JsonValue, path: str = "metadata") -> JsonValue:
+    if isinstance(value, dict):
+        for key, child in value.items():
+            normalized = key.lower().replace("-", "_")
+            if normalized in _SECRET_KEYS or normalized.endswith(("_password", "_secret", "_token")):
+                raise ValueError(f"credential-like metadata key is not allowed: {path}.{key}")
+            _reject_credentials(child, f"{path}.{key}")
+    elif isinstance(value, list):
+        for index, child in enumerate(value):
+            _reject_credentials(child, f"{path}[{index}]")
+    return value
+
+
+class _EvidenceValue(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+
+class SourceObject(_EvidenceValue):
+    source_id: str = Field(min_length=1)
+    uri: str = Field(min_length=1)
+    fingerprint: str = Field(min_length=1)
+    modified_at: datetime | None = None
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+    @field_validator("metadata")
+    @classmethod
+    def metadata_has_no_credentials(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]:
+        _reject_credentials(value)
+        return value
+
+
+class AcquiredDocument(_EvidenceValue):
+    source: SourceObject
+    content: bytes
+    media_type: str | None = None
+    acquired_at: datetime | None = None
+    metadata: dict[str, JsonValue] = Field(default_factory=dict)
+
+    @field_validator("metadata")
+    @classmethod
+    def metadata_has_no_credentials(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]:
+        _reject_credentials(value)
+        return value
+
+
+@runtime_checkable
+class EvidenceSource(Protocol):
+    def discover(self) -> Iterable[SourceObject]: ...
+
+    def acquire(self, item: SourceObject) -> AcquiredDocument: ...