mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(evidence): harden canonical corpus contracts

Browse Source
This commit is contained in:
marcopan
2026-07-12 03:10:30 +02:00
parent d702aad93d
commit 4424fd3d90
5 changed files with 481 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/tests/test_corpus_models.py
+79 -11
View File
@@ -1,4 +1,4 @@
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta, timezone
import pytest
from pydantic import ValidationError
@@ -12,11 +12,11 @@ def document(source_uri: str = "https://host/a.md") -> CanonicalDocument:
source_id="source:a",
source_uri=source_uri,
source_fingerprint="etag:abc",
content_hash="sha256:def",
content_hash=f"sha256:{'d' * 64}",
title="A",
content="# A",
media_type="text/markdown",
pipeline_version="normalize-v1",
pipeline_version="evidence-v1",
)
@@ -26,9 +26,9 @@ def chunk() -> CanonicalChunk:
document_id="doc:abc",
ordinal=0,
content="# A",
content_hash="sha256:def",
content_hash=f"sha256:{'e' * 64}",
source_uri="https://host/a.md",
pipeline_version="chunk-v1",
pipeline_version="evidence-v1",
)
@@ -65,17 +65,19 @@ def test_canonical_records_are_frozen(model):
model.pipeline_version = "changed" # type: ignore[misc]
def test_manifest_collections_have_independent_defaults():
def test_manifest_collections_are_immutable_tuples_with_json_arrays():
first = CorpusManifest(
manifest_id="one", created_at=datetime.now(UTC), pipeline_version="v1"
manifest_id="manifest:one", created_at=datetime.now(UTC), pipeline_version="v1"
)
second = CorpusManifest(
manifest_id="two", created_at=datetime.now(UTC), pipeline_version="v1"
manifest_id="manifest:two", created_at=datetime.now(UTC), pipeline_version="v1"
)
first.documents.append(document())
assert second.documents == []
with pytest.raises(AttributeError):
first.documents.append(document())
assert first.documents == ()
assert second.documents == ()
assert '"documents":[]' in first.model_dump_json()
def test_manifest_validates_embedding_compatibility_fields():
@@ -104,3 +106,69 @@ def test_canonical_metadata_rejects_secrets_and_non_json_values():
pipeline_version="v1",
metadata={"bad": object()},
)
def test_manifest_rejects_duplicate_ids_and_source_ids():
first = document()
duplicate_source = first.model_copy(
update={"document_id": "doc:other", "source_uri": "https://host/b.md"}
)
with pytest.raises(ValidationError, match="source_id"):
CorpusManifest(pipeline_version="evidence-v1", documents=[first, duplicate_source])
with pytest.raises(ValidationError, match="chunk_id"):
CorpusManifest(
pipeline_version="evidence-v1", documents=[first], chunks=[chunk(), chunk()]
)
def test_manifest_rejects_orphan_noncontiguous_and_inconsistent_chunks():
with pytest.raises(ValidationError, match="unknown document"):
CorpusManifest(pipeline_version="evidence-v1", chunks=[chunk()])
second = chunk().model_copy(update={"chunk_id": "chunk:abc:2", "ordinal": 2})
with pytest.raises(ValidationError, match="contiguous"):
CorpusManifest(
pipeline_version="evidence-v1", documents=[document()], chunks=[chunk(), second]
)
wrong_uri = chunk().model_copy(update={"source_uri": "https://host/wrong.md"})
with pytest.raises(ValidationError, match="source_uri"):
CorpusManifest(
pipeline_version="evidence-v1", documents=[document()], chunks=[wrong_uri]
)
def test_manifest_rejects_inconsistent_pipeline_versions():
wrong = document().model_copy(update={"pipeline_version": "other-v1"})
with pytest.raises(ValidationError, match="pipeline_version"):
CorpusManifest(pipeline_version="evidence-v1", documents=[wrong])
def test_vector_generation_requires_embedding_compatibility():
with pytest.raises(ValidationError, match="vector_generation"):
CorpusManifest(pipeline_version="evidence-v1", vector_generation="generation:one")
@pytest.mark.parametrize(
("field", "value"),
[
("document_id", "not-namespaced"),
("content_hash", "sha256:not-hex"),
("source_uri", "https://user:pass@host/a"),
("source_uri", "https://host/a?refresh_token=secret"),
],
)
def test_canonical_document_rejects_malformed_or_sensitive_provenance(field, value):
with pytest.raises(ValidationError):
CanonicalDocument.model_validate({**document().model_dump(), field: value})
def test_manifest_datetimes_are_aware_and_normalized_to_utc():
with pytest.raises(ValidationError, match="timezone-aware"):
CorpusManifest(created_at=datetime(2026, 7, 12), pipeline_version="evidence-v1")
plus_two = datetime(2026, 7, 12, 12, tzinfo=timezone(timedelta(hours=2)))
manifest = CorpusManifest(created_at=plus_two, pipeline_version="evidence-v1")
assert manifest.created_at.tzinfo is UTC
assert manifest.created_at.hour == 10
harness/tests/test_evidence_port_contract.py
+146 -11
View File
@@ -1,9 +1,15 @@
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta, timezone
import pytest
from pydantic import ValidationError
from tht.ports.evidence import AcquiredDocument, EvidenceSource, SourceObject
from tht.ports.evidence import (
AcquiredDocument,
EvidenceSource,
EvidenceSourceError,
EvidenceSourceErrorCategory,
SourceObject,
)
class StubSource:
@@ -11,7 +17,7 @@ class StubSource:
return iter(
[
SourceObject(
source_id="handbook",
source_id="source:handbook",
uri="https://host/handbook.md",
fingerprint="sha256:abc",
)
@@ -35,35 +41,164 @@ def test_runtime_checkable_source_protocol():
def test_source_objects_are_frozen_and_metadata_defaults_are_independent():
first = SourceObject(source_id="a", uri="file:///a", fingerprint="sha256:a")
second = SourceObject(source_id="b", uri="file:///b", fingerprint="sha256:b")
first = SourceObject(source_id="source:a", uri="file:///a", fingerprint="sha256:a")
second = SourceObject(source_id="source:b", uri="file:///b", fingerprint="sha256:b")
with pytest.raises(ValidationError):
first.uri = "file:///changed" # type: ignore[misc]
first.metadata["owner"] = "team-a"
with pytest.raises(TypeError):
first.metadata["owner"] = "team-a"
assert second.metadata == {}
@pytest.mark.parametrize("metadata", [{"api_key": "secret"}, {"auth": {"token": "secret"}}])
def test_source_metadata_rejects_credentials(metadata):
@pytest.mark.parametrize(
"key",
[
"password",
"PassWd",
"api_key",
"x-api-key",
"accessToken",
"refresh.token",
"client secret",
"privateKey",
"session_cookie",
"Authorization",
],
)
def test_source_metadata_rejects_credential_specific_keys(key):
with pytest.raises(ValidationError, match="credential-like"):
SourceObject(
source_id="a", uri="https://host/a", fingerprint="etag:abc", metadata=metadata
source_id="source:a",
uri="https://host/a",
fingerprint="etag:abc",
metadata={"nested": [{key: "secret"}]},
)
def test_source_metadata_allows_benign_generic_token_and_secret_labels():
source = SourceObject(
source_id="source:a",
uri="https://host/a",
fingerprint="etag:abc",
metadata={"token": "word count token", "secret": False},
)
assert source.metadata["token"] == "word count token"
def test_nested_metadata_is_recursively_immutable_and_serializes_as_json():
source = SourceObject(
source_id="source:a",
uri="https://host/a",
fingerprint="etag:abc",
metadata={"nested": {"items": [1, {"ok": True}]}},
)
with pytest.raises(TypeError):
source.metadata["nested"]["items"][1]["ok"] = False
assert '"items":[1,{"ok":true}]' in source.model_dump_json()
@pytest.mark.parametrize(
"uri",
[
"https://user:pass@host/a",
"https://host/a?api_key=secret",
"https://host/a?accessToken=secret",
],
)
def test_source_uri_rejects_embedded_credentials(uri):
with pytest.raises(ValidationError, match="credentials"):
SourceObject(source_id="source:a", uri=uri, fingerprint="etag:abc")
def test_source_metadata_must_be_json_safe():
with pytest.raises(ValidationError):
SourceObject(
source_id="a",
source_id="source:a",
uri="file:///a",
fingerprint="sha256:a",
metadata={"path": object()},
)
def test_source_identity_and_fingerprint_must_be_namespaced():
with pytest.raises(ValidationError, match="namespaced"):
SourceObject(source_id="plain", uri="file:///a", fingerprint="sha256:a")
with pytest.raises(ValidationError, match="namespaced"):
SourceObject(source_id="source:a", uri="file:///a", fingerprint="plain")
def test_acquired_document_does_not_accept_credentials_as_extra_fields():
item = SourceObject(source_id="a", uri="https://host/a", fingerprint="etag:abc")
item = SourceObject(source_id="source:a", uri="https://host/a", fingerprint="etag:abc")
with pytest.raises(ValidationError):
AcquiredDocument(source=item, content=b"a", api_key="secret")
def test_acquired_binary_content_has_explicit_json_round_trip():
item = SourceObject(source_id="source:a", uri="https://host/a", fingerprint="etag:abc")
acquired = AcquiredDocument(source=item, content=b"\x00\xffbinary\x80")
payload = acquired.model_dump_json()
restored = AcquiredDocument.model_validate_json(payload)
assert restored.content == acquired.content
assert "binary" not in payload
def test_datetimes_must_be_aware_and_are_normalized_to_utc():
with pytest.raises(ValidationError, match="timezone-aware"):
SourceObject(
source_id="source:a",
uri="https://host/a",
fingerprint="etag:abc",
modified_at=datetime(2026, 7, 12),
)
source = SourceObject(
source_id="source:a",
uri="https://host/a",
fingerprint="etag:abc",
modified_at=datetime(2026, 7, 12, 4, tzinfo=timezone(timedelta(hours=2))),
)
assert source.modified_at.tzinfo is UTC
assert source.modified_at.hour == 2
acquired = AcquiredDocument(
source=source,
content=b"a",
acquired_at=datetime(2026, 7, 12, 2, tzinfo=UTC) + timedelta(hours=0),
)
assert acquired.acquired_at.utcoffset() == timedelta(0)
def test_source_errors_are_typed_retryable_and_safe():
transient = EvidenceSourceError(
"remote source unavailable",
category=EvidenceSourceErrorCategory.TRANSIENT,
details={"status": 503},
)
permanent = EvidenceSourceError(
"unsupported media type",
category=EvidenceSourceErrorCategory.PERMANENT,
)
assert transient.retryable is True
assert permanent.retryable is False
assert transient.details["status"] == 503
with pytest.raises(TypeError):
transient.details["status"] = 200
with pytest.raises(ValueError, match="credential-like"):
EvidenceSourceError(
"bad",
category=EvidenceSourceErrorCategory.PERMANENT,
details={"apiKey": "must-not-leak"},
)
with pytest.raises(ValidationError):
EvidenceSourceError(
"bad",
category=EvidenceSourceErrorCategory.PERMANENT,
details={"not_json": object()},
)