import hashlib from datetime import UTC, datetime, timedelta, timezone import pytest from pydantic import ValidationError from tht.corpus.models import CanonicalChunk, CanonicalDocument, CorpusManifest def document(source_uri: str = "https://host/a.md") -> CanonicalDocument: content = "# A" return CanonicalDocument( document_id="doc:abc", source_id="source:a", source_uri=source_uri, source_fingerprint="etag:abc", content_hash=f"sha256:{hashlib.sha256(content.encode()).hexdigest()}", title="A", content=content, media_type="text/markdown", pipeline_version="evidence-v1", ) def chunk() -> CanonicalChunk: content = "# A" return CanonicalChunk( chunk_id="chunk:abc:0", document_id="doc:abc", ordinal=0, content=content, content_hash=f"sha256:{hashlib.sha256(content.encode()).hexdigest()}", source_uri="https://host/a.md", pipeline_version="evidence-v1", ) def test_manifest_contains_provenance_without_credentials(): manifest = CorpusManifest( manifest_id="manifest:abc", created_at=datetime(2026, 7, 12, tzinfo=UTC), pipeline_version="evidence-v1", embedding_model="nomic-embed-text", embedding_dimensions=768, documents=[document()], chunks=[chunk()], ) payload = manifest.model_dump_json() assert "https://host/a.md" in payload assert "etag:abc" in payload assert "evidence-v1" in payload assert "nomic-embed-text" in payload assert "api_key" not in payload def test_manifest_can_be_assembled_before_publish_identifiers_are_assigned(): manifest = CorpusManifest(documents=[document()]) assert manifest.documents[0].source_uri == "https://host/a.md" assert manifest.manifest_id is None @pytest.mark.parametrize("model", [document(), chunk()]) def test_canonical_records_are_frozen(model): with pytest.raises(ValidationError): model.pipeline_version = "changed" # type: ignore[misc] def test_manifest_collections_are_immutable_tuples_with_json_arrays(): first = CorpusManifest( manifest_id="manifest:one", created_at=datetime.now(UTC), pipeline_version="v1" ) second = CorpusManifest( manifest_id="manifest:two", created_at=datetime.now(UTC), pipeline_version="v1" ) with pytest.raises(AttributeError): first.documents.append(document()) assert first.documents == () assert second.documents == () assert '"documents":[]' in first.model_dump_json() def test_manifest_validates_embedding_compatibility_fields(): with pytest.raises(ValidationError): CorpusManifest( manifest_id="bad", created_at=datetime.now(UTC), pipeline_version="v1", embedding_dimensions=0, ) def test_canonical_metadata_rejects_secrets_and_non_json_values(): with pytest.raises(ValidationError, match="credential-like"): CanonicalDocument.model_validate( {**document().model_dump(), "metadata": {"password": "secret"}} ) with pytest.raises(ValidationError): CanonicalChunk( chunk_id="c", document_id="d", ordinal=0, content="x", content_hash="sha256:x", source_uri="file:///x", pipeline_version="v1", metadata={"bad": object()}, ) def test_manifest_rejects_duplicate_ids_and_source_ids(): first = document() duplicate_source = first.model_copy( update={"document_id": "doc:other", "source_uri": "https://host/b.md"} ) with pytest.raises(ValidationError, match="source_id"): CorpusManifest(pipeline_version="evidence-v1", documents=[first, duplicate_source]) with pytest.raises(ValidationError, match="chunk_id"): CorpusManifest( pipeline_version="evidence-v1", documents=[first], chunks=[chunk(), chunk()] ) def test_manifest_rejects_orphan_noncontiguous_and_inconsistent_chunks(): with pytest.raises(ValidationError, match="unknown document"): CorpusManifest(pipeline_version="evidence-v1", chunks=[chunk()]) second = chunk().model_copy(update={"chunk_id": "chunk:abc:2", "ordinal": 2}) with pytest.raises(ValidationError, match="contiguous"): CorpusManifest( pipeline_version="evidence-v1", documents=[document()], chunks=[chunk(), second] ) wrong_uri = chunk().model_copy(update={"source_uri": "https://host/wrong.md"}) with pytest.raises(ValidationError, match="source_uri"): CorpusManifest( pipeline_version="evidence-v1", documents=[document()], chunks=[wrong_uri] ) def test_manifest_rejects_inconsistent_pipeline_versions(): wrong = document().model_copy(update={"pipeline_version": "other-v1"}) with pytest.raises(ValidationError, match="pipeline_version"): CorpusManifest(pipeline_version="evidence-v1", documents=[wrong]) def test_vector_generation_requires_embedding_compatibility(): with pytest.raises(ValidationError, match="vector_generation"): CorpusManifest(pipeline_version="evidence-v1", vector_generation="generation:one") @pytest.mark.parametrize( ("field", "value"), [ ("document_id", "not-namespaced"), ("content_hash", "sha256:not-hex"), ("source_uri", "https://user:pass@host/a"), ], ) def test_canonical_document_rejects_malformed_or_sensitive_provenance(field, value): with pytest.raises(ValidationError): CanonicalDocument.model_validate({**document().model_dump(), field: value}) @pytest.mark.parametrize( "source_uri", [ "https://host/a?X-Amz-Credential=abc&X-Amz-Signature=secret#access_token=bad", "https://host/a?sig=sas-secret&sp=r#section", ], ) def test_canonical_provenance_strips_query_and_fragment(source_uri): doc = document(source_uri=source_uri) canonical_chunk = chunk().model_copy(update={"source_uri": source_uri}) manifest = CorpusManifest( pipeline_version="evidence-v1", documents=[doc], chunks=[canonical_chunk] ) assert doc.source_uri == "https://host/a" assert canonical_chunk.source_uri == "https://host/a" payload = manifest.model_dump_json() assert "X-Amz" not in payload assert "sas-secret" not in payload assert "access_token" not in payload @pytest.mark.parametrize("factory", [document, chunk]) def test_content_hash_must_match_exact_canonical_utf8(factory): record = factory() with pytest.raises(ValidationError, match="exact canonical UTF-8 content"): type(record).model_validate({**record.model_dump(), "content": record.content + "\n"}) def test_model_copy_revalidates_records_and_manifests(): with pytest.raises(ValidationError, match="namespaced"): document().model_copy(update={"document_id": "invalid"}) manifest = CorpusManifest( pipeline_version="evidence-v1", embedding_model="embed-v1", embedding_dimensions=768, ) with pytest.raises(ValidationError, match="set together"): manifest.model_copy(update={"embedding_dimensions": None}) def test_manifest_datetimes_are_aware_and_normalized_to_utc(): with pytest.raises(ValidationError, match="timezone-aware"): CorpusManifest(created_at=datetime(2026, 7, 12), pipeline_version="evidence-v1") plus_two = datetime(2026, 7, 12, 12, tzinfo=timezone(timedelta(hours=2))) manifest = CorpusManifest(created_at=plus_two, pipeline_version="evidence-v1") assert manifest.created_at.tzinfo is UTC assert manifest.created_at.hour == 10