mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(evidence): validate active corpus artifacts

Browse Source
This commit is contained in:
marcopan
2026-07-12 06:35:27 +02:00
parent 5cc023f390
commit 03ee3ffda8
3 changed files with 153 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/tht/corpus/pipeline.py
+68 -8
View File
@@ -15,7 +15,7 @@ from tht.corpus.chunk import ChunkPolicy, chunk
from tht.corpus.models import CanonicalChunk, CanonicalDocument, CorpusManifest
from tht.corpus.normalize import normalize
from tht.corpus.store import CorpusStore
from tht.ports.evidence import EvidenceSource, SourceObject
from tht.ports.evidence import EvidenceSource, SourceObject, canonical_provenance_uri
from tht.ports.vector import VectorStore, VectorWriteRecord
from tht.vectorstore.records import VectorRecord
from tht.jobs.models import JobSpec
@@ -259,12 +259,21 @@ class CorpusPipeline:
}
previous = self.store.active_manifest()
def document_sources(manifest: CorpusManifest) -> dict[str, dict[str, str]]:
def document_sources(manifest: CorpusManifest) -> dict[str, dict]:
return {
document.document_id: {
"document_id": document.document_id,
"source_id": document.source_id,
"source_uri": document.source_uri,
"source_fingerprint": document.source_fingerprint,
"modified_at": (
document.modified_at.isoformat().replace("+00:00", "Z")
if document.modified_at else None
),
"source_metadata": _canonical_json(document.metadata.get("source")),
"media_type": document.media_type,
"content_hash": document.content_hash,
"pipeline_version": document.pipeline_version,
}
for document in manifest.documents
}
@@ -283,19 +292,59 @@ class CorpusPipeline:
):
return False
for source_id, document in actual_documents.items():
source = persisted_snapshot[source_id]
source_payload = persisted_snapshot[source_id]
source = SourceObject.model_validate({
"source_id": source_payload["source_id"],
"uri": source_payload["uri"],
"fingerprint": source_payload["fingerprint"],
"modified_at": source_payload["modified_at"],
"metadata": source_payload["metadata"],
})
content = self.store.read_document(document.document_id, manifest.manifest_id)
expected_uri = canonical_provenance_uri(source.uri)
expected_id = "doc:" + hashlib.sha256(
f"{source.source_id}\n{expected_uri}".encode()
).hexdigest()
expected_media_type = source_payload.get("media_type")
if (
document.source_uri != source["uri"]
or document.source_fingerprint != source["fingerprint"]
or self.store.read_document(document.document_id, manifest.manifest_id)
!= document.content
content != document.content
or document.document_id != expected_id
or document.source_id != source.source_id
or document.source_uri != expected_uri
or document.source_fingerprint != source.fingerprint
or document.modified_at != source.modified_at
or _canonical_json(document.metadata.get("source"))
!= _canonical_json(source.metadata)
or (
isinstance(expected_media_type, str)
and document.media_type != expected_media_type
)
or document.pipeline_version != self.pipeline_version
):
return False
expected_chunks = tuple(
part for document in manifest.documents for part in chunk(document, self.chunk_policy)
)
if any(document.content and not chunk(document, self.chunk_policy)
for document in manifest.documents):
return False
if _canonical_json([part.model_dump(mode="json") for part in manifest.chunks]) != (
_canonical_json([part.model_dump(mode="json") for part in expected_chunks])
):
return False
generations = manifest.metadata.get("document_generations")
if not isinstance(generations, Mapping):
return False
health = self.vector_store.health()
if (
not health.ok
or health.dimension_compatible is not True
or health.expected_dimension != self.embedding_dimensions
or health.observed_dimensions != (self.embedding_dimensions,)
):
return False
existing = self.vector_store.existing_hashes("evidence", ["evidence"])
for part in manifest.chunks:
for part in expected_chunks:
generation = generations.get(part.document_id)
if not isinstance(generation, str):
return False
@@ -408,9 +457,20 @@ class CorpusPipeline:
"source_snapshot": plan["source_snapshot"],
"document_sources": {
document.document_id: {
"document_id": document.document_id,
"source_id": document.source_id,
"source_uri": document.source_uri,
"source_fingerprint": document.source_fingerprint,
"modified_at": (
document.modified_at.isoformat().replace("+00:00", "Z")
if document.modified_at else None
),
"source_metadata": _canonical_json(
document.metadata.get("source")
),
"media_type": document.media_type,
"content_hash": document.content_hash,
"pipeline_version": document.pipeline_version,
}
for document in documents
},