mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix: harden model catalog projections

Browse Source
This commit is contained in:
Codex
2026-09-02 19:25:01 +02:00
parent ce4c31a6fb
commit a6a5bf2036
38 changed files with 573 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/tht/evidence/corpus/pipeline.py
+10 -1
View File
@@ -125,6 +125,7 @@ class CorpusPipeline:
def __init__(
self, *, store: CorpusStore, sources: list[EvidenceSource], embedder,
vector_store: VectorStore, embedding_model: str, embedding_dimensions: int,
embedding_id: str | None = None,
chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3,
workspace_id: str | None = None, sparse_language: str = "italian",
candidate_evaluator: Callable[[CorpusManifest], object] | None = None,
@@ -133,6 +134,7 @@ class CorpusPipeline:
self.sources = sources
self.embedder = embedder
self.vector_store = vector_store
self.embedding_id = embedding_id or f"ollama/{embedding_model}"
self.embedding_model = embedding_model
self.embedding_dimensions = embedding_dimensions
self.chunk_policy = chunk_policy
@@ -284,6 +286,7 @@ class CorpusPipeline:
source_by_id = {item.source_id: (source, item) for source, item in discovered}
compatibility = _fingerprint({
"pipeline": self.pipeline_version,
"embedding_id": self.embedding_id,
"model": self.embedding_model,
"dimensions": self.embedding_dimensions,
"chunk_policy": asdict(self.chunk_policy),
@@ -294,6 +297,7 @@ class CorpusPipeline:
"compatibility_fingerprint": compatibility,
"pipeline_version": self.pipeline_version,
"chunk_policy_version": self.chunk_policy.version,
"embedding_id": self.embedding_id,
"embedding_model": self.embedding_model,
"embedding_dimensions": self.embedding_dimensions,
}
@@ -492,7 +496,9 @@ class CorpusPipeline:
) for document in documents
}
manifest = CorpusManifest(
schema_version=2,
pipeline_version=self.pipeline_version,
embedding_id=self.embedding_id,
embedding_model=self.embedding_model,
embedding_dimensions=self.embedding_dimensions,
vector_generation=plan["generation"],
@@ -723,7 +729,8 @@ class CorpusPipeline:
prior_documents = {doc.source_id: doc for doc in previous.documents} if previous else {}
fingerprints = {item.source_id: item.fingerprint for _, item in discovered}
compatibility = _fingerprint({
"pipeline": self.pipeline_version, "model": self.embedding_model,
"pipeline": self.pipeline_version, "embedding_id": self.embedding_id,
"model": self.embedding_model,
"dimensions": self.embedding_dimensions, "chunk_policy": asdict(self.chunk_policy),
})
previous_compatibility = previous.metadata.get("compatibility_fingerprint") if previous else None
@@ -761,7 +768,9 @@ class CorpusPipeline:
for document in documents
}
manifest = CorpusManifest(
schema_version=2,
pipeline_version=self.pipeline_version,
embedding_id=self.embedding_id,
embedding_model=self.embedding_model,
embedding_dimensions=self.embedding_dimensions,
vector_generation=generation,