mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(evidence): isolate vectors by workspace

Browse Source
This commit is contained in:
marcopan
2026-07-12 05:50:59 +02:00
parent 24f6a5daa6
commit 05accc1443
13 changed files with 163 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/tht/corpus/pipeline.py
+33 -11
View File
@@ -47,9 +47,17 @@ class PipelineResult:
resumed_from: str | None = None
def model_dump(self, mode=None):
value = asdict(self)
value["manifest"] = self.manifest.model_dump(mode="json")
return value
return {
"status": self.status,
"generation": self.generation,
"published": self.published,
"changed": list(self.changed),
"unchanged": list(self.unchanged),
"removed": list(self.removed),
"manifest": self.manifest.model_dump(mode="json"),
"run_id": self.run_id,
"resumed_from": self.resumed_from,
}
def _fingerprint(value) -> str:
@@ -62,6 +70,7 @@ class CorpusPipeline:
self, *, store: CorpusStore, sources: list[EvidenceSource], embedder,
vector_store: VectorStore, embedding_model: str, embedding_dimensions: int,
chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3,
workspace_id: str = "default",
) -> None:
self.store = store
self.sources = sources
@@ -74,6 +83,7 @@ class CorpusPipeline:
if isinstance(retain_published_generations, bool) or retain_published_generations < 1:
raise ValueError("retain_published_generations must be at least 1")
self.retain_published_generations = retain_published_generations
self.workspace_id = workspace_id
def _protected_generations(self, workspace_root: Path) -> set[str]:
protected = {value for value in (self.store.active_generation(),) if value}
@@ -92,9 +102,14 @@ class CorpusPipeline:
return protected
def gc(self, *, workspace_root: Path, dry_run: bool = False) -> dict:
active_manifest = self.store.active_manifest()
if active_manifest is not None:
persisted_workspace = active_manifest.metadata.get("workspace_id")
if isinstance(persisted_workspace, str):
self.workspace_id = persisted_workspace
published = self.store.published_generations()
list_vectors = getattr(self.vector_store, "list_evidence_generations", None)
vector_generations = set(list_vectors("evidence")) if list_vectors else set()
vector_generations = set(list_vectors("evidence", self.workspace_id)) if list_vectors else set()
generations = sorted(set(published) | vector_generations)
job_protected = self._protected_generations(workspace_root)
active = self.store.active_generation()
@@ -124,7 +139,7 @@ class CorpusPipeline:
continue
if purge_vector:
try:
self.vector_store.delete_generation("evidence", generation)
self.vector_store.delete_generation("evidence", generation, self.workspace_id)
except Exception:
failures.append({"generation": generation, "error": "vector cleanup failed"})
continue
@@ -154,6 +169,7 @@ class CorpusPipeline:
return self._run(dry_run=dry_run, resume=resume)
def run_as_job(self, **kwargs) -> PipelineResult:
self.workspace_id = kwargs["workspace_id"]
with self.store.writer_lock():
return self._run_as_job(**kwargs)
@@ -259,6 +275,7 @@ class CorpusPipeline:
vector_generation=plan["generation"],
documents=tuple(documents), chunks=tuple(chunks),
metadata={
"workspace_id": self.workspace_id,
"compatibility_fingerprint": compatibility,
"fingerprints": plan["fingerprints"],
"removed": plan["removed"],
@@ -289,7 +306,7 @@ class CorpusPipeline:
changed_docs = {doc.document_id for doc in manifest.documents if doc.source_id in plan["changed"]}
parts = [part for part in manifest.chunks if part.document_id in changed_docs]
embeddings = read(context, "embeddings.json")
return [self._vector_record(part, vector, plan["generation"])
return [self._vector_record(part, vector, plan["generation"], self.workspace_id)
for part, vector in zip(parts, embeddings, strict=True)]
def compensate(context: JobContext) -> None:
@@ -297,7 +314,7 @@ class CorpusPipeline:
if self.store.active_generation() != generation:
self.store.discard(generation)
try:
self.vector_store.delete_generation("evidence", generation)
self.vector_store.delete_generation("evidence", generation, self.workspace_id)
except Exception:
pass
write(context, "compensated.json", {"generation": generation})
@@ -495,6 +512,7 @@ class CorpusPipeline:
vector_generation=generation,
documents=tuple(documents), chunks=tuple(chunks),
metadata={
"workspace_id": self.workspace_id,
"compatibility_fingerprint": compatibility,
"fingerprints": fingerprints,
"removed": list(removed),
@@ -508,7 +526,7 @@ class CorpusPipeline:
raise PipelineError("embedding count mismatch")
if any(len(vector) != self.embedding_dimensions for vector in embeddings):
raise PipelineError("embedding dimension mismatch")
records = [self._vector_record(part, vector, generation) for part, vector in zip(changed_chunks, embeddings, strict=True)]
records = [self._vector_record(part, vector, generation, self.workspace_id) for part, vector in zip(changed_chunks, embeddings, strict=True)]
if records:
written = self.vector_store.upsert("evidence", records)
vector_written = True
@@ -547,17 +565,21 @@ class CorpusPipeline:
pass
if vector_written:
try:
self.vector_store.delete_generation("evidence", generation)
self.vector_store.delete_generation("evidence", generation, self.workspace_id)
except Exception:
pass
@staticmethod
def _vector_record(chunk: CanonicalChunk, embedding: list[float], generation: str):
def _vector_record(
chunk: CanonicalChunk, embedding: list[float], generation: str, workspace_id: str,
):
record = VectorRecord(
id=f"{generation}:{chunk.chunk_id}", kind="evidence", ref=chunk.document_id,
id=f"{workspace_id}:{generation}:{chunk.chunk_id}",
kind="evidence", ref=chunk.document_id,
title=str(chunk.metadata.get("title", "")), content=chunk.content,
metadata={
**dict(chunk.metadata), "document_id": chunk.document_id,
"workspace_id": workspace_id,
"source_uri": chunk.source_uri, "ordinal": chunk.ordinal,
"vector_generation": generation,
},