mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(evidence): isolate vectors by workspace

Browse Source
This commit is contained in:
marcopan
2026-07-12 05:50:59 +02:00
parent 24f6a5daa6
commit 05accc1443
13 changed files with 163 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/tht/adapters/vector/pgvector.py
+17 -9
View File
@@ -273,16 +273,18 @@ class PgVectorStore:
filter_params.append(collection_kinds)
if metadata_filter is not None:
if collection != "evidence" or set(metadata_filter) != {
"vector_generation", "document_ids"
"vector_generation", "document_ids", "workspace_id"
}:
raise VectorStoreError("Unsupported vector metadata filter")
generation = metadata_filter["vector_generation"]
document_ids = metadata_filter["document_ids"]
if not isinstance(generation, str) or not isinstance(document_ids, list):
workspace_id = metadata_filter["workspace_id"]
if not isinstance(generation, str) or not isinstance(document_ids, list) or not isinstance(workspace_id, str):
raise VectorStoreError("Invalid vector metadata filter")
clauses.append(sql.SQL("metadata->>'vector_generation' = %s"))
clauses.append(sql.SQL("metadata->>'document_id' = ANY(%s)"))
filter_params.extend((generation, document_ids))
clauses.append(sql.SQL("metadata->>'workspace_id' = %s"))
filter_params.extend((generation, document_ids, workspace_id))
where = (
sql.SQL(" WHERE ") + sql.SQL(" AND ").join(clauses)
if clauses else sql.SQL("")
@@ -404,9 +406,11 @@ class PgVectorStore:
raw.close()
return len(records)
def delete_generation(self, collection: str, generation: str) -> int:
def delete_generation(self, collection: str, generation: str, workspace_id: str) -> int:
if collection != "evidence" or re.fullmatch(r"gen:[0-9a-f]{32}", generation) is None:
raise VectorStoreError("Only exact Evidence generations may be deleted")
if re.fullmatch(r"[a-z][a-z0-9_-]{0,63}", workspace_id) is None:
raise VectorStoreError("Invalid Evidence workspace namespace")
raw = None
try:
raw = self._require_writer().raw_connection()
@@ -414,9 +418,10 @@ class PgVectorStore:
cursor.execute(
sql.SQL(
"DELETE FROM {} WHERE kind = 'evidence' "
"AND metadata->>'vector_generation' = %s"
"AND metadata->>'vector_generation' = %s "
"AND metadata->>'workspace_id' = %s"
).format(_collection(self._schema, collection)),
(generation,),
(generation, workspace_id),
)
count = cursor.rowcount
raw.commit()
@@ -429,9 +434,11 @@ class PgVectorStore:
if raw is not None:
raw.close()
def list_evidence_generations(self, collection: str) -> list[str]:
def list_evidence_generations(self, collection: str, workspace_id: str) -> list[str]:
if collection != "evidence":
raise VectorStoreError("Only exact Evidence generations may be listed")
if re.fullmatch(r"[a-z][a-z0-9_-]{0,63}", workspace_id) is None:
raise VectorStoreError("Invalid Evidence workspace namespace")
raw = None
try:
raw = self._require_writer().raw_connection()
@@ -440,8 +447,9 @@ class PgVectorStore:
sql.SQL(
"SELECT DISTINCT metadata->>'vector_generation' FROM {} "
"WHERE kind = 'evidence' AND metadata->>'vector_generation' "
"~ '^gen:[0-9a-f]{{32}}$' ORDER BY 1"
).format(_collection(self._schema, collection))
"~ '^gen:[0-9a-f]{{32}}$' AND metadata->>'workspace_id' = %s ORDER BY 1"
).format(_collection(self._schema, collection)),
(workspace_id,),
)
return [row[0] for row in cursor.fetchall()]
except Exception as exc:
harness/tht/adapters/vector/thoth_http.py
+4 -4
View File
@@ -155,19 +155,19 @@ class ThothHttpVectorStore:
except VectorRestError as exc:
raise VectorStoreError(str(exc)) from exc
def delete_generation(self, collection: str, generation: str) -> int:
def delete_generation(self, collection: str, generation: str, workspace_id: str) -> int:
if collection != "evidence" or re.fullmatch(r"gen:[0-9a-f]{32}", generation) is None:
raise VectorStoreError("Only exact Evidence generations may be deleted")
try:
return self._require_writer().delete_generation(collection, generation)
return self._require_writer().delete_generation(collection, generation, workspace_id)
except VectorRestError as exc:
raise VectorStoreError(str(exc)) from exc
def list_evidence_generations(self, collection: str) -> list[str]:
def list_evidence_generations(self, collection: str, workspace_id: str) -> list[str]:
if collection != "evidence":
raise VectorStoreError("Only exact Evidence generations may be listed")
try:
return self._require_writer().list_evidence_generations(collection)
return self._require_writer().list_evidence_generations(collection, workspace_id)
except VectorRestError as exc:
raise VectorWriteUnavailable("Vector generation inventory unavailable") from exc
harness/tht/corpus/pipeline.py
+33 -11
View File
@@ -47,9 +47,17 @@ class PipelineResult:
resumed_from: str | None = None
def model_dump(self, mode=None):
value = asdict(self)
value["manifest"] = self.manifest.model_dump(mode="json")
return value
return {
"status": self.status,
"generation": self.generation,
"published": self.published,
"changed": list(self.changed),
"unchanged": list(self.unchanged),
"removed": list(self.removed),
"manifest": self.manifest.model_dump(mode="json"),
"run_id": self.run_id,
"resumed_from": self.resumed_from,
}
def _fingerprint(value) -> str:
@@ -62,6 +70,7 @@ class CorpusPipeline:
self, *, store: CorpusStore, sources: list[EvidenceSource], embedder,
vector_store: VectorStore, embedding_model: str, embedding_dimensions: int,
chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3,
workspace_id: str = "default",
) -> None:
self.store = store
self.sources = sources
@@ -74,6 +83,7 @@ class CorpusPipeline:
if isinstance(retain_published_generations, bool) or retain_published_generations < 1:
raise ValueError("retain_published_generations must be at least 1")
self.retain_published_generations = retain_published_generations
self.workspace_id = workspace_id
def _protected_generations(self, workspace_root: Path) -> set[str]:
protected = {value for value in (self.store.active_generation(),) if value}
@@ -92,9 +102,14 @@ class CorpusPipeline:
return protected
def gc(self, *, workspace_root: Path, dry_run: bool = False) -> dict:
active_manifest = self.store.active_manifest()
if active_manifest is not None:
persisted_workspace = active_manifest.metadata.get("workspace_id")
if isinstance(persisted_workspace, str):
self.workspace_id = persisted_workspace
published = self.store.published_generations()
list_vectors = getattr(self.vector_store, "list_evidence_generations", None)
vector_generations = set(list_vectors("evidence")) if list_vectors else set()
vector_generations = set(list_vectors("evidence", self.workspace_id)) if list_vectors else set()
generations = sorted(set(published) | vector_generations)
job_protected = self._protected_generations(workspace_root)
active = self.store.active_generation()
@@ -124,7 +139,7 @@ class CorpusPipeline:
continue
if purge_vector:
try:
self.vector_store.delete_generation("evidence", generation)
self.vector_store.delete_generation("evidence", generation, self.workspace_id)
except Exception:
failures.append({"generation": generation, "error": "vector cleanup failed"})
continue
@@ -154,6 +169,7 @@ class CorpusPipeline:
return self._run(dry_run=dry_run, resume=resume)
def run_as_job(self, **kwargs) -> PipelineResult:
self.workspace_id = kwargs["workspace_id"]
with self.store.writer_lock():
return self._run_as_job(**kwargs)
@@ -259,6 +275,7 @@ class CorpusPipeline:
vector_generation=plan["generation"],
documents=tuple(documents), chunks=tuple(chunks),
metadata={
"workspace_id": self.workspace_id,
"compatibility_fingerprint": compatibility,
"fingerprints": plan["fingerprints"],
"removed": plan["removed"],
@@ -289,7 +306,7 @@ class CorpusPipeline:
changed_docs = {doc.document_id for doc in manifest.documents if doc.source_id in plan["changed"]}
parts = [part for part in manifest.chunks if part.document_id in changed_docs]
embeddings = read(context, "embeddings.json")
return [self._vector_record(part, vector, plan["generation"])
return [self._vector_record(part, vector, plan["generation"], self.workspace_id)
for part, vector in zip(parts, embeddings, strict=True)]
def compensate(context: JobContext) -> None:
@@ -297,7 +314,7 @@ class CorpusPipeline:
if self.store.active_generation() != generation:
self.store.discard(generation)
try:
self.vector_store.delete_generation("evidence", generation)
self.vector_store.delete_generation("evidence", generation, self.workspace_id)
except Exception:
pass
write(context, "compensated.json", {"generation": generation})
@@ -495,6 +512,7 @@ class CorpusPipeline:
vector_generation=generation,
documents=tuple(documents), chunks=tuple(chunks),
metadata={
"workspace_id": self.workspace_id,
"compatibility_fingerprint": compatibility,
"fingerprints": fingerprints,
"removed": list(removed),
@@ -508,7 +526,7 @@ class CorpusPipeline:
raise PipelineError("embedding count mismatch")
if any(len(vector) != self.embedding_dimensions for vector in embeddings):
raise PipelineError("embedding dimension mismatch")
records = [self._vector_record(part, vector, generation) for part, vector in zip(changed_chunks, embeddings, strict=True)]
records = [self._vector_record(part, vector, generation, self.workspace_id) for part, vector in zip(changed_chunks, embeddings, strict=True)]
if records:
written = self.vector_store.upsert("evidence", records)
vector_written = True
@@ -547,17 +565,21 @@ class CorpusPipeline:
pass
if vector_written:
try:
self.vector_store.delete_generation("evidence", generation)
self.vector_store.delete_generation("evidence", generation, self.workspace_id)
except Exception:
pass
@staticmethod
def _vector_record(chunk: CanonicalChunk, embedding: list[float], generation: str):
def _vector_record(
chunk: CanonicalChunk, embedding: list[float], generation: str, workspace_id: str,
):
record = VectorRecord(
id=f"{generation}:{chunk.chunk_id}", kind="evidence", ref=chunk.document_id,
id=f"{workspace_id}:{generation}:{chunk.chunk_id}",
kind="evidence", ref=chunk.document_id,
title=str(chunk.metadata.get("title", "")), content=chunk.content,
metadata={
**dict(chunk.metadata), "document_id": chunk.document_id,
"workspace_id": workspace_id,
"source_uri": chunk.source_uri, "ordinal": chunk.ordinal,
"vector_generation": generation,
},
harness/tht/ports/vector.py
+2 -2
View File
@@ -80,9 +80,9 @@ class VectorStore(Protocol):
def upsert(self, collection: str, records: list[VectorWriteRecord]) -> int: ...
def delete_generation(self, collection: str, generation: str) -> int: ...
def delete_generation(self, collection: str, generation: str, workspace_id: str) -> int: ...
def list_evidence_generations(self, collection: str) -> list[str]: ...
def list_evidence_generations(self, collection: str, workspace_id: str) -> list[str]: ...
__all__ = [
harness/tht/search/evidence.py
+3 -1
View File
@@ -30,7 +30,8 @@ class ActiveEvidenceSearcher:
hits.extend(self.delegate.search(embedding, **kwargs))
if include_evidence:
manifest = self.corpus.active_manifest()
if manifest is not None:
workspace_id = manifest.metadata.get("workspace_id") if manifest else None
if manifest is not None and isinstance(workspace_id, str):
by_generation: dict[str, list[str]] = {}
mapping = dict(manifest.metadata.get("document_generations", {}))
for document in manifest.documents:
@@ -43,6 +44,7 @@ class ActiveEvidenceSearcher:
metadata_filter={
"vector_generation": generation,
"document_ids": sorted(document_ids),
"workspace_id": workspace_id,
},
))
return sorted(hits, key=lambda hit: (-hit.similarity, hit.id))[:top_n]
harness/tht/vectorstore/rest_client.py
+9 -4
View File
@@ -128,13 +128,16 @@ class VectorRestClient:
return len(payload)
return len(rows)
def delete_generation(self, table_name: str, generation: str) -> int:
def delete_generation(self, table_name: str, generation: str, workspace_id: str) -> int:
if table_name != "evidence" or re.fullmatch(r"gen:[0-9a-f]{32}", generation) is None:
raise ValueError("generation must be canonical")
if re.fullmatch(r"[a-z][a-z0-9_-]{0,63}", workspace_id) is None:
raise ValueError("workspace namespace must be canonical")
try:
payload = self._call(
"delete_vector_generation",
{"table_name": table_name, "kind": "evidence", "generation": generation},
{"table_name": table_name, "kind": "evidence", "generation": generation,
"workspace_id": workspace_id},
)
except VectorRestError as error:
if "HTTP 404" in str(error):
@@ -146,11 +149,13 @@ class VectorRestClient:
return int(payload.get("deleted", 0))
return 0
def list_evidence_generations(self, table_name: str) -> list[str]:
def list_evidence_generations(self, table_name: str, workspace_id: str) -> list[str]:
if re.fullmatch(r"[a-z][a-z0-9_-]{0,63}", workspace_id) is None:
raise ValueError("workspace namespace must be canonical")
try:
rows = self._call(
"list_evidence_generations",
{"table_name": table_name, "kind": "evidence"},
{"table_name": table_name, "kind": "evidence", "workspace_id": workspace_id},
) or []
except VectorRestError as error:
if "HTTP 404" in str(error):