mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(evidence): isolate vectors by workspace

Browse Source
This commit is contained in:
marcopan
2026-07-12 05:50:59 +02:00
parent 24f6a5daa6
commit 05accc1443
13 changed files with 163 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
harness/tests/l0/test_pgvector_corpus_lifecycle.py
+5 -2
View File
@@ -241,7 +241,8 @@ def test_real_pgvector_corpus_job_lifecycle(tmp_path, persistent_pgvector):
record=VectorRecord(
id="chunk:exact-vector-orphan", kind="evidence", ref="doc:orphan",
title="orphan", content="orphan",
metadata={"document_id": "doc:orphan", "vector_generation": orphan},
metadata={"document_id": "doc:orphan", "vector_generation": orphan,
"workspace_id": "pgvector-lifecycle"},
),
embedding=query, content_hash="sha256:" + "f" * 64,
)])
@@ -251,5 +252,7 @@ def test_real_pgvector_corpus_job_lifecycle(tmp_path, persistent_pgvector):
expected_fs = set(generations[-2:])
assert set(CorpusStore(tmp_path / "corpus").list_generations()) == expected_fs
expected_vectors = expected_fs | {generations[0]}
assert set(recreated.list_evidence_generations("evidence")) == expected_vectors
assert set(recreated.list_evidence_generations(
"evidence", "pgvector-lifecycle"
)) == expected_vectors
assert final_pipeline.gc(workspace_root=tmp_path)["evicted"] == []
harness/tests/l0/test_pgvector_store.py
+19 -4
View File
@@ -130,14 +130,29 @@ def test_pgvector_lists_and_deletes_exact_evidence_generation(store):
value = VectorWriteRecord(
record=VectorRecord(
id="evidence-generation-a", kind="evidence", ref="doc:a", title="a",
content="content", metadata={"vector_generation": generation},
content="content", metadata={"vector_generation": generation, "workspace_id": "default"},
),
embedding=[1.0, 0.0], content_hash="sha256:" + "a" * 64,
)
store.upsert("evidence", [value])
assert generation in store.list_evidence_generations("evidence")
assert store.delete_generation("evidence", generation) == 1
assert generation not in store.list_evidence_generations("evidence")
assert generation in store.list_evidence_generations("evidence", "default")
assert store.delete_generation("evidence", generation, "default") == 1
assert generation not in store.list_evidence_generations("evidence", "default")
def test_pgvector_generation_cleanup_isolated_between_workspaces(store):
generation = "gen:" + "b" * 32
records = [VectorWriteRecord(
record=VectorRecord(
id=f"evidence-{workspace}", kind="evidence", ref=f"doc:{workspace}",
title=workspace, content=workspace,
metadata={"vector_generation": generation, "workspace_id": workspace},
), embedding=[1.0, 0.0], content_hash="sha256:" + key * 64,
) for workspace, key in (("workspace-a", "b"), ("workspace-b", "c"))]
store.upsert("evidence", records)
assert store.delete_generation("evidence", generation, "workspace-a") == 1
assert generation not in store.list_evidence_generations("evidence", "workspace-a")
assert generation in store.list_evidence_generations("evidence", "workspace-b")
def test_pgvector_search_filters_kinds_before_limit(store):
harness/tests/l0/test_vector_adapter_parity.py
+7 -6
View File
@@ -235,9 +235,10 @@ def test_http_delete_generation_uses_exact_allowlisted_rpc_payload(monkeypatch):
lambda url, json, **kwargs: calls.append((url, json)) or Response({"deleted": 2}),
)
client = VectorRestClient(RestConfig(base_url="https://vectors.test", api_key="writer"))
assert client.delete_generation("evidence", "gen:" + "a" * 32) == 2
assert client.delete_generation("evidence", "gen:" + "a" * 32, "default") == 2
assert calls == [("https://vectors.test/rpc/delete_vector_generation", {
"table_name": "evidence", "kind": "evidence", "generation": "gen:" + "a" * 32,
"workspace_id": "default",
})]
@@ -248,7 +249,7 @@ def test_http_delete_generation_legacy_404_fails_closed_without_body_leak(monkey
)
client = VectorRestClient(RestConfig(base_url="https://vectors.test", api_key="writer"))
with pytest.raises(VectorRestError, match="delete_vector_generation RPC is unavailable") as error:
client.delete_generation("evidence", "gen:" + "a" * 32)
client.delete_generation("evidence", "gen:" + "a" * 32, "default")
assert "secret" not in str(error.value)
@@ -261,9 +262,9 @@ def test_http_list_evidence_generations_exact_rpc_and_legacy_fail_closed(monkeyp
]),
)
client = VectorRestClient(RestConfig(base_url="https://vectors.test", api_key="writer"))
assert client.list_evidence_generations("evidence") == ["gen:" + "a" * 32]
assert client.list_evidence_generations("evidence", "default") == ["gen:" + "a" * 32]
assert calls[0][0].endswith("/rpc/list_evidence_generations")
assert calls[0][1] == {"table_name": "evidence", "kind": "evidence"}
assert calls[0][1] == {"table_name": "evidence", "kind": "evidence", "workspace_id": "default"}
@pytest.mark.parametrize("generation", ["gen:a", "gen:" + "A" * 32, "gen:" + "a" * 33])
@@ -274,7 +275,7 @@ def test_http_generation_operations_reject_noncanonical_values(monkeypatch, gene
)
client = VectorRestClient(RestConfig(base_url="https://vectors.test", api_key="writer"))
with pytest.raises(ValueError, match="canonical"):
client.delete_generation("evidence", generation)
client.delete_generation("evidence", generation, "default")
def test_http_inventory_rejects_malformed_rpc_output(monkeypatch):
@@ -284,4 +285,4 @@ def test_http_inventory_rejects_malformed_rpc_output(monkeypatch):
)
client = VectorRestClient(RestConfig(base_url="https://vectors.test", api_key="writer"))
with pytest.raises(VectorRestError, match="malformed"):
client.list_evidence_generations("evidence")
client.list_evidence_generations("evidence", "default")
harness/tests/test_corpus_pipeline.py
+19 -9
View File
@@ -1,7 +1,7 @@
import pytest
from tht.corpus.chunk import ChunkPolicy
from tht.corpus.pipeline import CorpusPipeline, PipelineError
from tht.corpus.pipeline import CorpusPipeline, PipelineError, PipelineResult
from tht.corpus.store import CorpusStore
from tht.corpus.models import CorpusManifest
from tht.ports.evidence import AcquiredDocument, SourceObject
@@ -55,17 +55,19 @@ class Vectors:
value.record.id: value.content_hash for value in self.records
}
def delete_generation(self, collection, generation):
def delete_generation(self, collection, generation, workspace_id):
self.records = [
value for value in self.records
if value.record.metadata["vector_generation"] != generation
if not (value.record.metadata["vector_generation"] == generation
and value.record.metadata.get("workspace_id") == workspace_id)
]
return 0
def list_evidence_generations(self, collection):
def list_evidence_generations(self, collection, workspace_id):
return sorted({
value.record.metadata["vector_generation"] for value in self.records
if value.record.kind == "evidence"
and value.record.metadata.get("workspace_id") == workspace_id
})
@@ -128,10 +130,10 @@ def test_retention_keeps_filesystem_when_vector_purge_fails_then_retries(tmp_pat
super().__init__()
self.fail_delete = True
def delete_generation(self, collection, generation):
def delete_generation(self, collection, generation, workspace_id):
if self.fail_delete:
raise RuntimeError("credential secret")
return super().delete_generation(collection, generation)
return super().delete_generation(collection, generation, workspace_id)
vectors = FailingDelete()
for index in range(2):
@@ -156,13 +158,13 @@ def test_gc_reconciles_vector_only_generation(tmp_path):
from tht.vectorstore.records import VectorRecord
vectors.records.append(VectorWriteRecord(
record=VectorRecord(id="orphan", kind="evidence", ref="doc:x", title="", content="x",
metadata={"vector_generation": orphan}),
metadata={"vector_generation": orphan, "workspace_id": "default"}),
embedding=[0.0, 0.0, 0.0], content_hash="sha256:" + "0" * 64,
))
candidate = pipeline(tmp_path, Source([]), vectors=vectors, retain=1)
report = candidate.gc(workspace_root=tmp_path)
assert report["evicted"] == [orphan]
assert vectors.list_evidence_generations("evidence") == []
assert vectors.list_evidence_generations("evidence", "default") == []
assert candidate.gc(workspace_root=tmp_path)["evicted"] == []
@@ -233,7 +235,7 @@ def test_gc_preserves_vector_dependencies_of_retained_manifests(tmp_path):
vectors=vectors, retain=2,
).run().generation
assert CorpusStore(tmp_path / "corpus").list_generations() == [second, third]
assert first in vectors.list_evidence_generations("evidence")
assert first in vectors.list_evidence_generations("evidence", "default")
def test_active_searcher_without_active_fails_closed_for_evidence(tmp_path):
@@ -319,6 +321,14 @@ def test_active_evidence_query_holds_lock_against_publish(tmp_path):
assert published.is_set()
def test_pipeline_result_dump_does_not_deepcopy_frozen_metadata():
manifest = CorpusManifest(metadata={"nested": {"value": ["safe"]}})
payload = PipelineResult(
"succeeded", None, False, (), (), (), manifest,
).model_dump(mode="json")
assert payload["manifest"]["metadata"] == {"nested": {"value": ["safe"]}}
def test_unchanged_documents_skip_acquire_normalize_chunk_and_embed(tmp_path):
one = item("one", "a")
first_source = Source([(one, "hello")])