fix(evidence): isolate vectors by workspace

This commit is contained in:
2026-07-12 05:50:59 +02:00
parent 24f6a5daa6
commit 05accc1443
13 changed files with 163 additions and 66 deletions
@@ -102,3 +102,23 @@ JobRunner tests.
Focused default/mixed/no-ACTIVE/search-pack tests pass, the real Docker pgvector lifecycle passes, Focused default/mixed/no-ACTIVE/search-pack tests pass, the real Docker pgvector lifecycle passes,
and the final full harness plus scoped Ruff/diff invocation completed with exit code 0. and the final full harness plus scoped Ruff/diff invocation completed with exit code 0.
## Workspace-scoped Evidence isolation
- Evidence manifests, vector metadata, and record keys now carry the stable JobRunner workspace id
derived from the configured workspace identity (config stem), never credentials or absolute paths.
- Every ACTIVE server-side predicate includes `workspace_id`. Legacy unscoped rows therefore fail
closed and cannot appear in Evidence results.
- Vector generation inventory and deletion require the workspace namespace across the port, direct
pgvector adapter, HTTP client/adapter, and allowlisted RPC SQL. Legacy unscoped RPC overloads are
explicitly dropped during migration; destructive SQL matches collection, kind, generation, and
workspace together.
- GC recovers the persisted namespace from ACTIVE for explicit/restarted cleanup and can only list
or delete that workspace's generations. Real shared-pgvector coverage proves deleting a generation
for workspace A preserves the same generation in workspace B.
- `PipelineResult.model_dump` now serializes fields explicitly instead of `dataclasses.asdict`,
avoiding deepcopy of immutable `FrozenDict` metadata while preserving pristine JSON CLI output.
Final focused verification: `89 passed` across corpus/CLI JSON, direct/HTTP parity, migrations, and
real Docker pgvector lifecycle; scoped Ruff and `git diff --check` clean. A contemporaneous full-suite
run reached unrelated Task 6 immutable-file tamper tests; those files were deliberately not changed.
+3 -1
View File
@@ -43,7 +43,8 @@ begin
table_name <> 'evidence' table_name <> 'evidence'
or not (metadata_filter ? 'vector_generation') or not (metadata_filter ? 'vector_generation')
or not (metadata_filter ? 'document_ids') or not (metadata_filter ? 'document_ids')
or jsonb_object_length(metadata_filter) <> 2 or not (metadata_filter ? 'workspace_id')
or jsonb_object_length(metadata_filter) <> 3
or jsonb_typeof(metadata_filter->'document_ids') <> 'array' or jsonb_typeof(metadata_filter->'document_ids') <> 'array'
) then ) then
raise exception 'invalid Evidence metadata filter'; raise exception 'invalid Evidence metadata filter';
@@ -56,6 +57,7 @@ begin
where ($3 is null or t.kind = any ($3)) where ($3 is null or t.kind = any ($3))
and ($4 is null or ( and ($4 is null or (
t.metadata->>''vector_generation'' = $4->>''vector_generation'' t.metadata->>''vector_generation'' = $4->>''vector_generation''
and t.metadata->>''workspace_id'' = $4->>''workspace_id''
and t.metadata->>''document_id'' in ( and t.metadata->>''document_id'' in (
select jsonb_array_elements_text($4->''document_ids'') select jsonb_array_elements_text($4->''document_ids'')
) )
+22 -13
View File
@@ -40,14 +40,18 @@ begin
end; end;
$$; $$;
create or replace function public.list_evidence_generations(table_name text, kind text) drop function if exists public.list_evidence_generations(text, text);
create or replace function public.list_evidence_generations(
table_name text, kind text, workspace_id text
)
returns table(generation text) returns table(generation text)
language plpgsql language plpgsql
security definer security definer
set search_path = public, vectors, extensions set search_path = public, vectors, extensions
as $$ as $$
begin begin
if table_name <> 'evidence' or kind <> 'evidence' then if table_name <> 'evidence' or kind <> 'evidence' or workspace_id !~ '^[a-z][a-z0-9_-]{0,63}$' then
raise exception 'only exact Evidence generations may be listed'; raise exception 'only exact Evidence generations may be listed';
end if; end if;
return query return query
@@ -55,6 +59,7 @@ begin
from vectors.evidence e from vectors.evidence e
where e.kind = 'evidence' where e.kind = 'evidence'
and e.metadata->>'vector_generation' ~ '^gen:[0-9a-f]{32}$' and e.metadata->>'vector_generation' ~ '^gen:[0-9a-f]{32}$'
and e.metadata->>'workspace_id' = workspace_id
order by 1; order by 1;
end; end;
$$; $$;
@@ -120,8 +125,10 @@ begin
end; end;
$$; $$;
drop function if exists public.delete_vector_generation(text, text, text);
create or replace function public.delete_vector_generation( create or replace function public.delete_vector_generation(
table_name text, kind text, generation text table_name text, kind text, generation text, workspace_id text
) )
returns jsonb returns jsonb
language plpgsql language plpgsql
@@ -130,11 +137,13 @@ set search_path = public, vectors, extensions
as $$ as $$
declare affected integer; declare affected integer;
begin begin
if table_name <> 'evidence' or kind <> 'evidence' or generation !~ '^gen:[0-9a-f]{32}$' then if table_name <> 'evidence' or kind <> 'evidence' or generation !~ '^gen:[0-9a-f]{32}$'
or workspace_id !~ '^[a-z][a-z0-9_-]{0,63}$' then
raise exception 'only an exact Evidence generation may be deleted'; raise exception 'only an exact Evidence generation may be deleted';
end if; end if;
delete from vectors.evidence e delete from vectors.evidence e
where e.kind = 'evidence' and e.metadata->>'vector_generation' = generation; where e.kind = 'evidence' and e.metadata->>'vector_generation' = generation
and e.metadata->>'workspace_id' = workspace_id;
get diagnostics affected = row_count; get diagnostics affected = row_count;
return jsonb_build_object('deleted', affected); return jsonb_build_object('deleted', affected);
end; end;
@@ -143,8 +152,8 @@ $$;
revoke all on function public._assert_vector_write_table(text, text[]) from public; revoke all on function public._assert_vector_write_table(text, text[]) from public;
revoke all on function public.existing_vector_hashes(text, text[]) from public; revoke all on function public.existing_vector_hashes(text, text[]) from public;
revoke all on function public.upsert_vector_records(text, jsonb) from public; revoke all on function public.upsert_vector_records(text, jsonb) from public;
revoke all on function public.delete_vector_generation(text, text, text) from public; revoke all on function public.delete_vector_generation(text, text, text, text) from public;
revoke all on function public.list_evidence_generations(text, text) from public; revoke all on function public.list_evidence_generations(text, text, text) from public;
-- Su alcuni progetti Supabase le funzioni in `public` ricevono grant automatici: revoca -- Su alcuni progetti Supabase le funzioni in `public` ricevono grant automatici: revoca
-- esplicitamente dai ruoli client generici, poi abilita solo il writer dedicato. -- esplicitamente dai ruoli client generici, poi abilita solo il writer dedicato.
@@ -153,20 +162,20 @@ begin
if exists (select 1 from pg_roles where rolname = 'anon') then if exists (select 1 from pg_roles where rolname = 'anon') then
revoke all on function public.existing_vector_hashes(text, text[]) from anon; revoke all on function public.existing_vector_hashes(text, text[]) from anon;
revoke all on function public.upsert_vector_records(text, jsonb) from anon; revoke all on function public.upsert_vector_records(text, jsonb) from anon;
revoke all on function public.delete_vector_generation(text, text, text) from anon; revoke all on function public.delete_vector_generation(text, text, text, text) from anon;
revoke all on function public.list_evidence_generations(text, text) from anon; revoke all on function public.list_evidence_generations(text, text, text) from anon;
end if; end if;
if exists (select 1 from pg_roles where rolname = 'authenticated') then if exists (select 1 from pg_roles where rolname = 'authenticated') then
revoke all on function public.existing_vector_hashes(text, text[]) from authenticated; revoke all on function public.existing_vector_hashes(text, text[]) from authenticated;
revoke all on function public.upsert_vector_records(text, jsonb) from authenticated; revoke all on function public.upsert_vector_records(text, jsonb) from authenticated;
revoke all on function public.delete_vector_generation(text, text, text) from authenticated; revoke all on function public.delete_vector_generation(text, text, text, text) from authenticated;
revoke all on function public.list_evidence_generations(text, text) from authenticated; revoke all on function public.list_evidence_generations(text, text, text) from authenticated;
end if; end if;
if exists (select 1 from pg_roles where rolname = 'vector_writer') then if exists (select 1 from pg_roles where rolname = 'vector_writer') then
grant execute on function public.existing_vector_hashes(text, text[]) to vector_writer; grant execute on function public.existing_vector_hashes(text, text[]) to vector_writer;
grant execute on function public.upsert_vector_records(text, jsonb) to vector_writer; grant execute on function public.upsert_vector_records(text, jsonb) to vector_writer;
grant execute on function public.delete_vector_generation(text, text, text) to vector_writer; grant execute on function public.delete_vector_generation(text, text, text, text) to vector_writer;
grant execute on function public.list_evidence_generations(text, text) to vector_writer; grant execute on function public.list_evidence_generations(text, text, text) to vector_writer;
end if; end if;
end $$; end $$;
@@ -241,7 +241,8 @@ def test_real_pgvector_corpus_job_lifecycle(tmp_path, persistent_pgvector):
record=VectorRecord( record=VectorRecord(
id="chunk:exact-vector-orphan", kind="evidence", ref="doc:orphan", id="chunk:exact-vector-orphan", kind="evidence", ref="doc:orphan",
title="orphan", content="orphan", title="orphan", content="orphan",
metadata={"document_id": "doc:orphan", "vector_generation": orphan}, metadata={"document_id": "doc:orphan", "vector_generation": orphan,
"workspace_id": "pgvector-lifecycle"},
), ),
embedding=query, content_hash="sha256:" + "f" * 64, embedding=query, content_hash="sha256:" + "f" * 64,
)]) )])
@@ -251,5 +252,7 @@ def test_real_pgvector_corpus_job_lifecycle(tmp_path, persistent_pgvector):
expected_fs = set(generations[-2:]) expected_fs = set(generations[-2:])
assert set(CorpusStore(tmp_path / "corpus").list_generations()) == expected_fs assert set(CorpusStore(tmp_path / "corpus").list_generations()) == expected_fs
expected_vectors = expected_fs | {generations[0]} expected_vectors = expected_fs | {generations[0]}
assert set(recreated.list_evidence_generations("evidence")) == expected_vectors assert set(recreated.list_evidence_generations(
"evidence", "pgvector-lifecycle"
)) == expected_vectors
assert final_pipeline.gc(workspace_root=tmp_path)["evicted"] == [] assert final_pipeline.gc(workspace_root=tmp_path)["evicted"] == []
+19 -4
View File
@@ -130,14 +130,29 @@ def test_pgvector_lists_and_deletes_exact_evidence_generation(store):
value = VectorWriteRecord( value = VectorWriteRecord(
record=VectorRecord( record=VectorRecord(
id="evidence-generation-a", kind="evidence", ref="doc:a", title="a", id="evidence-generation-a", kind="evidence", ref="doc:a", title="a",
content="content", metadata={"vector_generation": generation}, content="content", metadata={"vector_generation": generation, "workspace_id": "default"},
), ),
embedding=[1.0, 0.0], content_hash="sha256:" + "a" * 64, embedding=[1.0, 0.0], content_hash="sha256:" + "a" * 64,
) )
store.upsert("evidence", [value]) store.upsert("evidence", [value])
assert generation in store.list_evidence_generations("evidence") assert generation in store.list_evidence_generations("evidence", "default")
assert store.delete_generation("evidence", generation) == 1 assert store.delete_generation("evidence", generation, "default") == 1
assert generation not in store.list_evidence_generations("evidence") assert generation not in store.list_evidence_generations("evidence", "default")
def test_pgvector_generation_cleanup_isolated_between_workspaces(store):
generation = "gen:" + "b" * 32
records = [VectorWriteRecord(
record=VectorRecord(
id=f"evidence-{workspace}", kind="evidence", ref=f"doc:{workspace}",
title=workspace, content=workspace,
metadata={"vector_generation": generation, "workspace_id": workspace},
), embedding=[1.0, 0.0], content_hash="sha256:" + key * 64,
) for workspace, key in (("workspace-a", "b"), ("workspace-b", "c"))]
store.upsert("evidence", records)
assert store.delete_generation("evidence", generation, "workspace-a") == 1
assert generation not in store.list_evidence_generations("evidence", "workspace-a")
assert generation in store.list_evidence_generations("evidence", "workspace-b")
def test_pgvector_search_filters_kinds_before_limit(store): def test_pgvector_search_filters_kinds_before_limit(store):
@@ -235,9 +235,10 @@ def test_http_delete_generation_uses_exact_allowlisted_rpc_payload(monkeypatch):
lambda url, json, **kwargs: calls.append((url, json)) or Response({"deleted": 2}), lambda url, json, **kwargs: calls.append((url, json)) or Response({"deleted": 2}),
) )
client = VectorRestClient(RestConfig(base_url="https://vectors.test", api_key="writer")) client = VectorRestClient(RestConfig(base_url="https://vectors.test", api_key="writer"))
assert client.delete_generation("evidence", "gen:" + "a" * 32) == 2 assert client.delete_generation("evidence", "gen:" + "a" * 32, "default") == 2
assert calls == [("https://vectors.test/rpc/delete_vector_generation", { assert calls == [("https://vectors.test/rpc/delete_vector_generation", {
"table_name": "evidence", "kind": "evidence", "generation": "gen:" + "a" * 32, "table_name": "evidence", "kind": "evidence", "generation": "gen:" + "a" * 32,
"workspace_id": "default",
})] })]
@@ -248,7 +249,7 @@ def test_http_delete_generation_legacy_404_fails_closed_without_body_leak(monkey
) )
client = VectorRestClient(RestConfig(base_url="https://vectors.test", api_key="writer")) client = VectorRestClient(RestConfig(base_url="https://vectors.test", api_key="writer"))
with pytest.raises(VectorRestError, match="delete_vector_generation RPC is unavailable") as error: with pytest.raises(VectorRestError, match="delete_vector_generation RPC is unavailable") as error:
client.delete_generation("evidence", "gen:" + "a" * 32) client.delete_generation("evidence", "gen:" + "a" * 32, "default")
assert "secret" not in str(error.value) assert "secret" not in str(error.value)
@@ -261,9 +262,9 @@ def test_http_list_evidence_generations_exact_rpc_and_legacy_fail_closed(monkeyp
]), ]),
) )
client = VectorRestClient(RestConfig(base_url="https://vectors.test", api_key="writer")) client = VectorRestClient(RestConfig(base_url="https://vectors.test", api_key="writer"))
assert client.list_evidence_generations("evidence") == ["gen:" + "a" * 32] assert client.list_evidence_generations("evidence", "default") == ["gen:" + "a" * 32]
assert calls[0][0].endswith("/rpc/list_evidence_generations") assert calls[0][0].endswith("/rpc/list_evidence_generations")
assert calls[0][1] == {"table_name": "evidence", "kind": "evidence"} assert calls[0][1] == {"table_name": "evidence", "kind": "evidence", "workspace_id": "default"}
@pytest.mark.parametrize("generation", ["gen:a", "gen:" + "A" * 32, "gen:" + "a" * 33]) @pytest.mark.parametrize("generation", ["gen:a", "gen:" + "A" * 32, "gen:" + "a" * 33])
@@ -274,7 +275,7 @@ def test_http_generation_operations_reject_noncanonical_values(monkeypatch, gene
) )
client = VectorRestClient(RestConfig(base_url="https://vectors.test", api_key="writer")) client = VectorRestClient(RestConfig(base_url="https://vectors.test", api_key="writer"))
with pytest.raises(ValueError, match="canonical"): with pytest.raises(ValueError, match="canonical"):
client.delete_generation("evidence", generation) client.delete_generation("evidence", generation, "default")
def test_http_inventory_rejects_malformed_rpc_output(monkeypatch): def test_http_inventory_rejects_malformed_rpc_output(monkeypatch):
@@ -284,4 +285,4 @@ def test_http_inventory_rejects_malformed_rpc_output(monkeypatch):
) )
client = VectorRestClient(RestConfig(base_url="https://vectors.test", api_key="writer")) client = VectorRestClient(RestConfig(base_url="https://vectors.test", api_key="writer"))
with pytest.raises(VectorRestError, match="malformed"): with pytest.raises(VectorRestError, match="malformed"):
client.list_evidence_generations("evidence") client.list_evidence_generations("evidence", "default")
+19 -9
View File
@@ -1,7 +1,7 @@
import pytest import pytest
from tht.corpus.chunk import ChunkPolicy from tht.corpus.chunk import ChunkPolicy
from tht.corpus.pipeline import CorpusPipeline, PipelineError from tht.corpus.pipeline import CorpusPipeline, PipelineError, PipelineResult
from tht.corpus.store import CorpusStore from tht.corpus.store import CorpusStore
from tht.corpus.models import CorpusManifest from tht.corpus.models import CorpusManifest
from tht.ports.evidence import AcquiredDocument, SourceObject from tht.ports.evidence import AcquiredDocument, SourceObject
@@ -55,17 +55,19 @@ class Vectors:
value.record.id: value.content_hash for value in self.records value.record.id: value.content_hash for value in self.records
} }
def delete_generation(self, collection, generation): def delete_generation(self, collection, generation, workspace_id):
self.records = [ self.records = [
value for value in self.records value for value in self.records
if value.record.metadata["vector_generation"] != generation if not (value.record.metadata["vector_generation"] == generation
and value.record.metadata.get("workspace_id") == workspace_id)
] ]
return 0 return 0
def list_evidence_generations(self, collection): def list_evidence_generations(self, collection, workspace_id):
return sorted({ return sorted({
value.record.metadata["vector_generation"] for value in self.records value.record.metadata["vector_generation"] for value in self.records
if value.record.kind == "evidence" if value.record.kind == "evidence"
and value.record.metadata.get("workspace_id") == workspace_id
}) })
@@ -128,10 +130,10 @@ def test_retention_keeps_filesystem_when_vector_purge_fails_then_retries(tmp_pat
super().__init__() super().__init__()
self.fail_delete = True self.fail_delete = True
def delete_generation(self, collection, generation): def delete_generation(self, collection, generation, workspace_id):
if self.fail_delete: if self.fail_delete:
raise RuntimeError("credential secret") raise RuntimeError("credential secret")
return super().delete_generation(collection, generation) return super().delete_generation(collection, generation, workspace_id)
vectors = FailingDelete() vectors = FailingDelete()
for index in range(2): for index in range(2):
@@ -156,13 +158,13 @@ def test_gc_reconciles_vector_only_generation(tmp_path):
from tht.vectorstore.records import VectorRecord from tht.vectorstore.records import VectorRecord
vectors.records.append(VectorWriteRecord( vectors.records.append(VectorWriteRecord(
record=VectorRecord(id="orphan", kind="evidence", ref="doc:x", title="", content="x", record=VectorRecord(id="orphan", kind="evidence", ref="doc:x", title="", content="x",
metadata={"vector_generation": orphan}), metadata={"vector_generation": orphan, "workspace_id": "default"}),
embedding=[0.0, 0.0, 0.0], content_hash="sha256:" + "0" * 64, embedding=[0.0, 0.0, 0.0], content_hash="sha256:" + "0" * 64,
)) ))
candidate = pipeline(tmp_path, Source([]), vectors=vectors, retain=1) candidate = pipeline(tmp_path, Source([]), vectors=vectors, retain=1)
report = candidate.gc(workspace_root=tmp_path) report = candidate.gc(workspace_root=tmp_path)
assert report["evicted"] == [orphan] assert report["evicted"] == [orphan]
assert vectors.list_evidence_generations("evidence") == [] assert vectors.list_evidence_generations("evidence", "default") == []
assert candidate.gc(workspace_root=tmp_path)["evicted"] == [] assert candidate.gc(workspace_root=tmp_path)["evicted"] == []
@@ -233,7 +235,7 @@ def test_gc_preserves_vector_dependencies_of_retained_manifests(tmp_path):
vectors=vectors, retain=2, vectors=vectors, retain=2,
).run().generation ).run().generation
assert CorpusStore(tmp_path / "corpus").list_generations() == [second, third] assert CorpusStore(tmp_path / "corpus").list_generations() == [second, third]
assert first in vectors.list_evidence_generations("evidence") assert first in vectors.list_evidence_generations("evidence", "default")
def test_active_searcher_without_active_fails_closed_for_evidence(tmp_path): def test_active_searcher_without_active_fails_closed_for_evidence(tmp_path):
@@ -319,6 +321,14 @@ def test_active_evidence_query_holds_lock_against_publish(tmp_path):
assert published.is_set() assert published.is_set()
def test_pipeline_result_dump_does_not_deepcopy_frozen_metadata():
manifest = CorpusManifest(metadata={"nested": {"value": ["safe"]}})
payload = PipelineResult(
"succeeded", None, False, (), (), (), manifest,
).model_dump(mode="json")
assert payload["manifest"]["metadata"] == {"nested": {"value": ["safe"]}}
def test_unchanged_documents_skip_acquire_normalize_chunk_and_embed(tmp_path): def test_unchanged_documents_skip_acquire_normalize_chunk_and_embed(tmp_path):
one = item("one", "a") one = item("one", "a")
first_source = Source([(one, "hello")]) first_source = Source([(one, "hello")])
+17 -9
View File
@@ -273,16 +273,18 @@ class PgVectorStore:
filter_params.append(collection_kinds) filter_params.append(collection_kinds)
if metadata_filter is not None: if metadata_filter is not None:
if collection != "evidence" or set(metadata_filter) != { if collection != "evidence" or set(metadata_filter) != {
"vector_generation", "document_ids" "vector_generation", "document_ids", "workspace_id"
}: }:
raise VectorStoreError("Unsupported vector metadata filter") raise VectorStoreError("Unsupported vector metadata filter")
generation = metadata_filter["vector_generation"] generation = metadata_filter["vector_generation"]
document_ids = metadata_filter["document_ids"] document_ids = metadata_filter["document_ids"]
if not isinstance(generation, str) or not isinstance(document_ids, list): workspace_id = metadata_filter["workspace_id"]
if not isinstance(generation, str) or not isinstance(document_ids, list) or not isinstance(workspace_id, str):
raise VectorStoreError("Invalid vector metadata filter") raise VectorStoreError("Invalid vector metadata filter")
clauses.append(sql.SQL("metadata->>'vector_generation' = %s")) clauses.append(sql.SQL("metadata->>'vector_generation' = %s"))
clauses.append(sql.SQL("metadata->>'document_id' = ANY(%s)")) clauses.append(sql.SQL("metadata->>'document_id' = ANY(%s)"))
filter_params.extend((generation, document_ids)) clauses.append(sql.SQL("metadata->>'workspace_id' = %s"))
filter_params.extend((generation, document_ids, workspace_id))
where = ( where = (
sql.SQL(" WHERE ") + sql.SQL(" AND ").join(clauses) sql.SQL(" WHERE ") + sql.SQL(" AND ").join(clauses)
if clauses else sql.SQL("") if clauses else sql.SQL("")
@@ -404,9 +406,11 @@ class PgVectorStore:
raw.close() raw.close()
return len(records) return len(records)
def delete_generation(self, collection: str, generation: str) -> int: def delete_generation(self, collection: str, generation: str, workspace_id: str) -> int:
if collection != "evidence" or re.fullmatch(r"gen:[0-9a-f]{32}", generation) is None: if collection != "evidence" or re.fullmatch(r"gen:[0-9a-f]{32}", generation) is None:
raise VectorStoreError("Only exact Evidence generations may be deleted") raise VectorStoreError("Only exact Evidence generations may be deleted")
if re.fullmatch(r"[a-z][a-z0-9_-]{0,63}", workspace_id) is None:
raise VectorStoreError("Invalid Evidence workspace namespace")
raw = None raw = None
try: try:
raw = self._require_writer().raw_connection() raw = self._require_writer().raw_connection()
@@ -414,9 +418,10 @@ class PgVectorStore:
cursor.execute( cursor.execute(
sql.SQL( sql.SQL(
"DELETE FROM {} WHERE kind = 'evidence' " "DELETE FROM {} WHERE kind = 'evidence' "
"AND metadata->>'vector_generation' = %s" "AND metadata->>'vector_generation' = %s "
"AND metadata->>'workspace_id' = %s"
).format(_collection(self._schema, collection)), ).format(_collection(self._schema, collection)),
(generation,), (generation, workspace_id),
) )
count = cursor.rowcount count = cursor.rowcount
raw.commit() raw.commit()
@@ -429,9 +434,11 @@ class PgVectorStore:
if raw is not None: if raw is not None:
raw.close() raw.close()
def list_evidence_generations(self, collection: str) -> list[str]: def list_evidence_generations(self, collection: str, workspace_id: str) -> list[str]:
if collection != "evidence": if collection != "evidence":
raise VectorStoreError("Only exact Evidence generations may be listed") raise VectorStoreError("Only exact Evidence generations may be listed")
if re.fullmatch(r"[a-z][a-z0-9_-]{0,63}", workspace_id) is None:
raise VectorStoreError("Invalid Evidence workspace namespace")
raw = None raw = None
try: try:
raw = self._require_writer().raw_connection() raw = self._require_writer().raw_connection()
@@ -440,8 +447,9 @@ class PgVectorStore:
sql.SQL( sql.SQL(
"SELECT DISTINCT metadata->>'vector_generation' FROM {} " "SELECT DISTINCT metadata->>'vector_generation' FROM {} "
"WHERE kind = 'evidence' AND metadata->>'vector_generation' " "WHERE kind = 'evidence' AND metadata->>'vector_generation' "
"~ '^gen:[0-9a-f]{{32}}$' ORDER BY 1" "~ '^gen:[0-9a-f]{{32}}$' AND metadata->>'workspace_id' = %s ORDER BY 1"
).format(_collection(self._schema, collection)) ).format(_collection(self._schema, collection)),
(workspace_id,),
) )
return [row[0] for row in cursor.fetchall()] return [row[0] for row in cursor.fetchall()]
except Exception as exc: except Exception as exc:
+4 -4
View File
@@ -155,19 +155,19 @@ class ThothHttpVectorStore:
except VectorRestError as exc: except VectorRestError as exc:
raise VectorStoreError(str(exc)) from exc raise VectorStoreError(str(exc)) from exc
def delete_generation(self, collection: str, generation: str) -> int: def delete_generation(self, collection: str, generation: str, workspace_id: str) -> int:
if collection != "evidence" or re.fullmatch(r"gen:[0-9a-f]{32}", generation) is None: if collection != "evidence" or re.fullmatch(r"gen:[0-9a-f]{32}", generation) is None:
raise VectorStoreError("Only exact Evidence generations may be deleted") raise VectorStoreError("Only exact Evidence generations may be deleted")
try: try:
return self._require_writer().delete_generation(collection, generation) return self._require_writer().delete_generation(collection, generation, workspace_id)
except VectorRestError as exc: except VectorRestError as exc:
raise VectorStoreError(str(exc)) from exc raise VectorStoreError(str(exc)) from exc
def list_evidence_generations(self, collection: str) -> list[str]: def list_evidence_generations(self, collection: str, workspace_id: str) -> list[str]:
if collection != "evidence": if collection != "evidence":
raise VectorStoreError("Only exact Evidence generations may be listed") raise VectorStoreError("Only exact Evidence generations may be listed")
try: try:
return self._require_writer().list_evidence_generations(collection) return self._require_writer().list_evidence_generations(collection, workspace_id)
except VectorRestError as exc: except VectorRestError as exc:
raise VectorWriteUnavailable("Vector generation inventory unavailable") from exc raise VectorWriteUnavailable("Vector generation inventory unavailable") from exc
+33 -11
View File
@@ -47,9 +47,17 @@ class PipelineResult:
resumed_from: str | None = None resumed_from: str | None = None
def model_dump(self, mode=None): def model_dump(self, mode=None):
value = asdict(self) return {
value["manifest"] = self.manifest.model_dump(mode="json") "status": self.status,
return value "generation": self.generation,
"published": self.published,
"changed": list(self.changed),
"unchanged": list(self.unchanged),
"removed": list(self.removed),
"manifest": self.manifest.model_dump(mode="json"),
"run_id": self.run_id,
"resumed_from": self.resumed_from,
}
def _fingerprint(value) -> str: def _fingerprint(value) -> str:
@@ -62,6 +70,7 @@ class CorpusPipeline:
self, *, store: CorpusStore, sources: list[EvidenceSource], embedder, self, *, store: CorpusStore, sources: list[EvidenceSource], embedder,
vector_store: VectorStore, embedding_model: str, embedding_dimensions: int, vector_store: VectorStore, embedding_model: str, embedding_dimensions: int,
chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3, chunk_policy: ChunkPolicy, pipeline_version: str, retain_published_generations: int = 3,
workspace_id: str = "default",
) -> None: ) -> None:
self.store = store self.store = store
self.sources = sources self.sources = sources
@@ -74,6 +83,7 @@ class CorpusPipeline:
if isinstance(retain_published_generations, bool) or retain_published_generations < 1: if isinstance(retain_published_generations, bool) or retain_published_generations < 1:
raise ValueError("retain_published_generations must be at least 1") raise ValueError("retain_published_generations must be at least 1")
self.retain_published_generations = retain_published_generations self.retain_published_generations = retain_published_generations
self.workspace_id = workspace_id
def _protected_generations(self, workspace_root: Path) -> set[str]: def _protected_generations(self, workspace_root: Path) -> set[str]:
protected = {value for value in (self.store.active_generation(),) if value} protected = {value for value in (self.store.active_generation(),) if value}
@@ -92,9 +102,14 @@ class CorpusPipeline:
return protected return protected
def gc(self, *, workspace_root: Path, dry_run: bool = False) -> dict: def gc(self, *, workspace_root: Path, dry_run: bool = False) -> dict:
active_manifest = self.store.active_manifest()
if active_manifest is not None:
persisted_workspace = active_manifest.metadata.get("workspace_id")
if isinstance(persisted_workspace, str):
self.workspace_id = persisted_workspace
published = self.store.published_generations() published = self.store.published_generations()
list_vectors = getattr(self.vector_store, "list_evidence_generations", None) list_vectors = getattr(self.vector_store, "list_evidence_generations", None)
vector_generations = set(list_vectors("evidence")) if list_vectors else set() vector_generations = set(list_vectors("evidence", self.workspace_id)) if list_vectors else set()
generations = sorted(set(published) | vector_generations) generations = sorted(set(published) | vector_generations)
job_protected = self._protected_generations(workspace_root) job_protected = self._protected_generations(workspace_root)
active = self.store.active_generation() active = self.store.active_generation()
@@ -124,7 +139,7 @@ class CorpusPipeline:
continue continue
if purge_vector: if purge_vector:
try: try:
self.vector_store.delete_generation("evidence", generation) self.vector_store.delete_generation("evidence", generation, self.workspace_id)
except Exception: except Exception:
failures.append({"generation": generation, "error": "vector cleanup failed"}) failures.append({"generation": generation, "error": "vector cleanup failed"})
continue continue
@@ -154,6 +169,7 @@ class CorpusPipeline:
return self._run(dry_run=dry_run, resume=resume) return self._run(dry_run=dry_run, resume=resume)
def run_as_job(self, **kwargs) -> PipelineResult: def run_as_job(self, **kwargs) -> PipelineResult:
self.workspace_id = kwargs["workspace_id"]
with self.store.writer_lock(): with self.store.writer_lock():
return self._run_as_job(**kwargs) return self._run_as_job(**kwargs)
@@ -259,6 +275,7 @@ class CorpusPipeline:
vector_generation=plan["generation"], vector_generation=plan["generation"],
documents=tuple(documents), chunks=tuple(chunks), documents=tuple(documents), chunks=tuple(chunks),
metadata={ metadata={
"workspace_id": self.workspace_id,
"compatibility_fingerprint": compatibility, "compatibility_fingerprint": compatibility,
"fingerprints": plan["fingerprints"], "fingerprints": plan["fingerprints"],
"removed": plan["removed"], "removed": plan["removed"],
@@ -289,7 +306,7 @@ class CorpusPipeline:
changed_docs = {doc.document_id for doc in manifest.documents if doc.source_id in plan["changed"]} changed_docs = {doc.document_id for doc in manifest.documents if doc.source_id in plan["changed"]}
parts = [part for part in manifest.chunks if part.document_id in changed_docs] parts = [part for part in manifest.chunks if part.document_id in changed_docs]
embeddings = read(context, "embeddings.json") embeddings = read(context, "embeddings.json")
return [self._vector_record(part, vector, plan["generation"]) return [self._vector_record(part, vector, plan["generation"], self.workspace_id)
for part, vector in zip(parts, embeddings, strict=True)] for part, vector in zip(parts, embeddings, strict=True)]
def compensate(context: JobContext) -> None: def compensate(context: JobContext) -> None:
@@ -297,7 +314,7 @@ class CorpusPipeline:
if self.store.active_generation() != generation: if self.store.active_generation() != generation:
self.store.discard(generation) self.store.discard(generation)
try: try:
self.vector_store.delete_generation("evidence", generation) self.vector_store.delete_generation("evidence", generation, self.workspace_id)
except Exception: except Exception:
pass pass
write(context, "compensated.json", {"generation": generation}) write(context, "compensated.json", {"generation": generation})
@@ -495,6 +512,7 @@ class CorpusPipeline:
vector_generation=generation, vector_generation=generation,
documents=tuple(documents), chunks=tuple(chunks), documents=tuple(documents), chunks=tuple(chunks),
metadata={ metadata={
"workspace_id": self.workspace_id,
"compatibility_fingerprint": compatibility, "compatibility_fingerprint": compatibility,
"fingerprints": fingerprints, "fingerprints": fingerprints,
"removed": list(removed), "removed": list(removed),
@@ -508,7 +526,7 @@ class CorpusPipeline:
raise PipelineError("embedding count mismatch") raise PipelineError("embedding count mismatch")
if any(len(vector) != self.embedding_dimensions for vector in embeddings): if any(len(vector) != self.embedding_dimensions for vector in embeddings):
raise PipelineError("embedding dimension mismatch") raise PipelineError("embedding dimension mismatch")
records = [self._vector_record(part, vector, generation) for part, vector in zip(changed_chunks, embeddings, strict=True)] records = [self._vector_record(part, vector, generation, self.workspace_id) for part, vector in zip(changed_chunks, embeddings, strict=True)]
if records: if records:
written = self.vector_store.upsert("evidence", records) written = self.vector_store.upsert("evidence", records)
vector_written = True vector_written = True
@@ -547,17 +565,21 @@ class CorpusPipeline:
pass pass
if vector_written: if vector_written:
try: try:
self.vector_store.delete_generation("evidence", generation) self.vector_store.delete_generation("evidence", generation, self.workspace_id)
except Exception: except Exception:
pass pass
@staticmethod @staticmethod
def _vector_record(chunk: CanonicalChunk, embedding: list[float], generation: str): def _vector_record(
chunk: CanonicalChunk, embedding: list[float], generation: str, workspace_id: str,
):
record = VectorRecord( record = VectorRecord(
id=f"{generation}:{chunk.chunk_id}", kind="evidence", ref=chunk.document_id, id=f"{workspace_id}:{generation}:{chunk.chunk_id}",
kind="evidence", ref=chunk.document_id,
title=str(chunk.metadata.get("title", "")), content=chunk.content, title=str(chunk.metadata.get("title", "")), content=chunk.content,
metadata={ metadata={
**dict(chunk.metadata), "document_id": chunk.document_id, **dict(chunk.metadata), "document_id": chunk.document_id,
"workspace_id": workspace_id,
"source_uri": chunk.source_uri, "ordinal": chunk.ordinal, "source_uri": chunk.source_uri, "ordinal": chunk.ordinal,
"vector_generation": generation, "vector_generation": generation,
}, },
+2 -2
View File
@@ -80,9 +80,9 @@ class VectorStore(Protocol):
def upsert(self, collection: str, records: list[VectorWriteRecord]) -> int: ... def upsert(self, collection: str, records: list[VectorWriteRecord]) -> int: ...
def delete_generation(self, collection: str, generation: str) -> int: ... def delete_generation(self, collection: str, generation: str, workspace_id: str) -> int: ...
def list_evidence_generations(self, collection: str) -> list[str]: ... def list_evidence_generations(self, collection: str, workspace_id: str) -> list[str]: ...
__all__ = [ __all__ = [
+3 -1
View File
@@ -30,7 +30,8 @@ class ActiveEvidenceSearcher:
hits.extend(self.delegate.search(embedding, **kwargs)) hits.extend(self.delegate.search(embedding, **kwargs))
if include_evidence: if include_evidence:
manifest = self.corpus.active_manifest() manifest = self.corpus.active_manifest()
if manifest is not None: workspace_id = manifest.metadata.get("workspace_id") if manifest else None
if manifest is not None and isinstance(workspace_id, str):
by_generation: dict[str, list[str]] = {} by_generation: dict[str, list[str]] = {}
mapping = dict(manifest.metadata.get("document_generations", {})) mapping = dict(manifest.metadata.get("document_generations", {}))
for document in manifest.documents: for document in manifest.documents:
@@ -43,6 +44,7 @@ class ActiveEvidenceSearcher:
metadata_filter={ metadata_filter={
"vector_generation": generation, "vector_generation": generation,
"document_ids": sorted(document_ids), "document_ids": sorted(document_ids),
"workspace_id": workspace_id,
}, },
)) ))
return sorted(hits, key=lambda hit: (-hit.similarity, hit.id))[:top_n] return sorted(hits, key=lambda hit: (-hit.similarity, hit.id))[:top_n]
+9 -4
View File
@@ -128,13 +128,16 @@ class VectorRestClient:
return len(payload) return len(payload)
return len(rows) return len(rows)
def delete_generation(self, table_name: str, generation: str) -> int: def delete_generation(self, table_name: str, generation: str, workspace_id: str) -> int:
if table_name != "evidence" or re.fullmatch(r"gen:[0-9a-f]{32}", generation) is None: if table_name != "evidence" or re.fullmatch(r"gen:[0-9a-f]{32}", generation) is None:
raise ValueError("generation must be canonical") raise ValueError("generation must be canonical")
if re.fullmatch(r"[a-z][a-z0-9_-]{0,63}", workspace_id) is None:
raise ValueError("workspace namespace must be canonical")
try: try:
payload = self._call( payload = self._call(
"delete_vector_generation", "delete_vector_generation",
{"table_name": table_name, "kind": "evidence", "generation": generation}, {"table_name": table_name, "kind": "evidence", "generation": generation,
"workspace_id": workspace_id},
) )
except VectorRestError as error: except VectorRestError as error:
if "HTTP 404" in str(error): if "HTTP 404" in str(error):
@@ -146,11 +149,13 @@ class VectorRestClient:
return int(payload.get("deleted", 0)) return int(payload.get("deleted", 0))
return 0 return 0
def list_evidence_generations(self, table_name: str) -> list[str]: def list_evidence_generations(self, table_name: str, workspace_id: str) -> list[str]:
if re.fullmatch(r"[a-z][a-z0-9_-]{0,63}", workspace_id) is None:
raise ValueError("workspace namespace must be canonical")
try: try:
rows = self._call( rows = self._call(
"list_evidence_generations", "list_evidence_generations",
{"table_name": table_name, "kind": "evidence"}, {"table_name": table_name, "kind": "evidence", "workspace_id": workspace_id},
) or [] ) or []
except VectorRestError as error: except VectorRestError as error:
if "HTTP 404" in str(error): if "HTTP 404" in str(error):