mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(evidence): enforce active vector generations

Browse Source
This commit is contained in:
marcopan
2026-07-12 04:33:07 +02:00
parent b96d4f13b9
commit 981b149249
15 changed files with 261 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.superpowers/sdd/evidence-task-5-report.md
+21
View File
@@ -59,3 +59,24 @@ Live pgvector integration needs Docker or an authorized local pgvector endpoint.
strategy is logical isolation rather than destructive cleanup because the shared `VectorStore`
port intentionally exposes no delete/transaction API; unreachable failed generations can be
garbage-collected by a future maintenance job.
## Review integration wave
Added an enforceable `metadata_filter` vector-port contract and capability flags. Direct pgvector
places exact Evidence generation/document predicates in SQL before `LIMIT`; HTTP sends the same
filter to the RPC and deliberately does not use the legacy 404 fallback. The reader RPC script now
validates and applies that filter. Normal Evidence search and search-pack use an ACTIVE-aware
searcher that groups active documents by generation, executes complete server-filtered searches,
and merges the results.
Added exact-generation Evidence cleanup to direct and HTTP writers plus the allowlisted writer RPC.
Pipeline failures compensate both staged filesystem state and vector writes; cleanup failures stay
sanitized and ACTIVE filtering remains the exposure boundary. Corpus-present session artifact
resolution now fails closed on corrupt/missing ACTIVE rather than falling through to source files.
Focused review-wave verification: 45 passed, scoped Ruff clean. A mocked REST regression proves
the exact filter payload and fail-closed legacy 404 behavior.
Still outstanding from the expanded review request: Task-4 JobRunner stage-by-stage integration,
published-generation retention/garbage collection, same-fd `dirfd` materialized-file reads, and
live local pgvector integration could not be completed in this wave.
harness/scripts/create_vector_reader_rpc.sql
+25 -6
View File
@@ -28,7 +28,9 @@ $$;
create or replace function public.search_similar(
table_name text,
query_embedding vector,
limit_count integer
limit_count integer,
kinds text[] default null,
metadata_filter jsonb default null
)
returns table(id bigint, similarity real, metadata jsonb)
language plpgsql
@@ -37,15 +39,31 @@ set search_path = public, vectors, extensions
as $$
begin
perform public._assert_vector_read_table(table_name);
if metadata_filter is not null and (
table_name <> 'evidence'
or not (metadata_filter ? 'vector_generation')
or not (metadata_filter ? 'document_ids')
or jsonb_object_length(metadata_filter) <> 2
or jsonb_typeof(metadata_filter->'document_ids') <> 'array'
) then
raise exception 'invalid Evidence metadata filter';
end if;
return query execute format(
'select t.id,
(1 - (t.embedding <=> $1))::real as similarity,
t.metadata
from vectors.%I t
where ($3 is null or t.kind = any ($3))
and ($4 is null or (
t.metadata->>''vector_generation'' = $4->>''vector_generation''
and t.metadata->>''document_id'' in (
select jsonb_array_elements_text($4->''document_ids'')
)
))
order by t.embedding <=> $1
limit $2',
table_name
) using query_embedding, limit_count;
) using query_embedding, limit_count, kinds, metadata_filter;
end;
$$;
@@ -75,7 +93,7 @@ end;
$$;
revoke all on function public._assert_vector_read_table(text) from public;
revoke all on function public.search_similar(text, vector, integer) from public;
revoke all on function public.search_similar(text, vector, integer, text[], jsonb) from public;
revoke all on function public.list_tables() from public;
-- Revoca dai ruoli client generici, poi abilita solo il reader dedicato
@@ -83,15 +101,16 @@ revoke all on function public.list_tables() from public;
do $$
begin
if exists (select 1 from pg_roles where rolname = 'anon') then
revoke all on function public.search_similar(text, vector, integer) from anon;
revoke all on function public.search_similar(text, vector, integer, text[], jsonb) from anon;
revoke all on function public.list_tables() from anon;
end if;
if exists (select 1 from pg_roles where rolname = 'authenticated') then
revoke all on function public.search_similar(text, vector, integer) from authenticated;
revoke all on function public.search_similar(text, vector, integer, text[], jsonb) from authenticated;
revoke all on function public.list_tables() from authenticated;
end if;
if exists (select 1 from pg_roles where rolname = 'vector_reader') then
grant execute on function public.search_similar(text, vector, integer) to vector_reader;
grant execute on function public.search_similar(text, vector, integer, text[], jsonb)
to vector_reader;
grant execute on function public.list_tables() to vector_reader;
end if;
end $$;
harness/scripts/create_vector_writer_rpc.sql
+24
View File
@@ -101,9 +101,30 @@ begin
end;
$$;
create or replace function public.delete_vector_generation(
table_name text, kind text, generation text
)
returns jsonb
language plpgsql
security definer
set search_path = public, vectors, extensions
as $$
declare affected integer;
begin
if table_name <> 'evidence' or kind <> 'evidence' or generation !~ '^gen:[0-9a-f]{32}$' then
raise exception 'only an exact Evidence generation may be deleted';
end if;
delete from vectors.evidence e
where e.kind = 'evidence' and e.metadata->>'vector_generation' = generation;
get diagnostics affected = row_count;
return jsonb_build_object('deleted', affected);
end;
$$;
revoke all on function public._assert_vector_write_table(text, text[]) from public;
revoke all on function public.existing_vector_hashes(text, text[]) from public;
revoke all on function public.upsert_vector_records(text, jsonb) from public;
revoke all on function public.delete_vector_generation(text, text, text) from public;
-- Su alcuni progetti Supabase le funzioni in `public` ricevono grant automatici: revoca
-- esplicitamente dai ruoli client generici, poi abilita solo il writer dedicato.
@@ -112,14 +133,17 @@ begin
if exists (select 1 from pg_roles where rolname = 'anon') then
revoke all on function public.existing_vector_hashes(text, text[]) from anon;
revoke all on function public.upsert_vector_records(text, jsonb) from anon;
revoke all on function public.delete_vector_generation(text, text, text) from anon;
end if;
if exists (select 1 from pg_roles where rolname = 'authenticated') then
revoke all on function public.existing_vector_hashes(text, text[]) from authenticated;
revoke all on function public.upsert_vector_records(text, jsonb) from authenticated;
revoke all on function public.delete_vector_generation(text, text, text) from authenticated;
end if;
if exists (select 1 from pg_roles where rolname = 'vector_writer') then
grant execute on function public.existing_vector_hashes(text, text[]) to vector_writer;
grant execute on function public.upsert_vector_records(text, jsonb) to vector_writer;
grant execute on function public.delete_vector_generation(text, text, text) to vector_writer;
end if;
end $$;
harness/tests/test_search_similar_kinds.py
+19
View File
@@ -90,6 +90,25 @@ def test_search_similar_reraises_non_404_with_kinds(monkeypatch):
_client().search_similar("memory", [0.1] * 4, 5, kinds=["memory"])
def test_generation_filter_is_sent_exactly_and_legacy_404_fails_closed(monkeypatch):
calls = []
def fake_call(self, function, payload):
calls.append(payload)
raise VectorRestError("HTTP 404 missing filtered RPC")
monkeypatch.setattr(VectorRestClient, "_call", fake_call)
metadata_filter = {"vector_generation": "gen:abc", "document_ids": ["doc:1"]}
with pytest.raises(VectorRestError, match="404"):
_client().search_similar(
"evidence", [0.1] * 4, 5, kinds=["evidence"], metadata_filter=metadata_filter
)
assert calls == [{
"query_embedding": [0.1] * 4, "limit_count": 5, "table_name": "evidence",
"kinds": ["evidence"], "metadata_filter": metadata_filter,
}]
def test_rest_searcher_forwards_kinds_to_client():
calls = []
harness/tht/adapters/vector/legacy_direct.py
+4
View File
@@ -5,6 +5,7 @@ from sqlalchemy import Engine
from tht.ports.vector import (
VectorCapabilities,
VectorHealth,
VectorStoreError,
VectorWriteRecord,
VectorWriteUnavailable,
require_positive_limit,
@@ -49,8 +50,11 @@ class LegacyDirectVectorStore:
*,
limit: int,
kinds: list[str] | None = None,
metadata_filter: dict[str, object] | None = None,
) -> list[VectorHit]:
require_positive_limit(limit)
if metadata_filter is not None:
raise VectorStoreError("Legacy vector store cannot enforce metadata filtering")
hits: list[VectorHit] = []
for collection in collections:
table = TableVectorStore(
harness/tht/adapters/vector/pgvector.py
+50 -3
View File
@@ -85,6 +85,8 @@ class PgVectorStore:
search=self._reader is not None,
existing_hashes=writable,
upsert=writable,
metadata_filter=self._reader is not None,
delete_generation=writable,
)
def _probe(
@@ -242,6 +244,7 @@ class PgVectorStore:
*,
limit: int,
kinds: list[str] | None = None,
metadata_filter: dict[str, object] | None = None,
) -> list[VectorHit]:
require_positive_limit(limit)
if self._reader is None:
@@ -262,7 +265,27 @@ class PgVectorStore:
)
if kinds and not collection_kinds:
continue
where = sql.SQL(" WHERE kind = ANY(%s)") if collection_kinds else sql.SQL("")
clauses = []
filter_params = []
if collection_kinds:
clauses.append(sql.SQL("kind = ANY(%s)"))
filter_params.append(collection_kinds)
if metadata_filter is not None:
if collection != "evidence" or set(metadata_filter) != {
"vector_generation", "document_ids"
}:
raise VectorStoreError("Unsupported vector metadata filter")
generation = metadata_filter["vector_generation"]
document_ids = metadata_filter["document_ids"]
if not isinstance(generation, str) or not isinstance(document_ids, list):
raise VectorStoreError("Invalid vector metadata filter")
clauses.append(sql.SQL("metadata->>'vector_generation' = %s"))
clauses.append(sql.SQL("metadata->>'document_id' = ANY(%s)"))
filter_params.extend((generation, document_ids))
where = (
sql.SQL(" WHERE ") + sql.SQL(" AND ").join(clauses)
if clauses else sql.SQL("")
)
query = sql.SQL(
"SELECT metadata, 1 - (embedding {} %s::{}) AS similarity "
"FROM {}{} ORDER BY embedding {} %s::{}, record_key LIMIT %s"
@@ -275,8 +298,7 @@ class PgVectorStore:
_vector_type(self._schema),
)
params = [_vector_literal(embedding)]
if collection_kinds:
params.append(collection_kinds)
params.extend(filter_params)
params.extend((_vector_literal(embedding), limit))
cursor.execute(query, params)
hits.extend(hit_from_metadata(row[1], row[0]) for row in cursor.fetchall())
@@ -381,5 +403,30 @@ class PgVectorStore:
raw.close()
return len(records)
def delete_generation(self, collection: str, generation: str) -> int:
if collection != "evidence" or not generation.startswith("gen:"):
raise VectorStoreError("Only exact Evidence generations may be deleted")
raw = None
try:
raw = self._require_writer().raw_connection()
with raw.cursor() as cursor:
cursor.execute(
sql.SQL(
"DELETE FROM {} WHERE kind = 'evidence' "
"AND metadata->>'vector_generation' = %s"
).format(_collection(self._schema, collection)),
(generation,),
)
count = cursor.rowcount
raw.commit()
return count
except Exception as exc:
if raw is not None:
raw.rollback()
raise VectorWriteUnavailable("Vector generation cleanup unavailable") from exc
finally:
if raw is not None:
raw.close()
__all__ = ["ALLOWED_COLLECTIONS", "PgVectorStore"]
harness/tht/adapters/vector/thoth_http.py
+18 -2
View File
@@ -40,7 +40,8 @@ class ThothHttpVectorStore:
def capabilities(self) -> VectorCapabilities:
writable = self._writer is not None
return VectorCapabilities(
search=self._reader is not None, existing_hashes=writable, upsert=writable
search=self._reader is not None, existing_hashes=writable, upsert=writable,
metadata_filter=self._reader is not None, delete_generation=writable,
)
def health(self) -> VectorHealth:
@@ -91,6 +92,7 @@ class ThothHttpVectorStore:
*,
limit: int,
kinds: list[str] | None = None,
metadata_filter: dict[str, object] | None = None,
) -> list[VectorHit]:
require_positive_limit(limit)
if self._reader is None:
@@ -103,7 +105,13 @@ class ThothHttpVectorStore:
for collection in collections:
_collection("vectors", collection)
try:
rows = self._reader.search_similar(collection, embedding, limit, kinds=kinds)
if metadata_filter is None:
rows = self._reader.search_similar(collection, embedding, limit, kinds=kinds)
else:
rows = self._reader.search_similar(
collection, embedding, limit, kinds=kinds,
metadata_filter=metadata_filter,
)
except VectorRestError as exc:
raise VectorStoreError(str(exc)) from exc
hits.extend(
@@ -144,6 +152,14 @@ class ThothHttpVectorStore:
except VectorRestError as exc:
raise VectorStoreError(str(exc)) from exc
def delete_generation(self, collection: str, generation: str) -> int:
if collection != "evidence" or not generation.startswith("gen:"):
raise VectorStoreError("Only exact Evidence generations may be deleted")
try:
return self._require_writer().delete_generation(collection, generation)
except VectorRestError as exc:
raise VectorStoreError(str(exc)) from exc
@staticmethod
def _row(write_record: VectorWriteRecord) -> dict:
record = write_record.record
harness/tht/cli/search_cmd.py
+8 -3
View File
@@ -48,6 +48,9 @@ def search_cmd(
cfg = _load_config_or_exit(config)
require_vector_cfg(cfg)
from tht.search.evidence import active_searcher
runtime_searcher = active_searcher(cfg, open_searcher(cfg))
if kind is not None and kind not in KIND_MAP:
typer.secho(
f"ERRORE: --kind sconosciuto: {kind} (validi: {', '.join(KIND_MAP)})",
@@ -112,7 +115,7 @@ def search_cmd(
candidates = combined_search(
keyword=keyword, lsh_hits=lsh_hits,
store=open_searcher(cfg), embedder=make_embedder(cfg.embeddings),
store=runtime_searcher, embedder=make_embedder(cfg.embeddings),
top=cfg.search.schema_chunk_pool, rrf_k=cfg.search.rrf_k,
kinds=KIND_MAP["schema"],
)
@@ -156,7 +159,7 @@ def search_cmd(
kinds = KIND_MAP.get(kind) if kind else None
results = combined_search(
keyword=keyword, lsh_hits=lsh_hits if kind != "evidence" else None,
store=open_searcher(cfg), embedder=make_embedder(cfg.embeddings),
store=runtime_searcher, embedder=make_embedder(cfg.embeddings),
top=top, rrf_k=cfg.search.rrf_k, kinds=kinds,
)
@@ -249,7 +252,9 @@ def pack_cmd(
vec = None
searcher = embedder = None
try:
searcher = open_searcher(cfg)
from tht.search.evidence import active_searcher
searcher = active_searcher(cfg, open_searcher(cfg))
embedder = make_embedder(cfg.embeddings)
vec = embedder.embed_query(question)
except degrade as e:
harness/tht/cli/vector_cmd.py
+3 -2
View File
@@ -60,9 +60,10 @@ def open_searcher(cfg):
store = build_vector_store(cfg)
class AdapterSearcher:
def search(self, query_vec, top_n=10, kinds=None):
def search(self, query_vec, top_n=10, kinds=None, metadata_filter=None):
return store.search(
tables_for_kinds(kinds), query_vec, limit=top_n, kinds=kinds
tables_for_kinds(kinds), query_vec, limit=top_n, kinds=kinds,
metadata_filter=metadata_filter,
)
return AdapterSearcher()
harness/tht/corpus/pipeline.py
+18
View File
@@ -72,6 +72,8 @@ class CorpusPipeline:
return self._run(dry_run=dry_run, resume=resume)
def _run(self, *, dry_run: bool = False, resume: str | None = None) -> PipelineResult:
generation = None
vector_written = False
previous = self.store.active_manifest()
try:
discovered = self._discover()
@@ -138,6 +140,7 @@ class CorpusPipeline:
records = [self._vector_record(part, vector, generation) for part, vector in zip(changed_chunks, embeddings, strict=True)]
if records:
written = self.vector_store.upsert("evidence", records)
vector_written = True
if written != len(records):
raise PipelineError("vector write count mismatch")
generation_path = self.store.generation_path(generation)
@@ -156,11 +159,26 @@ class CorpusPipeline:
)
self.store.publish(staged)
except PipelineError:
self._compensate(generation, vector_written)
raise
except Exception as error:
self._compensate(generation, vector_written)
raise PipelineError("Evidence preprocessing failed") from error
return PipelineResult("succeeded", generation, True, changed, unchanged, removed, self.store.manifest(generation))
def _compensate(self, generation: str | None, vector_written: bool) -> None:
if generation is None:
return
try:
self.store.discard(generation)
except Exception:
pass
if vector_written:
try:
self.vector_store.delete_generation("evidence", generation)
except Exception:
pass
@staticmethod
def _vector_record(chunk: CanonicalChunk, embedding: list[float], generation: str):
record = VectorRecord(
harness/tht/corpus/store.py
+8
View File
@@ -7,6 +7,7 @@ import fcntl
import os
import re
import stat
import shutil
import uuid
from pathlib import Path
from contextlib import contextmanager
@@ -137,6 +138,13 @@ class CorpusStore:
raise UnsafeCorpusPath("manifest must not be a symlink")
return CorpusManifest.model_validate_json(manifest_path.read_text(encoding="utf-8"))
def discard(self, generation: str) -> None:
path = self.generation_path(generation)
if path.exists():
if path.is_symlink() or not stat.S_ISDIR(path.lstat().st_mode):
raise UnsafeCorpusPath("generation cleanup target is unsafe")
shutil.rmtree(path)
def active_manifest(self) -> CorpusManifest | None:
generation = self.active_generation()
return self.manifest(generation) if generation else None
harness/tht/ports/vector.py
+5
View File
@@ -12,6 +12,8 @@ class VectorCapabilities:
search: bool = True
existing_hashes: bool = False
upsert: bool = False
metadata_filter: bool = False
delete_generation: bool = False
@dataclass(frozen=True)
@@ -70,12 +72,15 @@ class VectorStore(Protocol):
*,
limit: int,
kinds: list[str] | None = None,
metadata_filter: dict[str, object] | None = None,
) -> list[VectorHit]: ...
def existing_hashes(self, collection: str, kinds: list[str]) -> dict[str, str]: ...
def upsert(self, collection: str, records: list[VectorWriteRecord]) -> int: ...
def delete_generation(self, collection: str, generation: str) -> int: ...
__all__ = [
"VectorCapabilities",
harness/tht/search/evidence.py
+38
View File
@@ -3,6 +3,44 @@
from tht.corpus.store import CorpusStore
class ActiveEvidenceSearcher:
"""Searcher facade that enforces ACTIVE generation predicates before LIMIT."""
def __init__(self, corpus: CorpusStore, delegate):
self.corpus = corpus
self.delegate = delegate
def search(self, embedding, top_n=10, kinds=None, metadata_filter=None):
if kinds != ["evidence"]:
return self.delegate.search(embedding, top_n=top_n, kinds=kinds)
manifest = self.corpus.active_manifest()
if manifest is None:
return []
by_generation: dict[str, list[str]] = {}
mapping = dict(manifest.metadata.get("document_generations", {}))
for document in manifest.documents:
generation = mapping.get(document.document_id, manifest.vector_generation)
if generation:
by_generation.setdefault(generation, []).append(document.document_id)
hits = []
for generation, document_ids in by_generation.items():
hits.extend(self.delegate.search(
embedding, top_n=top_n, kinds=["evidence"],
metadata_filter={
"vector_generation": generation,
"document_ids": document_ids,
},
))
return sorted(hits, key=lambda hit: (-hit.similarity, hit.id))[:top_n]
def active_searcher(cfg, delegate):
corpus_root = cfg.paths.artifacts.parent / "corpus"
if not corpus_root.exists():
return delegate
return ActiveEvidenceSearcher(CorpusStore(corpus_root), delegate)
def active_evidence_hits(store: CorpusStore, vector_store, embedding, *, limit: int):
manifest = store.active_manifest()
if manifest is None or manifest.vector_generation is None:
harness/tht/session/artifacts.py
+3 -7
View File
@@ -7,15 +7,11 @@ from tht.session.models import SchemaLinking
def _find_evidence_file(evidence_root: Path, evidence_id: str) -> str:
# New deployments resolve only immutable materialized files from ACTIVE. Keep
# the legacy curated-tree fallback for sessions created before a corpus exists.
try:
corpus_root = evidence_root.parent.parent / "corpus"
if corpus_root.exists():
from tht.corpus.store import CorpusStore
from tht.search.evidence import resolve_evidence_file
corpus_root = evidence_root.parent.parent / "corpus"
if corpus_root.exists():
return resolve_evidence_file(CorpusStore(corpus_root), evidence_id)
except (OSError, RuntimeError, ValueError):
pass
return resolve_evidence_file(CorpusStore(corpus_root), evidence_id)
for match in evidence_root.rglob(f"{evidence_id}.md"):
return str(match)
return ""
harness/tht/vectorstore/rest_client.py
+17
View File
@@ -60,6 +60,7 @@ class VectorRestClient:
def search_similar(
self, table_name: str, query_embedding: list[float], limit_count: int,
kinds: list[str] | None = None,
metadata_filter: dict | None = None,
) -> list[dict]:
"""Ricerca per similarità coseno su `vectors.<table_name>`: ritorna le righe
`{id, similarity, metadata}` ordinate per similarity decrescente. Con `kinds`
@@ -72,6 +73,13 @@ class VectorRestClient:
"limit_count": limit_count,
"table_name": table_name,
}
if metadata_filter is not None:
# ACTIVE corpus reads must never degrade to an unfiltered legacy RPC:
# filtering after LIMIT is incomplete and could expose stale generations.
return self._call(
"search_similar",
{**args, "kinds": kinds, "metadata_filter": metadata_filter},
) or []
if kinds is not None:
try:
return self._call("search_similar", {**args, "kinds": kinds}) or []
@@ -118,3 +126,12 @@ class VectorRestClient:
return int(payload[0]["upserted"])
return len(payload)
return len(rows)
def delete_generation(self, table_name: str, generation: str) -> int:
payload = self._call(
"delete_vector_generation",
{"table_name": table_name, "kind": "evidence", "generation": generation},
)
if isinstance(payload, dict):
return int(payload.get("deleted", 0))
return 0