fix(evidence): reconcile generations safely
This commit is contained in:
@@ -79,6 +79,13 @@ def vector_configs():
|
||||
f"GRANT INSERT, UPDATE ON vectors.{table} "
|
||||
"TO vector_l0_writer, vector_l0_no_sequence"
|
||||
)
|
||||
if table == "evidence":
|
||||
connection.exec_driver_sql(
|
||||
"GRANT DELETE ON vectors.evidence TO vector_l0_writer"
|
||||
)
|
||||
connection.exec_driver_sql(
|
||||
"GRANT SELECT (kind, metadata) ON vectors.evidence TO vector_l0_writer"
|
||||
)
|
||||
connection.exec_driver_sql(
|
||||
f"GRANT SELECT (record_key, kind, content_hash) "
|
||||
f"ON vectors.{table} TO vector_l0_writer, vector_l0_no_sequence"
|
||||
@@ -118,6 +125,21 @@ def test_pgvector_round_trip_hash_and_upsert(store):
|
||||
assert store.search(["memory"], [0.8, 0.2], limit=1)[0].id == "record:a"
|
||||
|
||||
|
||||
def test_pgvector_lists_and_deletes_exact_evidence_generation(store):
|
||||
generation = "gen:" + "a" * 32
|
||||
value = VectorWriteRecord(
|
||||
record=VectorRecord(
|
||||
id="evidence-generation-a", kind="evidence", ref="doc:a", title="a",
|
||||
content="content", metadata={"vector_generation": generation},
|
||||
),
|
||||
embedding=[1.0, 0.0], content_hash="sha256:" + "a" * 64,
|
||||
)
|
||||
store.upsert("evidence", [value])
|
||||
assert generation in store.list_evidence_generations("evidence")
|
||||
assert store.delete_generation("evidence", generation) == 1
|
||||
assert generation not in store.list_evidence_generations("evidence")
|
||||
|
||||
|
||||
def test_pgvector_search_filters_kinds_before_limit(store):
|
||||
store.upsert("memory", [_record("solved", [1.0, 0.0], kind="solved_question")])
|
||||
hits = store.search("memory".split(), [1.0, 0.0], limit=1, kinds=["memory"])
|
||||
|
||||
@@ -250,3 +250,17 @@ def test_http_delete_generation_legacy_404_fails_closed_without_body_leak(monkey
|
||||
with pytest.raises(VectorRestError, match="delete_vector_generation RPC is unavailable") as error:
|
||||
client.delete_generation("evidence", "gen:" + "a" * 32)
|
||||
assert "secret" not in str(error.value)
|
||||
|
||||
|
||||
def test_http_list_evidence_generations_exact_rpc_and_legacy_fail_closed(monkeypatch):
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
"tht.vectorstore.rest_client.requests.post",
|
||||
lambda url, json, **kwargs: calls.append((url, json)) or Response([
|
||||
{"generation": "gen:" + "a" * 32}
|
||||
]),
|
||||
)
|
||||
client = VectorRestClient(RestConfig(base_url="https://vectors.test", api_key="writer"))
|
||||
assert client.list_evidence_generations("evidence") == ["gen:" + "a" * 32]
|
||||
assert calls[0][0].endswith("/rpc/list_evidence_generations")
|
||||
assert calls[0][1] == {"table_name": "evidence", "kind": "evidence"}
|
||||
|
||||
@@ -61,6 +61,12 @@ class Vectors:
|
||||
]
|
||||
return 0
|
||||
|
||||
def list_evidence_generations(self, collection):
|
||||
return sorted({
|
||||
value.record.metadata["vector_generation"] for value in self.records
|
||||
if value.record.kind == "evidence"
|
||||
})
|
||||
|
||||
|
||||
class InterruptingVectors(Vectors):
|
||||
def __init__(self):
|
||||
@@ -142,6 +148,22 @@ def test_retention_keeps_filesystem_when_vector_purge_fails_then_retries(tmp_pat
|
||||
assert len(CorpusStore(tmp_path / "corpus").list_generations()) == 1
|
||||
|
||||
|
||||
def test_gc_reconciles_vector_only_generation(tmp_path):
|
||||
vectors = Vectors()
|
||||
orphan = "gen:" + "f" * 32
|
||||
from tht.ports.vector import VectorWriteRecord
|
||||
from tht.vectorstore.records import VectorRecord
|
||||
vectors.records.append(VectorWriteRecord(
|
||||
record=VectorRecord(id="orphan", kind="evidence", ref="doc:x", title="", content="x",
|
||||
metadata={"vector_generation": orphan}),
|
||||
embedding=[0.0, 0.0, 0.0], content_hash="sha256:" + "0" * 64,
|
||||
))
|
||||
candidate = pipeline(tmp_path, Source([]), vectors=vectors, retain=1)
|
||||
report = candidate.gc(workspace_root=tmp_path)
|
||||
assert report["evicted"] == [orphan]
|
||||
assert vectors.list_evidence_generations("evidence") == []
|
||||
|
||||
|
||||
def test_unchanged_documents_skip_acquire_normalize_chunk_and_embed(tmp_path):
|
||||
one = item("one", "a")
|
||||
first_source = Source([(one, "hello")])
|
||||
|
||||
@@ -97,3 +97,14 @@ def test_generation_inventory_is_validated_and_sorted(tmp_path):
|
||||
second = store.stage(CorpusManifest(), {}, generation="gen:" + "2" * 32)
|
||||
(store.root / "unrelated").mkdir()
|
||||
assert store.list_generations() == [first, second]
|
||||
|
||||
|
||||
def test_published_inventory_excludes_staged_and_invalid_newer_directories(tmp_path):
|
||||
store = CorpusStore(tmp_path / "corpus")
|
||||
first = store.stage(CorpusManifest(), {}, generation="gen:" + "1" * 32)
|
||||
store.publish(first)
|
||||
store.stage(CorpusManifest(), {}, generation="gen:" + "2" * 32)
|
||||
invalid = store.generation_path("gen:" + "3" * 32)
|
||||
invalid.mkdir()
|
||||
(invalid / "PUBLISHED").write_text("2026-01-01T00:00:00Z\n")
|
||||
assert store.published_generations() == [first]
|
||||
|
||||
Reference in New Issue
Block a user