fix(evidence): enforce active vector generations
This commit is contained in:
@@ -72,6 +72,8 @@ class CorpusPipeline:
|
||||
return self._run(dry_run=dry_run, resume=resume)
|
||||
|
||||
def _run(self, *, dry_run: bool = False, resume: str | None = None) -> PipelineResult:
|
||||
generation = None
|
||||
vector_written = False
|
||||
previous = self.store.active_manifest()
|
||||
try:
|
||||
discovered = self._discover()
|
||||
@@ -138,6 +140,7 @@ class CorpusPipeline:
|
||||
records = [self._vector_record(part, vector, generation) for part, vector in zip(changed_chunks, embeddings, strict=True)]
|
||||
if records:
|
||||
written = self.vector_store.upsert("evidence", records)
|
||||
vector_written = True
|
||||
if written != len(records):
|
||||
raise PipelineError("vector write count mismatch")
|
||||
generation_path = self.store.generation_path(generation)
|
||||
@@ -156,11 +159,26 @@ class CorpusPipeline:
|
||||
)
|
||||
self.store.publish(staged)
|
||||
except PipelineError:
|
||||
self._compensate(generation, vector_written)
|
||||
raise
|
||||
except Exception as error:
|
||||
self._compensate(generation, vector_written)
|
||||
raise PipelineError("Evidence preprocessing failed") from error
|
||||
return PipelineResult("succeeded", generation, True, changed, unchanged, removed, self.store.manifest(generation))
|
||||
|
||||
def _compensate(self, generation: str | None, vector_written: bool) -> None:
|
||||
if generation is None:
|
||||
return
|
||||
try:
|
||||
self.store.discard(generation)
|
||||
except Exception:
|
||||
pass
|
||||
if vector_written:
|
||||
try:
|
||||
self.vector_store.delete_generation("evidence", generation)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def _vector_record(chunk: CanonicalChunk, embedding: list[float], generation: str):
|
||||
record = VectorRecord(
|
||||
|
||||
Reference in New Issue
Block a user