fix(preprocess): verify canonical generations and cleanup

This commit is contained in:
2026-07-12 06:18:18 +02:00
parent 4a29086fe4
commit 7d41c4cefc
6 changed files with 86 additions and 18 deletions
+3 -2
View File
@@ -29,8 +29,9 @@ class S3EvidenceSource:
if (not bucket_valid or bucket_is_ip
or any(value < 1 for value in (max_bytes, max_objects, max_pages, page_size))):
raise ValueError("S3 evidence limits and bucket must be non-empty and positive")
if prefix.startswith("/"):
raise ValueError("S3 prefix must not start with a slash")
if (prefix.startswith("/") or len(prefix.encode()) > 1024
or any(ord(char) < 32 or ord(char) == 127 for char in prefix)):
raise ValueError("S3 prefix is invalid")
if endpoint_url:
parsed = urlsplit(endpoint_url)
if parsed.username or parsed.password:
+10
View File
@@ -220,6 +220,16 @@ class CorpusPipeline:
"dimensions": self.embedding_dimensions,
"chunk_policy": asdict(self.chunk_policy),
})
previous = self.store.active_manifest()
if (not dry_run and resume_run_id is None and previous is not None
and previous.metadata.get("compatibility_fingerprint") == compatibility
and previous.metadata.get("fingerprints") == {
item.source_id: item.fingerprint for _, item in discovered
}):
return PipelineResult(
"succeeded", previous.manifest_id, False, (),
tuple(sorted(item.source_id for _, item in discovered)), (), previous,
)
spec = JobSpec(
workspace_id=workspace_id,
job_type="evidence",