feat(preprocess): add deployment jobs and S3 source

This commit is contained in:
2026-07-12 05:42:07 +02:00
parent 6656a69630
commit 4028ef7821
12 changed files with 380 additions and 10 deletions
+15
View File
@@ -0,0 +1,15 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
rendered=$(docker compose -f compose.yaml -f deploy/compose.preprocess.yaml --profile preprocess config)
printf '%s' "$rendered" | grep -q 'preprocess-evidence:'
printf '%s' "$rendered" | grep -q 'preprocess-dwh:'
if printf '%s' "$rendered" | grep -qi 'access-secret\|secret-key'; then
echo "preprocess Compose rendered secret material" >&2
exit 1
fi
(cd harness && .venv/bin/pytest -q \
tests/test_corpus_pipeline.py::test_unchanged_documents_skip_acquire_normalize_chunk_and_embed \
tests/test_corpus_pipeline.py::test_retention_bounds_generations_and_purges_vectors_after_publish)
echo "preprocess unchanged rerun and modified-generation smoke passed."