fix(preprocess): harden S3 and real Compose jobs

2026-07-12 06:01:06 +02:00
parent 950f88f23e
commit c6966f3d15
10 changed files with 222 additions and 43 deletions
@@ -2,14 +2,81 @@
 set -eu
 cd "$(dirname "$0")/.."

-rendered=$(docker compose -f compose.yaml -f deploy/compose.preprocess.yaml --profile preprocess config)
-printf '%s' "$rendered" | grep -q 'preprocess-evidence:'
-printf '%s' "$rendered" | grep -q 'preprocess-dwh:'
-if printf '%s' "$rendered" | grep -qi 'access-secret\|secret-key'; then
-  echo "preprocess Compose rendered secret material" >&2
-  exit 1
-fi
-(cd harness && .venv/bin/pytest -q \
-  tests/test_corpus_pipeline.py::test_unchanged_documents_skip_acquire_normalize_chunk_and_embed \
-  tests/test_corpus_pipeline.py::test_retention_bounds_generations_and_purges_vectors_after_publish)
-echo "preprocess unchanged rerun and modified-generation smoke passed."
+tmp=$(mktemp -d "${TMPDIR:-/tmp}/thoth-preprocess.XXXXXX")
+project="thoth-preprocess-$$"
+cleanup() {
+  docker compose -f compose.yaml -f deploy/compose.local-vector.yaml \
+    -f deploy/compose.preprocess.yaml -f "$tmp/smoke.yaml" --project-name "$project" \
+    --profile local-vector --profile preprocess down --volumes >/dev/null 2>&1 || true
+  rm -rf "$tmp"
+}
+trap cleanup EXIT HUP INT TERM
+mkdir -p "$tmp/source/evidence"
+printf '%s\n' '# Evidence' 'generation one' >"$tmp/source/evidence/a.md"
+for name in bootstrap migrator reader writer; do
+  printf '%s' "smoke-$name-$project" >"$tmp/$name"
+  chmod 0600 "$tmp/$name"
+done
+export THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE="$tmp/bootstrap"
+export THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE="$tmp/migrator"
+export THT_VECTOR_READER_PASSWORD_SECRET_FILE="$tmp/reader"
+export THT_VECTOR_WRITER_PASSWORD_SECRET_FILE="$tmp/writer"
+export THT_OLLAMA_URL=http://mock-embeddings:8081
+
+cat >"$tmp/smoke.yaml" <<YAML
+services:
+  mock-embeddings:
+    image: thothii-core:local
+    profiles: [preprocess]
+    entrypoint: [/opt/venv/bin/python, -c]
+    command:
+      - |
+        import json
+        from http.server import BaseHTTPRequestHandler, HTTPServer
+        class H(BaseHTTPRequestHandler):
+          def do_POST(self):
+            n=len(json.loads(self.rfile.read(int(self.headers['Content-Length'])))['input'])
+            body=json.dumps({'embeddings': [[1.0]+[0.0]*767 for _ in range(n)]}).encode()
+            self.send_response(200); self.send_header('Content-Length', str(len(body))); self.end_headers(); self.wfile.write(body)
+          def log_message(self, *args): pass
+        HTTPServer(('0.0.0.0',8081),H).serve_forever()
+  preprocess-evidence:
+    volumes:
+      - $tmp/source:/data/source:ro
+    depends_on:
+      vector-migrate: {condition: service_completed_successfully}
+      mock-embeddings: {condition: service_started}
+  preprocess-dwh:
+    depends_on:
+      vector-migrate: {condition: service_completed_successfully}
+YAML
+
+compose="docker compose -f compose.yaml -f deploy/compose.local-vector.yaml -f deploy/compose.preprocess.yaml -f $tmp/smoke.yaml --project-name $project --profile local-vector --profile preprocess"
+$compose build preprocess-evidence
+$compose up -d vector-db mock-embeddings
+$compose run --rm vector-reconcile >/dev/null
+$compose run --rm --no-deps vector-migrate >/dev/null
+first=$($compose run --rm preprocess-evidence)
+second=$($compose run --rm preprocess-evidence)
+printf '%s' 'generation two' >>"$tmp/source/evidence/a.md"
+third=$($compose run --rm preprocess-evidence)
+dwh=$($compose run --rm preprocess-dwh)
+python3 - "$first" "$second" "$third" <<'PY'
+import json, sys
+a, b, c = map(json.loads, sys.argv[1:])
+assert len(a["changed"]) == 1 and not a["unchanged"]
+assert len(b["unchanged"]) == 1 and not b["changed"]
+assert len(c["changed"]) == 1 and c["generation"] != a["generation"]
+assert all(row["published"] for row in (a, b, c))
+PY
+python3 - "$dwh" <<'PY'
+import json, sys
+assert json.loads(sys.argv[1])["status"] == "succeeded"
+PY
+active=$($compose run --rm --no-deps --entrypoint sh preprocess-evidence -c \
+  'cat /data/workspaces/preprocess-evidence/corpus/ACTIVE')
+python3 - "$third" "$active" <<'PY'
+import json, sys
+assert json.loads(sys.argv[1])["generation"] == sys.argv[2].strip()
+PY
+echo "real Compose preprocessing unchanged rerun, mutation, DWH job, and ACTIVE publish passed."