mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(preprocess): verify canonical generations and cleanup

Browse Source
This commit is contained in:
marcopan
2026-07-12 06:18:18 +02:00
parent 4a29086fe4
commit 7d41c4cefc
6 changed files with 86 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/preprocess-smoke.sh
+41 -16
View File
@@ -2,20 +2,50 @@
set -eu
cd "$(dirname "$0")/.."
tmp=$(mktemp -d "${TMPDIR:-/tmp}/thoth-preprocess.XXXXXX")
project="thoth-preprocess-$$"
if [ "${1:-}" = "--cleanup-failure" ] && [ -z "${PREPROCESS_SMOKE_CHILD:-}" ]; then
child_project="thoth-preprocess-failure-$$"
child_tmp=$(mktemp -d "${TMPDIR:-/tmp}/thoth-preprocess-failure.XXXXXX")
set +e
PREPROCESS_SMOKE_CHILD=1 PREPROCESS_SMOKE_INJECT_FAILURE=1 \
PREPROCESS_SMOKE_PROJECT="$child_project" PREPROCESS_SMOKE_TMP="$child_tmp" "$0"
child_status=$?
set -e
test "$child_status" -eq 97
test ! -e "$child_tmp"
test -z "$(docker ps -aq --filter "label=com.docker.compose.project=$child_project")"
test -z "$(docker volume ls -q --filter "label=com.docker.compose.project=$child_project")"
test -z "$(docker network ls -q --filter "label=com.docker.compose.project=$child_project")"
echo "injected preprocessing failure preserved status and cleaned every owned resource."
exit 0
fi
tmp=${PREPROCESS_SMOKE_TMP:-$(mktemp -d "${TMPDIR:-/tmp}/thoth-preprocess.XXXXXX")}
project=${PREPROCESS_SMOKE_PROJECT:-thoth-preprocess-$$}
compose=""
cleanup() {
trap - EXIT HUP INT TERM
original_status=$?
cleanup_failed=0
set +e
if [ -n "$compose" ]; then
$compose down --volumes >/dev/null
test "$?" -eq 0 || cleanup_failed=1
fi
test -z "$(docker ps -aq --filter "label=com.docker.compose.project=$project")"
test -z "$(docker volume ls -q --filter "label=com.docker.compose.project=$project")"
test -z "$(docker network ls -q --filter "label=com.docker.compose.project=$project")"
test -z "$(docker ps -aq --filter "label=com.docker.compose.project=$project")" || cleanup_failed=1
test -z "$(docker volume ls -q --filter "label=com.docker.compose.project=$project")" || cleanup_failed=1
test -z "$(docker network ls -q --filter "label=com.docker.compose.project=$project")" || cleanup_failed=1
rm -rf "$tmp"
test ! -e "$tmp" || cleanup_failed=1
if [ "$original_status" -ne 0 ]; then
exit "$original_status"
fi
if [ "$cleanup_failed" -ne 0 ]; then
exit 1
fi
}
trap cleanup EXIT HUP INT TERM
trap cleanup EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM
mkdir -p "$tmp/source/evidence"
printf '%s\n' '# Evidence' 'generation one' >"$tmp/source/evidence/a.md"
for name in bootstrap migrator reader writer; do
@@ -61,15 +91,12 @@ $compose build preprocess-evidence
$compose up -d vector-db mock-embeddings
$compose run --rm vector-reconcile >/dev/null
$compose run --rm --no-deps vector-migrate >/dev/null
if [ "${1:-}" = "--cleanup-failure" ]; then
cleanup
test ! -e "$tmp"
echo "injected preprocessing failure cleanup passed."
exit 0
if [ "${PREPROCESS_SMOKE_INJECT_FAILURE:-0}" = "1" ]; then
sh -c 'exit 97'
fi
generation_count() {
$compose run --rm --no-deps --entrypoint sh preprocess-evidence -c \
'find /data/workspaces/preprocess-evidence/corpus/generations -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l'
$compose run --rm --no-deps --entrypoint /opt/venv/bin/python preprocess-evidence -c \
'import pathlib,re; root=pathlib.Path("/data/workspaces/preprocess-evidence/corpus"); print(sum(1 for p in root.iterdir() if p.is_dir() and re.fullmatch(r"gen-[0-9a-f]{32}",p.name)) if root.exists() else 0)'
}
before=$(generation_count)
first=$($compose run --rm preprocess-evidence)
@@ -103,5 +130,3 @@ import json, sys
assert json.loads(sys.argv[1])["generation"] == sys.argv[2].strip()
PY
echo "real Compose preprocessing unchanged rerun, mutation, DWH job, and ACTIVE publish passed."
cleanup
test ! -e "$tmp"