126 lines
5.2 KiB
Bash
Executable File
126 lines
5.2 KiB
Bash
Executable File
#!/bin/sh
|
|
set -eu
|
|
cd "$(dirname "$0")/.."
|
|
|
|
if [ "${1:-}" = "--cleanup-failure" ] && [ -z "${PREPROCESS_SMOKE_CHILD:-}" ]; then
|
|
child_project="thoth-preprocess-failure-$$"
|
|
child_tmp=$(mktemp -d "${TMPDIR:-/tmp}/thoth-preprocess-failure.XXXXXX")
|
|
set +e
|
|
PREPROCESS_SMOKE_CHILD=1 PREPROCESS_SMOKE_INJECT_FAILURE=1 \
|
|
PREPROCESS_SMOKE_PROJECT="$child_project" PREPROCESS_SMOKE_TMP="$child_tmp" "$0"
|
|
child_status=$?
|
|
set -e
|
|
test "$child_status" -eq 97
|
|
test ! -e "$child_tmp"
|
|
test -z "$(docker ps -aq --filter "label=com.docker.compose.project=$child_project")"
|
|
test -z "$(docker volume ls -q --filter "label=com.docker.compose.project=$child_project")"
|
|
test -z "$(docker network ls -q --filter "label=com.docker.compose.project=$child_project")"
|
|
echo "injected preprocessing failure preserved status and cleaned every owned resource."
|
|
exit 0
|
|
fi
|
|
|
|
tmp=${PREPROCESS_SMOKE_TMP:-$(mktemp -d "${TMPDIR:-/tmp}/thoth-preprocess.XXXXXX")}
|
|
project=${PREPROCESS_SMOKE_PROJECT:-thoth-preprocess-$$}
|
|
compose=""
|
|
cleanup() {
|
|
original_status=$?
|
|
cleanup_failed=0
|
|
set +e
|
|
if [ -n "$compose" ]; then
|
|
$compose down --volumes >/dev/null
|
|
test "$?" -eq 0 || cleanup_failed=1
|
|
fi
|
|
test -z "$(docker ps -aq --filter "label=com.docker.compose.project=$project")" || cleanup_failed=1
|
|
test -z "$(docker volume ls -q --filter "label=com.docker.compose.project=$project")" || cleanup_failed=1
|
|
test -z "$(docker network ls -q --filter "label=com.docker.compose.project=$project")" || cleanup_failed=1
|
|
rm -rf "$tmp"
|
|
test ! -e "$tmp" || cleanup_failed=1
|
|
if [ "$original_status" -ne 0 ]; then
|
|
exit "$original_status"
|
|
fi
|
|
if [ "$cleanup_failed" -ne 0 ]; then
|
|
exit 1
|
|
fi
|
|
}
|
|
trap cleanup EXIT
|
|
trap 'exit 129' HUP
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
mkdir -p "$tmp/source/evidence"
|
|
printf '%s\n' '# Evidence' 'generation one' >"$tmp/source/evidence/a.md"
|
|
for name in bootstrap migrator reader writer; do
|
|
printf '%s' "smoke-$name-$project" >"$tmp/$name"
|
|
chmod 0600 "$tmp/$name"
|
|
done
|
|
export THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE="$tmp/bootstrap"
|
|
export THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE="$tmp/migrator"
|
|
export THT_VECTOR_READER_PASSWORD_SECRET_FILE="$tmp/reader"
|
|
export THT_VECTOR_WRITER_PASSWORD_SECRET_FILE="$tmp/writer"
|
|
export THT_OLLAMA_URL=http://mock-embeddings:8081
|
|
|
|
cat >"$tmp/smoke.yaml" <<YAML
|
|
services:
|
|
mock-embeddings:
|
|
image: thothii-core:local
|
|
profiles: [preprocess]
|
|
entrypoint: [/opt/venv/bin/python, -c]
|
|
command:
|
|
- |
|
|
import json
|
|
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
class H(BaseHTTPRequestHandler):
|
|
def do_POST(self):
|
|
n=len(json.loads(self.rfile.read(int(self.headers['Content-Length'])))['input'])
|
|
body=json.dumps({'embeddings': [[1.0]+[0.0]*767 for _ in range(n)]}).encode()
|
|
self.send_response(200); self.send_header('Content-Length', str(len(body))); self.end_headers(); self.wfile.write(body)
|
|
def log_message(self, *args): pass
|
|
HTTPServer(('0.0.0.0',8081),H).serve_forever()
|
|
preprocess-evidence:
|
|
volumes:
|
|
- $tmp/source:/data/source:ro
|
|
depends_on:
|
|
mock-embeddings: {condition: service_started}
|
|
YAML
|
|
|
|
compose="docker compose -f compose.yaml -f deploy/compose.local-vector.yaml -f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml -f $tmp/smoke.yaml --project-name $project --profile local-vector --profile preprocess"
|
|
$compose build preprocess-evidence
|
|
if [ "${PREPROCESS_SMOKE_INJECT_FAILURE:-0}" = "1" ]; then
|
|
sh -c 'exit 97'
|
|
fi
|
|
generation_count() {
|
|
$compose run --rm --no-deps --entrypoint /opt/venv/bin/python preprocess-evidence -c \
|
|
'import pathlib,re; root=pathlib.Path("/data/workspaces/preprocess-evidence/corpus"); print(sum(1 for p in root.iterdir() if p.is_dir() and re.fullmatch(r"gen-[0-9a-f]{32}",p.name)) if root.exists() else 0)'
|
|
}
|
|
before=0
|
|
first=$($compose run --rm preprocess-evidence)
|
|
after_first=$(generation_count)
|
|
second=$($compose run --rm preprocess-evidence)
|
|
after_second=$(generation_count)
|
|
printf '%s' 'generation two' >>"$tmp/source/evidence/a.md"
|
|
third=$($compose run --rm preprocess-evidence)
|
|
after_third=$(generation_count)
|
|
dwh=$($compose run --rm preprocess-dwh)
|
|
python3 - "$first" "$second" "$third" "$before" "$after_first" "$after_second" "$after_third" <<'PY'
|
|
import json, sys
|
|
a, b, c = map(json.loads, sys.argv[1:4])
|
|
before, first_count, second_count, third_count = map(int, sys.argv[4:])
|
|
assert len(a["changed"]) == 1 and not a["unchanged"]
|
|
assert len(b["unchanged"]) == 1 and not b["changed"]
|
|
assert len(c["changed"]) == 1 and c["generation"] != a["generation"]
|
|
assert a["published"] and not b["published"] and c["published"]
|
|
assert first_count == before + 1
|
|
assert second_count == first_count
|
|
assert third_count == second_count + 1
|
|
PY
|
|
python3 - "$dwh" <<'PY'
|
|
import json, sys
|
|
assert json.loads(sys.argv[1])["status"] == "succeeded"
|
|
PY
|
|
active=$($compose run --rm --no-deps --entrypoint sh preprocess-evidence -c \
|
|
'cat /data/workspaces/preprocess-evidence/corpus/ACTIVE')
|
|
python3 - "$third" "$active" <<'PY'
|
|
import json, sys
|
|
assert json.loads(sys.argv[1])["generation"] == sys.argv[2].strip()
|
|
PY
|
|
echo "real Compose preprocessing unchanged rerun, mutation, DWH job, and ACTIVE publish passed."
|