fix(deploy): reconcile local vector credentials safely
This commit is contained in:
+115
-25
@@ -3,22 +3,23 @@ set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
smoke_project=${SMOKE_PROJECT:-"thothii-vector-smoke-$(date +%s)-$$"}
|
||||
keep_resources=${KEEP_SMOKE_RESOURCES:-0}
|
||||
secret_dir=$(mktemp -d)
|
||||
marker="local-vector-$smoke_project"
|
||||
|
||||
case "$smoke_project" in
|
||||
thothii)
|
||||
echo "SMOKE_PROJECT=thothii is reserved for the operator stack" >&2
|
||||
exit 2
|
||||
;;
|
||||
""|*[!a-z0-9_-]*|[!a-z0-9]*)
|
||||
echo "invalid SMOKE_PROJECT: use lowercase letters, digits, hyphens, or underscores" >&2
|
||||
exit 2
|
||||
;;
|
||||
mode=${1:-run}
|
||||
case "$mode" in
|
||||
run|--live-collision-test) ;;
|
||||
*) echo "usage: $0 [--live-collision-test]" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
keep_resources=${KEEP_SMOKE_RESOURCES:-0}
|
||||
if [ "${SMOKE_PROJECT+x}" = x ]; then
|
||||
echo "SMOKE_PROJECT is not accepted; the smoke always generates an owned namespace" >&2
|
||||
exit 2
|
||||
fi
|
||||
secret_dir=$(mktemp -d "${TMPDIR:-/tmp}/thothii-vector-smoke.XXXXXX")
|
||||
suffix=$(basename "$secret_dir" | tr -cd 'a-z0-9')
|
||||
smoke_project="thothii-vector-smoke-$(date +%s)-$$-$suffix"
|
||||
smoke_owner="$smoke_project-owner"
|
||||
marker="local-vector-$smoke_project"
|
||||
|
||||
for secret in bootstrap migrator reader writer; do
|
||||
password="smoke-${secret}-${smoke_project}"
|
||||
printf '%s' "$password" >"$secret_dir/$secret"
|
||||
@@ -31,23 +32,80 @@ export THT_VECTOR_READER_PASSWORD_SECRET_FILE="$secret_dir/reader"
|
||||
export THT_VECTOR_WRITER_PASSWORD_SECRET_FILE="$secret_dir/writer"
|
||||
export THT_VECTOR_READER_PASSWORD="smoke-reader-${smoke_project}"
|
||||
export THT_VECTOR_WRITER_PASSWORD="smoke-writer-${smoke_project}"
|
||||
export THOTH_SMOKE_OWNER="$smoke_owner"
|
||||
|
||||
compose() {
|
||||
docker compose --project-name "$smoke_project" --profile local-vector "$@"
|
||||
}
|
||||
|
||||
resource_ids() {
|
||||
case "$1" in
|
||||
container) docker ps -aq --filter "label=com.docker.compose.project=$smoke_project" ;;
|
||||
volume) docker volume ls -q --filter "label=com.docker.compose.project=$smoke_project" ;;
|
||||
network) docker network ls -q --filter "label=com.docker.compose.project=$smoke_project" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
resource_owner() {
|
||||
case "$1" in
|
||||
container) docker inspect --format '{{ index .Config.Labels "io.thothii.smoke-owner" }}' "$2" ;;
|
||||
volume) docker volume inspect --format '{{ index .Labels "io.thothii.smoke-owner" }}' "$2" ;;
|
||||
network) docker network inspect --format '{{ index .Labels "io.thothii.smoke-owner" }}' "$2" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
assert_no_collision() {
|
||||
for kind in container volume network; do
|
||||
ids=$(resource_ids "$kind")
|
||||
if [ -n "$ids" ]; then
|
||||
echo "refusing existing Compose project resources for generated namespace $smoke_project" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
verify_owned_resources() {
|
||||
for kind in container volume network; do
|
||||
for id in $(resource_ids "$kind"); do
|
||||
owner=$(resource_owner "$kind" "$id" 2>/dev/null || true)
|
||||
if [ "$owner" != "$smoke_owner" ]; then
|
||||
echo "refusing cleanup of resource not owned by this smoke: $kind $id" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
if [ "$keep_resources" = "1" ]; then
|
||||
echo "Keeping smoke resources for project $smoke_project (KEEP_SMOKE_RESOURCES=1)." >&2
|
||||
else
|
||||
compose down --volumes >/dev/null 2>&1 || true
|
||||
if verify_owned_resources; then
|
||||
compose down --volumes >/dev/null 2>&1 || true
|
||||
fi
|
||||
fi
|
||||
rm -rf "$secret_dir"
|
||||
}
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
if [ "$mode" = "--live-collision-test" ]; then
|
||||
collision_volume="${smoke_project}-collision"
|
||||
docker volume create \
|
||||
--label "com.docker.compose.project=$smoke_project" \
|
||||
--label 'io.thothii.smoke-owner=foreign-owner' \
|
||||
"$collision_volume" >/dev/null
|
||||
if assert_no_collision 2>/dev/null; then
|
||||
echo "live collision probe was not detected" >&2
|
||||
docker volume rm "$collision_volume" >/dev/null
|
||||
exit 1
|
||||
fi
|
||||
docker volume rm "$collision_volume" >/dev/null
|
||||
echo "live local-vector project collision refusal passed."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
probe_vector() {
|
||||
compose exec -T core /opt/venv/bin/python - "$marker" <<'PY'
|
||||
compose exec -T core /opt/venv/bin/python - "$marker" "$1" <<'PY'
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
@@ -58,6 +116,7 @@ from tht.ports.vector import VectorWriteRecord
|
||||
from tht.vectorstore.records import VectorRecord
|
||||
|
||||
marker = sys.argv[1]
|
||||
mode = sys.argv[2]
|
||||
database = "thoth"
|
||||
host = "vector-db"
|
||||
|
||||
@@ -85,28 +144,59 @@ record = VectorRecord(
|
||||
metadata={"smoke": True},
|
||||
)
|
||||
embedding = [1.0] + [0.0] * 767
|
||||
store.upsert(
|
||||
"memory",
|
||||
[VectorWriteRecord(record, embedding, hashlib.sha256(marker.encode()).hexdigest())],
|
||||
)
|
||||
if mode == "write":
|
||||
store.upsert(
|
||||
"memory",
|
||||
[VectorWriteRecord(record, embedding, hashlib.sha256(marker.encode()).hexdigest())],
|
||||
)
|
||||
hits = store.search(["memory"], embedding, limit=1, kinds=["memory"])
|
||||
assert hits and hits[0].id == marker, hits
|
||||
print(f"role health, upsert, and search passed for {marker}")
|
||||
print(f"role health and persisted search passed for {marker} ({mode})")
|
||||
PY
|
||||
}
|
||||
|
||||
assert_no_collision
|
||||
compose config --quiet
|
||||
services=$(compose config --services)
|
||||
printf '%s\n' "$services" | grep -qx vector-db
|
||||
printf '%s\n' "$services" | grep -qx vector-reconcile
|
||||
printf '%s\n' "$services" | grep -qx vector-migrate
|
||||
|
||||
compose up --build --wait vector-migrate core
|
||||
compose up --build --wait vector-reconcile vector-migrate core
|
||||
migration_status=$(compose run --rm --no-deps vector-migrate)
|
||||
printf '%s\n' "$migration_status" | grep -q '"pending": \[\]'
|
||||
probe_vector
|
||||
migrator_flags=$(compose run --rm --no-deps --entrypoint sh vector-reconcile -ec '
|
||||
export PGPASSWORD=$(cat /run/secrets/vector_bootstrap_password)
|
||||
psql -At --host vector-db --username postgres --dbname thoth \
|
||||
--command "SELECT (NOT rolcreaterole) AND (NOT rolcreatedb) AND (NOT rolsuper) FROM pg_roles WHERE rolname = '\''thoth_vector_migrator'\''"
|
||||
')
|
||||
test "$migrator_flags" = t
|
||||
probe_vector write
|
||||
|
||||
old_reader_password=$THT_VECTOR_READER_PASSWORD
|
||||
for secret in migrator reader writer; do
|
||||
password="rotated-${secret}-${smoke_project}"
|
||||
printf '%s' "$password" >"$secret_dir/$secret"
|
||||
done
|
||||
export THT_VECTOR_READER_PASSWORD="rotated-reader-${smoke_project}"
|
||||
export THT_VECTOR_WRITER_PASSWORD="rotated-writer-${smoke_project}"
|
||||
|
||||
compose run --rm vector-reconcile
|
||||
rotation_status=$(compose run --rm --no-deps vector-migrate)
|
||||
printf '%s\n' "$rotation_status" | grep -q '"pending": \[\]'
|
||||
if compose run --rm --no-deps --entrypoint psql \
|
||||
-e PGPASSWORD="$old_reader_password" vector-reconcile \
|
||||
--host vector-db --username thoth_vector_reader --dbname thoth --command 'SELECT 1' \
|
||||
>/dev/null 2>&1; then
|
||||
echo "old reader credential still works after rotation" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
compose up --force-recreate --no-deps --wait core
|
||||
probe_vector read
|
||||
|
||||
compose restart vector-db core
|
||||
compose up --wait vector-db core
|
||||
probe_vector
|
||||
probe_vector read
|
||||
|
||||
echo "Local pgvector migration, least-privilege roles, search, and restart persistence passed."
|
||||
echo "Local pgvector migration, credential rotation, least-privilege roles, and persistence passed."
|
||||
|
||||
Reference in New Issue
Block a user