mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(vector): close local pgvector final review

Browse Source
This commit is contained in:
marcopan
2026-07-12 02:48:10 +02:00
parent 407c4a6faf
commit 6c67235caf
16 changed files with 265 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/local-vector-smoke.sh
+18 -9
View File
@@ -32,8 +32,6 @@ export THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE="$secret_dir/bootstrap"
export THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE="$secret_dir/migrator"
export THT_VECTOR_READER_PASSWORD_SECRET_FILE="$secret_dir/reader"
export THT_VECTOR_WRITER_PASSWORD_SECRET_FILE="$secret_dir/writer"
export THT_VECTOR_READER_PASSWORD="smoke-reader-${smoke_project}"
export THT_VECTOR_WRITER_PASSWORD="smoke-writer-${smoke_project}"
export THT_VECTOR_BOOTSTRAP_USER=thoth_bootstrap_smoke
export THOTH_SMOKE_OWNER="$smoke_owner"
@@ -132,7 +130,7 @@ def credential(role: str) -> DatabaseConfig:
database=database,
schema="vectors",
user=f"thoth_vector_{role}",
password=os.environ[f"THT_VECTOR_{role.upper()}_PASSWORD"],
password=open(os.environ[f"THT_VECTOR_{role.upper()}_PASSWORD_FILE"]).read(),
)
store = PgVectorStore(credential("reader"), credential("writer"), expected_dimension=768)
@@ -168,6 +166,14 @@ printf '%s\n' "$services" | grep -qx vector-reconcile
printf '%s\n' "$services" | grep -qx vector-migrate
compose up --build --wait vector-reconcile vector-migrate core
core_id=$(compose ps -q core)
inspect_env=$(docker inspect --format '{{json .Config.Env}}' "$core_id")
if printf '%s' "$inspect_env" | grep -q "smoke-\(reader\|writer\)-${smoke_project}"; then
echo "docker inspect exposed a direct vector password" >&2
exit 1
fi
printf '%s' "$inspect_env" | grep -q 'THT_VECTOR_READER_PASSWORD_FILE=/run/secrets/vector_reader_password'
printf '%s' "$inspect_env" | grep -q 'THT_VECTOR_WRITER_PASSWORD_FILE=/run/secrets/vector_writer_password'
migration_status=$(compose run --rm --no-deps vector-migrate)
printf '%s\n' "$migration_status" | grep -q '"pending": \[\]'
migrator_flags=$(compose run --rm --no-deps --entrypoint sh vector-reconcile -ec '
@@ -178,13 +184,11 @@ migrator_flags=$(compose run --rm --no-deps --entrypoint sh vector-reconcile -ec
test "$migrator_flags" = t
probe_vector write
old_reader_password=$THT_VECTOR_READER_PASSWORD
old_reader_password=$(cat "$secret_dir/reader")
for secret in migrator reader writer; do
password="rotated-${secret}-${smoke_project}"
printf '%s' "$password" >"$secret_dir/$secret"
done
export THT_VECTOR_READER_PASSWORD="rotated-reader-${smoke_project}"
export THT_VECTOR_WRITER_PASSWORD="rotated-writer-${smoke_project}"
compose run --rm vector-reconcile
rotation_status=$(compose run --rm --no-deps vector-migrate)
@@ -205,6 +209,8 @@ printf '%s' "wrong-bootstrap-${smoke_project}" >"$secret_dir/bootstrap-wrong"
printf '%s' "next-bootstrap-'quoted-${smoke_project}" >"$secret_dir/bootstrap-next"
cp "$secret_dir/bootstrap" "$secret_dir/bootstrap-before-negative"
printf 'invalid bootstrap password\n' >"$secret_dir/bootstrap-whitespace"
chmod 0600 "$secret_dir/bootstrap-wrong" "$secret_dir/bootstrap-next" \
"$secret_dir/bootstrap-before-negative" "$secret_dir/bootstrap-whitespace"
if COMPOSE_PROJECT_NAME="$smoke_project" \
./scripts/vector-rotate-bootstrap-password.sh \
"$secret_dir/bootstrap" "$secret_dir/bootstrap-whitespace" \
@@ -344,8 +350,10 @@ if [ "$mode" = "--backup-restore" ]; then
--entrypoint /opt/thoth/reconcile-roles.sh "$image" >/dev/null
docker run --rm -i --network "$network" \
-e THT_VECTOR_READER_PASSWORD="$THT_VECTOR_READER_PASSWORD" \
-e THT_VECTOR_WRITER_PASSWORD="$THT_VECTOR_WRITER_PASSWORD" \
--mount "type=bind,source=$secret_dir/reader,target=/run/secrets/vector_reader_password,readonly" \
--mount "type=bind,source=$secret_dir/writer,target=/run/secrets/vector_writer_password,readonly" \
-e THT_VECTOR_READER_PASSWORD_FILE=/run/secrets/vector_reader_password \
-e THT_VECTOR_WRITER_PASSWORD_FILE=/run/secrets/vector_writer_password \
--entrypoint /opt/venv/bin/python thothii-core:local - "$marker" <<'PY'
import hashlib
import os
@@ -359,7 +367,8 @@ from tht.vectorstore.records import VectorRecord
def config(role):
return DatabaseConfig(
host="vector-db-restore", port=5432, database="thoth", schema="vectors",
user=f"thoth_vector_{role}", password=os.environ[f"THT_VECTOR_{role.upper()}_PASSWORD"],
user=f"thoth_vector_{role}",
password=open(os.environ[f"THT_VECTOR_{role.upper()}_PASSWORD_FILE"]).read(),
)
store = PgVectorStore(config("reader"), config("writer"), expected_dimension=768)
scripts/test-container-deployment.sh
+16
View File
@@ -15,6 +15,22 @@ if grep -q 'env_file:' "$tmp/base.yaml"; then
exit 1
fi
for secret in bootstrap migrator local_reader local_writer; do
printf '%s' "contract-$secret" >"$tmp/$secret"
chmod 0600 "$tmp/$secret"
done
THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE="$tmp/bootstrap" \
THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE="$tmp/migrator" \
THT_VECTOR_READER_PASSWORD_SECRET_FILE="$tmp/local_reader" \
THT_VECTOR_WRITER_PASSWORD_SECRET_FILE="$tmp/local_writer" \
docker compose --profile local-vector config >"$tmp/local-vector.yaml"
grep -q 'THT_VECTOR_READER_PASSWORD_FILE: /run/secrets/vector_reader_password' "$tmp/local-vector.yaml"
grep -q 'THT_VECTOR_WRITER_PASSWORD_FILE: /run/secrets/vector_writer_password' "$tmp/local-vector.yaml"
if grep -q 'contract-local_' "$tmp/local-vector.yaml"; then
echo "rendered local-vector config leaked a direct database secret" >&2
exit 1
fi
docker compose -f compose.yaml -f deploy/compose.local.yaml \
--profile external config >"$tmp/local.yaml"
grep -q 'env_file:' deploy/compose.local.yaml
scripts/test-vector-backup-restore-safety.sh
+1
View File
@@ -7,6 +7,7 @@ trap 'rm -rf "$tmp"' EXIT HUP INT TERM
fakebin="$tmp/bin"
mkdir "$fakebin"
printf '%s' secret >"$tmp/password"
chmod 0600 "$tmp/password"
cat >"$fakebin/pg_dump" <<'SH'
#!/bin/sh
scripts/test-vector-bootstrap-rotation.sh
+1
View File
@@ -20,6 +20,7 @@ printf '%s' "new-'quoted-\$-password" >"$tmp/new"
cp "$tmp/old" "$tmp/original"
printf 'invalid password\n' >"$tmp/whitespace"
chmod 0600 "$tmp/old" "$tmp/new" "$tmp/original" "$tmp/whitespace"
: >"$log"
if PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" THT_VECTOR_BOOTSTRAP_USER=custom_admin \
./scripts/vector-rotate-bootstrap-password.sh "$tmp/old" "$tmp/whitespace" \
scripts/test-vector-secret-policy.sh
+16
View File
@@ -11,6 +11,13 @@ trap 'rm -rf "$tmp"' EXIT HUP INT TERM
printf 'has newline\n' >"$tmp/newline"
printf 'has space' >"$tmp/space"
printf 'safe-quoted-\047-dollar-$' >"$tmp/valid"
printf 'docker-secret' >"$tmp/docker"
printf 'owner-readonly' >"$tmp/readonly"
printf 'too-open' >"$tmp/open"
chmod 0600 "$tmp/valid"
chmod 0444 "$tmp/docker"
chmod 0400 "$tmp/readonly"
chmod 0640 "$tmp/open"
for invalid in empty newline space; do
if validate_secret_file "$tmp/$invalid" "$invalid" >/dev/null 2>&1; then
@@ -19,6 +26,15 @@ for invalid in empty newline space; do
fi
done
validate_secret_file "$tmp/valid" valid
validate_secret_file "$tmp/readonly" readonly
if validate_secret_file "$tmp/docker" docker >/dev/null 2>&1; then
echo "secret policy accepted world-readable host secret" >&2
exit 1
fi
if validate_secret_file "$tmp/open" open >/dev/null 2>&1; then
echo "secret policy accepted group-readable host secret" >&2
exit 1
fi
test "$(read_secret_file "$tmp/valid" valid)" = "safe-quoted-'-dollar-$"
echo "shared vector secret policy contracts passed."