fix(deploy): support bootstrap password rotation

.superpowers/sdd/pgvector-task-3-report.md

@@ -86,3 +86,33 @@ Follow-up verification:
 Remaining operational constraint: the bootstrap admin secret must continue to match the PostgreSQL
 bootstrap account stored in the volume. Runtime migrator/reader/writer rotation is supported without
 data deletion; bootstrap-account password rotation is a distinct database-administration operation.
+
+## Final hardening — bootstrap account rotation
+
+The remaining operational constraint is now covered by
+`scripts/vector-rotate-bootstrap-password.sh OLD_SECRET_FILE NEW_SECRET_FILE`:
+
+- It does not rely on `POSTGRES_PASSWORD_FILE` after initialization.
+- It pre-stages the deployment-file replacement in the same directory, authenticates to the live
+  database with the explicit old file, and changes only the authenticated bootstrap role.
+- Passwords are passed as connection parameters and rendered with psycopg2 SQL composition, so
+  shell and SQL metacharacters are not interpolated.
+- A second connection must authenticate with the new password before the command succeeds. If that
+  verification fails, the still-open old connection restores the old database password.
+- Only after verified database login does an atomic rename replace the current deployment secret.
+  Wrong-old authentication and verification failures leave deployment configuration unchanged.
+
+Final live smoke evidence on one existing `vector_data` volume:
+
+- wrong-old bootstrap rotation rejected; current deployment secret unchanged
+- bootstrap password with quote characters rotated successfully
+- old bootstrap login rejected and new login accepted
+- `vector-reconcile`, packaged migrations, and core health passed afterward
+- the vector record written before rotation remained searchable after rotation and after a further
+  database/core restart
+
+Final tests:
+
+- `./scripts/test-vector-bootstrap-rotation.sh`: PASS
+- `./scripts/local-vector-smoke.sh`: PASS with negative and positive live bootstrap rotation
+- existing local-vector collision/safety and Compose deployment contracts: PASS

deploy/env.example

@@ -26,6 +26,8 @@ THT_VECTOR_WRITER_USER=thoth_vector_writer
 THT_VECTOR_READER_PASSWORD=
 THT_VECTOR_WRITER_PASSWORD=
 THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE=/absolute/path/to/vector_bootstrap_password
+# Changing the file alone does not rotate an initialized DB; use
+# scripts/vector-rotate-bootstrap-password.sh OLD_SECRET_FILE NEW_SECRET_FILE.
 THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE=/absolute/path/to/vector_migrator_password
 THT_VECTOR_READER_PASSWORD_SECRET_FILE=/absolute/path/to/vector_reader_password
 THT_VECTOR_WRITER_PASSWORD_SECRET_FILE=/absolute/path/to/vector_writer_password

deploy/secrets/README.md

@@ -14,3 +14,24 @@ docker compose -f compose.yaml -f deploy/compose.production.yaml \
 
 The CA file should contain only the public PEM certificate chain. API-key files should contain
 one value with no surrounding quotes.
+
+## Rotating the initialized local-vector bootstrap password
+
+Replacing `THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE` or changing its contents does **not** rotate
+an initialized PostgreSQL cluster. Use the supported workflow against the running local-vector
+project:
+
+```sh
+./scripts/vector-rotate-bootstrap-password.sh \
+  /absolute/path/to/current-bootstrap-secret \
+  /absolute/path/to/staged-new-bootstrap-secret
+```
+
+The command authenticates using the current file, changes only the authenticated bootstrap role,
+verifies a new login, and only then atomically replaces the current deployment secret file. If old
+authentication or new-login verification fails, it exits without changing the deployment file;
+verification failure also attempts to restore the old database password over the still-open
+authenticated connection. After success, run the printed `vector-reconcile`/migration/core command.
+
+Keep the staged new file on the same trusted host, mode `0600`, and retain a secure backup until the
+post-rotation reconciliation and application health checks pass.

deploy/vector/rotate-bootstrap-password.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""Rotate the initialized PostgreSQL bootstrap role and verify before returning success."""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+import psycopg2
+from psycopg2 import sql
+
+
+def read_secret(path: str) -> str:
+    value = Path(path).read_text().rstrip("\r\n")
+    if not value or "\x00" in value:
+        raise ValueError("secret must be non-empty and contain no NUL bytes")
+    return value
+
+
+def connect(password: str):
+    return psycopg2.connect(
+        host=os.environ.get("THT_VECTOR_HOST", "vector-db"),
+        port=int(os.environ.get("THT_VECTOR_PORT", "5432")),
+        dbname=os.environ.get("THT_VECTOR_DATABASE", "thoth"),
+        user=os.environ.get("THT_VECTOR_BOOTSTRAP_USER", "postgres"),
+        password=password,
+        connect_timeout=5,
+    )
+
+
+def alter_current_role(connection, password: str) -> None:
+    with connection.cursor() as cursor:
+        cursor.execute("SELECT current_user")
+        current_user = cursor.fetchone()[0]
+        expected = os.environ.get("THT_VECTOR_BOOTSTRAP_USER", "postgres")
+        if current_user != expected:
+            raise RuntimeError("authenticated role does not match THT_VECTOR_BOOTSTRAP_USER")
+        cursor.execute(
+            sql.SQL("ALTER ROLE {} PASSWORD {}").format(
+                sql.Identifier(current_user), sql.Literal(password)
+            )
+        )
+    connection.commit()
+
+
+def main() -> int:
+    if len(sys.argv) != 3:
+        print("usage: rotate-bootstrap-password.py OLD_SECRET NEW_SECRET", file=sys.stderr)
+        return 2
+    try:
+        old_password = read_secret(sys.argv[1])
+        new_password = read_secret(sys.argv[2])
+        if old_password == new_password:
+            raise ValueError("old and new bootstrap passwords must differ")
+        old_connection = connect(old_password)
+    except Exception as exc:
+        print(f"bootstrap rotation refused before change: {type(exc).__name__}", file=sys.stderr)
+        return 1
+
+    try:
+        alter_current_role(old_connection, new_password)
+        try:
+            verification = connect(new_password)
+            verification.close()
+        except Exception as verify_exc:
+            try:
+                alter_current_role(old_connection, old_password)
+            except Exception as restore_exc:
+                print(
+                    "bootstrap rotation verification failed and password restore failed: "
+                    f"{type(verify_exc).__name__}/{type(restore_exc).__name__}",
+                    file=sys.stderr,
+                )
+                return 3
+            print(
+                f"bootstrap rotation verification failed; old password restored: "
+                f"{type(verify_exc).__name__}",
+                file=sys.stderr,
+            )
+            return 1
+    except Exception as exc:
+        print(f"bootstrap rotation failed: {type(exc).__name__}", file=sys.stderr)
+        return 1
+    finally:
+        old_connection.close()
+
+    print("bootstrap database password rotated and new login verified")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/local-vector-smoke.sh

@@ -195,8 +195,43 @@ fi
 compose up --force-recreate --no-deps --wait core
 probe_vector read
 
+old_bootstrap_password=$(cat "$secret_dir/bootstrap")
+printf '%s' "wrong-bootstrap-${smoke_project}" >"$secret_dir/bootstrap-wrong"
+printf '%s' "next-bootstrap-'quoted-${smoke_project}" >"$secret_dir/bootstrap-next"
+cp "$secret_dir/bootstrap" "$secret_dir/bootstrap-before-negative"
+if COMPOSE_PROJECT_NAME="$smoke_project" \
+    ./scripts/vector-rotate-bootstrap-password.sh \
+    "$secret_dir/bootstrap-wrong" "$secret_dir/bootstrap-next" \
+    >/dev/null 2>&1; then
+  echo "bootstrap rotation accepted the wrong old secret" >&2
+  exit 1
+fi
+cmp "$secret_dir/bootstrap" "$secret_dir/bootstrap-before-negative"
+
+COMPOSE_PROJECT_NAME="$smoke_project" \
+  ./scripts/vector-rotate-bootstrap-password.sh \
+  "$secret_dir/bootstrap" "$secret_dir/bootstrap-next"
+new_bootstrap_password=$(cat "$secret_dir/bootstrap")
+test "$new_bootstrap_password" != "$old_bootstrap_password"
+if compose run --rm --no-deps --entrypoint psql \
+    -e PGPASSWORD="$old_bootstrap_password" vector-reconcile \
+    --host vector-db --username postgres --dbname thoth --command 'SELECT 1' \
+    >/dev/null 2>&1; then
+  echo "old bootstrap credential still works after rotation" >&2
+  exit 1
+fi
+compose run --rm --no-deps --entrypoint psql \
+  -e PGPASSWORD="$new_bootstrap_password" vector-reconcile \
+  --host vector-db --username postgres --dbname thoth --command 'SELECT 1' \
+  >/dev/null
+compose run --rm vector-reconcile
+bootstrap_rotation_status=$(compose run --rm --no-deps vector-migrate)
+printf '%s\n' "$bootstrap_rotation_status" | grep -q '"pending": \[\]'
+compose up --force-recreate --no-deps --wait core
+probe_vector read
+
 compose restart vector-db core
 compose up --wait vector-db core
 probe_vector read
 
-echo "Local pgvector migration, credential rotation, least-privilege roles, and persistence passed."
+echo "Local pgvector runtime/bootstrap rotation, least-privilege roles, and persistence passed."

scripts/test-vector-bootstrap-rotation.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+set -eu
+
+cd "$(dirname "$0")/.."
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT HUP INT TERM
+
+fake="$tmp/docker"
+log="$tmp/docker.log"
+cat >"$fake" <<'SH'
+#!/bin/sh
+set -eu
+printf '%s\n' "$*" >>"$FAKE_DOCKER_LOG"
+exit "${FAKE_DOCKER_EXIT:-0}"
+SH
+chmod 0755 "$fake"
+
+printf '%s' old-password >"$tmp/old"
+printf '%s' "new-'quoted-\$-password" >"$tmp/new"
+cp "$tmp/old" "$tmp/original"
+
+if PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" FAKE_DOCKER_EXIT=1 \
+    ./scripts/vector-rotate-bootstrap-password.sh "$tmp/old" "$tmp/new" \
+    >"$tmp/out" 2>"$tmp/err"; then
+  echo "rotation unexpectedly succeeded when database verification failed" >&2
+  exit 1
+fi
+cmp "$tmp/old" "$tmp/original"
+
+: >"$log"
+PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" \
+  ./scripts/vector-rotate-bootstrap-password.sh "$tmp/old" "$tmp/new" \
+  >"$tmp/out" 2>"$tmp/err"
+cmp "$tmp/old" "$tmp/new"
+grep -q '/run/secrets/bootstrap-old:ro' "$log"
+grep -q '/run/secrets/bootstrap-new:ro' "$log"
+grep -q 'atomically replaced only after verified database login' "$tmp/out"
+
+echo "bootstrap rotation ordering and no-config-change failure contracts passed."

scripts/vector-rotate-bootstrap-password.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+set -eu
+
+cd "$(dirname "$0")/.."
+
+if [ "$#" -ne 2 ]; then
+  echo "usage: $0 OLD_SECRET_FILE NEW_SECRET_FILE" >&2
+  exit 2
+fi
+
+absolute_file() {
+  directory=$(CDPATH= cd -- "$(dirname -- "$1")" && pwd)
+  printf '%s/%s\n' "$directory" "$(basename -- "$1")"
+}
+
+old_secret=$(absolute_file "$1")
+new_secret=$(absolute_file "$2")
+for secret in "$old_secret" "$new_secret"; do
+  if [ ! -f "$secret" ] || [ ! -r "$secret" ] || [ ! -s "$secret" ]; then
+    echo "secret file must be a readable, non-empty regular file: $secret" >&2
+    exit 2
+  fi
+done
+if [ "$old_secret" -ef "$new_secret" ]; then
+  echo "old and new secret files must be distinct" >&2
+  exit 2
+fi
+
+project=${COMPOSE_PROJECT_NAME:-thothii}
+replacement=$(mktemp "${old_secret}.rotate.XXXXXX")
+trap 'rm -f "$replacement"' EXIT HUP INT TERM
+cp "$new_secret" "$replacement"
+chmod 0600 "$replacement"
+
+docker compose --project-name "$project" --profile local-vector run --rm --no-deps \
+  --user 0:0 \
+  --entrypoint /opt/venv/bin/python \
+  --volume "$old_secret:/run/secrets/bootstrap-old:ro" \
+  --volume "$new_secret:/run/secrets/bootstrap-new:ro" \
+  --volume "$(pwd)/deploy/vector/rotate-bootstrap-password.py:/opt/thoth/rotate-bootstrap-password.py:ro" \
+  core /opt/thoth/rotate-bootstrap-password.py \
+  /run/secrets/bootstrap-old /run/secrets/bootstrap-new
+
+mv -f "$replacement" "$old_secret"
+trap - EXIT HUP INT TERM
+
+echo "Deployment bootstrap secret atomically replaced only after verified database login."
+echo "Re-run: docker compose --project-name $project --profile local-vector up --wait vector-reconcile vector-migrate core"