Merge origin/codex/portable-deployment into feat/docker-local-deploy
Unisce gli internals di Codex (secret-bundle, provider-credentials, auth upstream, security hardening, CI multiarch) mantenendo le fix portal-specific: - backend: configPath da THT_CONFIG (fix sessioni) + dataRoot di Codex; authMode 'upstream' - Docker/compose: TENUTO il mio (verificato live: omics_network+alias, env_file, pi npm-g) perche' il compose/Dockerfile/entrypoint di Codex sono accoppiati al suo modello secret-bundle (tht doctor inesistente, secret-policy.sh). Adottabile in futuro. - config.test.ts: preso Codex (superset) Verificato: tsc clean, 132/132 vitest.
This commit is contained in:
User
@@ -0,0 +1,68 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
source_env=${1:-"$root/../../harness/.env"}
|
||||
workspace=${2:-"$root/../../../tht-workspace-psd"}
|
||||
auth_file=${3:-"$HOME/.pi/agent/auth.json"}
|
||||
|
||||
value() {
|
||||
awk -F= -v key="$1" '$1 == key { sub(/^[^=]*=/, ""); sub(/[[:space:]].*$/, ""); print; exit }' "$source_env"
|
||||
}
|
||||
required() {
|
||||
result=$(value "$1")
|
||||
[ -n "$result" ] || { echo "missing $1 in local source configuration" >&2; exit 2; }
|
||||
printf '%s' "$result"
|
||||
}
|
||||
|
||||
test -f "$source_env"
|
||||
test -d "$workspace"
|
||||
test -f "$auth_file"
|
||||
ca=$(value THT_SSL_CA)
|
||||
[ -z "$ca" ] || test -f "$ca"
|
||||
model_key=$(jq -er '.zai.key' "$auth_file")
|
||||
test -n "$model_key"
|
||||
|
||||
umask 077
|
||||
mkdir -p "$root/deploy/secrets" "$root/deploy/workspaces"
|
||||
cp "$root/deploy/compose.psd-local.yaml.example" "$root/deploy/compose.psd-local.yaml"
|
||||
cp "$root/deploy/workspaces/psd.yaml.example" "$root/deploy/workspaces/psd.yaml"
|
||||
cat >"$root/.env" <<EOF
|
||||
COMPOSE_FILE=compose.yaml:deploy/compose.psd-local.yaml
|
||||
COMPOSE_PROFILES=
|
||||
THT_SECRETS_FILE=deploy/secrets/thothii.secrets
|
||||
THOTH_HTTP_PORT=8080
|
||||
AUTH_MODE=none
|
||||
THOTH_PUBLIC_EXPOSURE=false
|
||||
MAX_PI_PROCESSES=4
|
||||
PI_PROVIDER=zai
|
||||
PI_MODEL=glm-5.2
|
||||
PI_THINKING=medium
|
||||
THT_PROFILE=workstation
|
||||
THT_DB_NAME=$(required THT_DB_NAME)
|
||||
THT_DWH_REST_URL=$(required THT_DWH_REST_URL)
|
||||
THT_VEC_REST_URL=$(required THT_VEC_REST_URL)
|
||||
THT_VEC_WRITE_REST_URL=$(required THT_VEC_WRITE_REST_URL)
|
||||
THT_OLLAMA_URL=http://host.docker.internal:11434
|
||||
THT_DOCS_ROOT=/data/workspaces/psd
|
||||
THT_PSD_WORKSPACE_HOST_PATH=$workspace
|
||||
EOF
|
||||
cat >"$root/deploy/secrets/thothii.secrets" <<EOF
|
||||
THT_MODEL_API_KEY=$model_key
|
||||
THT_DWH_API_KEY=$(required THT_DWH_API_KEY)
|
||||
THT_VEC_API_KEY=$(required THT_VEC_API_KEY)
|
||||
THT_VEC_WRITE_API_KEY=$(required THT_VEC_WRITE_API_KEY)
|
||||
EOF
|
||||
if [ -n "$ca" ]; then
|
||||
cat >>"$root/deploy/compose.psd-local.yaml" <<EOF
|
||||
- type: bind
|
||||
source: $ca
|
||||
target: /run/secrets/ca-chain.pem
|
||||
read_only: true
|
||||
EOF
|
||||
printf '%s\n' 'THT_CA=/run/secrets/ca-chain.pem' >>"$root/deploy/secrets/thothii.secrets"
|
||||
else
|
||||
printf '%s\n' 'THT_CA=/etc/ssl/certs/ca-certificates.crt' >>"$root/deploy/secrets/thothii.secrets"
|
||||
fi
|
||||
chmod 600 "$root/.env" "$root/deploy/secrets/thothii.secrets"
|
||||
echo "Local PSD Docker configuration materialized without printing secret values."
|
||||
Executable
+436
@@ -0,0 +1,436 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
mode=${1:-run}
|
||||
case "$mode" in
|
||||
run|--live-collision-test|--backup-restore) ;;
|
||||
*) echo "usage: $0 [--live-collision-test|--backup-restore]" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
keep_resources=${KEEP_SMOKE_RESOURCES:-0}
|
||||
if [ "${SMOKE_PROJECT+x}" = x ]; then
|
||||
echo "SMOKE_PROJECT is not accepted; the smoke always generates an owned namespace" >&2
|
||||
exit 2
|
||||
fi
|
||||
secret_dir=$(mktemp -d "${TMPDIR:-/tmp}/thothii-vector-smoke.XXXXXX")
|
||||
suffix=$(basename "$secret_dir" | tr -cd 'a-z0-9')
|
||||
smoke_project="thothii-vector-smoke-$(date +%s)-$$-$suffix"
|
||||
smoke_owner="$smoke_project-owner"
|
||||
marker="local-vector-$smoke_project"
|
||||
restore_container="${smoke_project}-restore"
|
||||
restore_volume="${smoke_project}-restore-data"
|
||||
|
||||
bootstrap_password="smoke-bootstrap-$smoke_project"
|
||||
migrator_password="smoke-migrator-$smoke_project"
|
||||
reader_password="smoke-reader-$smoke_project"
|
||||
writer_password="smoke-writer-$smoke_project"
|
||||
bundle="$secret_dir/thothii.secrets"
|
||||
write_bundle() {
|
||||
umask 077
|
||||
{
|
||||
printf 'THT_VECTOR_BOOTSTRAP_PASSWORD=%s\n' "$bootstrap_password"
|
||||
printf 'THT_VECTOR_MIGRATOR_PASSWORD=%s\n' "$migrator_password"
|
||||
printf 'THT_VECTOR_READER_PASSWORD=%s\n' "$reader_password"
|
||||
printf 'THT_VECTOR_WRITER_PASSWORD=%s\n' "$writer_password"
|
||||
} >"$bundle"
|
||||
chmod 0600 "$bundle"
|
||||
}
|
||||
write_bundle
|
||||
export THT_SECRETS_FILE="$bundle"
|
||||
# The rotation helper has an old/new file interface; these are test-only
|
||||
# scratch files and are never mounted into a Compose service.
|
||||
printf '%s' "$bootstrap_password" >"$secret_dir/bootstrap"
|
||||
chmod 0600 "$secret_dir/bootstrap"
|
||||
export THT_VECTOR_BOOTSTRAP_USER=thoth_bootstrap_smoke
|
||||
export THOTH_SMOKE_OWNER="$smoke_owner"
|
||||
|
||||
compose() {
|
||||
docker compose -f compose.yaml -f deploy/compose.local-vector.yaml \
|
||||
--project-name "$smoke_project" --profile local-vector "$@"
|
||||
}
|
||||
|
||||
resource_ids() {
|
||||
case "$1" in
|
||||
container) docker ps -aq --filter "label=com.docker.compose.project=$smoke_project" ;;
|
||||
volume) docker volume ls -q --filter "label=com.docker.compose.project=$smoke_project" ;;
|
||||
network) docker network ls -q --filter "label=com.docker.compose.project=$smoke_project" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
resource_owner() {
|
||||
case "$1" in
|
||||
container) docker inspect --format '{{ index .Config.Labels "io.thothii.smoke-owner" }}' "$2" ;;
|
||||
volume) docker volume inspect --format '{{ index .Labels "io.thothii.smoke-owner" }}' "$2" ;;
|
||||
network) docker network inspect --format '{{ index .Labels "io.thothii.smoke-owner" }}' "$2" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
assert_no_collision() {
|
||||
for kind in container volume network; do
|
||||
ids=$(resource_ids "$kind")
|
||||
if [ -n "$ids" ]; then
|
||||
echo "refusing existing Compose project resources for generated namespace $smoke_project" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
verify_owned_resources() {
|
||||
for kind in container volume network; do
|
||||
for id in $(resource_ids "$kind"); do
|
||||
owner=$(resource_owner "$kind" "$id" 2>/dev/null || true)
|
||||
if [ "$owner" != "$smoke_owner" ]; then
|
||||
echo "refusing cleanup of resource not owned by this smoke: $kind $id" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
if [ "$keep_resources" = "1" ]; then
|
||||
echo "Keeping smoke resources for project $smoke_project (KEEP_SMOKE_RESOURCES=1)." >&2
|
||||
else
|
||||
if verify_owned_resources; then
|
||||
docker rm -f "$restore_container" >/dev/null 2>&1 || true
|
||||
docker volume rm "$restore_volume" >/dev/null 2>&1 || true
|
||||
compose down --volumes >/dev/null 2>&1 || true
|
||||
fi
|
||||
fi
|
||||
rm -rf "$secret_dir"
|
||||
}
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
if [ "$mode" = "--live-collision-test" ]; then
|
||||
collision_volume="${smoke_project}-collision"
|
||||
docker volume create \
|
||||
--label "com.docker.compose.project=$smoke_project" \
|
||||
--label 'io.thothii.smoke-owner=foreign-owner' \
|
||||
"$collision_volume" >/dev/null
|
||||
if assert_no_collision 2>/dev/null; then
|
||||
echo "live collision probe was not detected" >&2
|
||||
docker volume rm "$collision_volume" >/dev/null
|
||||
exit 1
|
||||
fi
|
||||
docker volume rm "$collision_volume" >/dev/null
|
||||
echo "live local-vector project collision refusal passed."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
probe_vector() {
|
||||
compose exec -T core sh -ec '
|
||||
. /app/docker/secret-policy.sh
|
||||
tmp=$(mktemp -d); trap "rm -rf \"$tmp\"" EXIT
|
||||
for role in READER WRITER; do
|
||||
file="$tmp/$role"
|
||||
read_bundle_secret /run/secrets/thothii.secrets "THT_VECTOR_${role}_PASSWORD" >"$file"
|
||||
export "THT_VECTOR_${role}_PASSWORD_FILE=$file"
|
||||
done
|
||||
exec /opt/venv/bin/python - "$1" "$2"
|
||||
' sh "$marker" "$1" <<'PY'
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
|
||||
from tht.adapters.vector.pgvector import PgVectorStore
|
||||
from tht.config import DatabaseConfig
|
||||
from tht.ports.vector import VectorWriteRecord
|
||||
from tht.vectorstore.records import VectorRecord
|
||||
|
||||
marker = sys.argv[1]
|
||||
mode = sys.argv[2]
|
||||
database = "thoth"
|
||||
host = "vector-db"
|
||||
|
||||
def credential(role: str) -> DatabaseConfig:
|
||||
return DatabaseConfig(
|
||||
host=host,
|
||||
port=5432,
|
||||
database=database,
|
||||
schema="vectors",
|
||||
user=f"thoth_vector_{role}",
|
||||
password=open(os.environ[f"THT_VECTOR_{role.upper()}_PASSWORD_FILE"]).read(),
|
||||
)
|
||||
|
||||
store = PgVectorStore(credential("reader"), credential("writer"), expected_dimension=768)
|
||||
health = store.health()
|
||||
assert health.ok, health
|
||||
assert health.read_reachable is True and health.write_reachable is True, health
|
||||
|
||||
record = VectorRecord(
|
||||
id=marker,
|
||||
kind="memory",
|
||||
ref=marker,
|
||||
title="Local vector persistence smoke",
|
||||
content=marker,
|
||||
metadata={"smoke": True},
|
||||
)
|
||||
embedding = [1.0] + [0.0] * 767
|
||||
if mode == "write":
|
||||
store.upsert(
|
||||
"memory",
|
||||
[VectorWriteRecord(record, embedding, hashlib.sha256(marker.encode()).hexdigest())],
|
||||
)
|
||||
hits = store.search(["memory"], embedding, limit=1, kinds=["memory"])
|
||||
assert hits and hits[0].id == marker, hits
|
||||
print(f"role health and persisted search passed for {marker} ({mode})")
|
||||
PY
|
||||
}
|
||||
|
||||
assert_no_collision
|
||||
compose config --quiet
|
||||
services=$(compose config --services)
|
||||
printf '%s\n' "$services" | grep -qx vector-db
|
||||
printf '%s\n' "$services" | grep -qx vector-reconcile
|
||||
printf '%s\n' "$services" | grep -qx vector-migrate
|
||||
|
||||
compose up --build --wait vector-reconcile vector-migrate core
|
||||
core_id=$(compose ps -q core)
|
||||
inspect_env=$(docker inspect --format '{{json .Config.Env}}' "$core_id")
|
||||
if printf '%s' "$inspect_env" | grep -q "smoke-\(reader\|writer\)-${smoke_project}"; then
|
||||
echo "docker inspect exposed a direct vector password" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s' "$inspect_env" | grep -q 'THT_SECRETS_FILE=/run/secrets/thothii.secrets'
|
||||
migration_status=$(compose run --rm --no-deps vector-migrate)
|
||||
printf '%s\n' "$migration_status" | grep -q '"pending": \[\]'
|
||||
migrator_flags=$(compose run --rm --no-deps --entrypoint sh vector-reconcile -ec '
|
||||
. /opt/thoth/secret-policy.sh
|
||||
export PGPASSWORD=$(read_bundle_secret /run/secrets/thothii.secrets THT_VECTOR_BOOTSTRAP_PASSWORD)
|
||||
psql -At --host vector-db --username "$THT_VECTOR_BOOTSTRAP_USER" --dbname thoth \
|
||||
--command "SELECT (NOT rolcreaterole) AND (NOT rolcreatedb) AND (NOT rolsuper) FROM pg_roles WHERE rolname = '\''thoth_vector_migrator'\''"
|
||||
')
|
||||
test "$migrator_flags" = t
|
||||
probe_vector write
|
||||
|
||||
old_reader_password="$reader_password"
|
||||
migrator_password="rotated-migrator-$smoke_project"
|
||||
reader_password="rotated-reader-$smoke_project"
|
||||
writer_password="rotated-writer-$smoke_project"
|
||||
write_bundle
|
||||
|
||||
compose run --rm vector-reconcile
|
||||
rotation_status=$(compose run --rm --no-deps vector-migrate)
|
||||
printf '%s\n' "$rotation_status" | grep -q '"pending": \[\]'
|
||||
if compose run --rm --no-deps --entrypoint psql \
|
||||
-e PGPASSWORD="$old_reader_password" vector-reconcile \
|
||||
--host vector-db --username thoth_vector_reader --dbname thoth --command 'SELECT 1' \
|
||||
>/dev/null 2>&1; then
|
||||
echo "old reader credential still works after rotation" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
compose up --force-recreate --no-deps --wait core
|
||||
probe_vector read
|
||||
|
||||
old_bootstrap_password="$bootstrap_password"
|
||||
printf '%s' "wrong-bootstrap-${smoke_project}" >"$secret_dir/bootstrap-wrong"
|
||||
printf '%s' "next-bootstrap-'quoted-${smoke_project}" >"$secret_dir/bootstrap-next"
|
||||
cp "$secret_dir/bootstrap" "$secret_dir/bootstrap-before-negative"
|
||||
printf 'invalid bootstrap password\n' >"$secret_dir/bootstrap-whitespace"
|
||||
chmod 0600 "$secret_dir/bootstrap-wrong" "$secret_dir/bootstrap-next" \
|
||||
"$secret_dir/bootstrap-before-negative" "$secret_dir/bootstrap-whitespace"
|
||||
if COMPOSE_PROJECT_NAME="$smoke_project" \
|
||||
./scripts/vector-rotate-bootstrap-password.sh \
|
||||
"$secret_dir/bootstrap" "$secret_dir/bootstrap-whitespace" \
|
||||
>/dev/null 2>&1; then
|
||||
echo "bootstrap rotation accepted whitespace in a secret" >&2
|
||||
exit 1
|
||||
fi
|
||||
cmp "$secret_dir/bootstrap" "$secret_dir/bootstrap-before-negative"
|
||||
compose run --rm --no-deps --entrypoint psql \
|
||||
-e PGPASSWORD="$old_bootstrap_password" vector-reconcile \
|
||||
--host vector-db --username "$THT_VECTOR_BOOTSTRAP_USER" --dbname thoth \
|
||||
--command 'SELECT 1' >/dev/null
|
||||
|
||||
if COMPOSE_PROJECT_NAME="$smoke_project" \
|
||||
./scripts/vector-rotate-bootstrap-password.sh \
|
||||
"$secret_dir/bootstrap-wrong" "$secret_dir/bootstrap-next" \
|
||||
>/dev/null 2>&1; then
|
||||
echo "bootstrap rotation accepted the wrong old secret" >&2
|
||||
exit 1
|
||||
fi
|
||||
cmp "$secret_dir/bootstrap" "$secret_dir/bootstrap-before-negative"
|
||||
|
||||
COMPOSE_PROJECT_NAME="$smoke_project" \
|
||||
./scripts/vector-rotate-bootstrap-password.sh \
|
||||
"$secret_dir/bootstrap" "$secret_dir/bootstrap-next"
|
||||
new_bootstrap_password=$(cat "$secret_dir/bootstrap")
|
||||
bootstrap_password="$new_bootstrap_password"
|
||||
write_bundle
|
||||
test "$new_bootstrap_password" != "$old_bootstrap_password"
|
||||
if compose run --rm --no-deps --entrypoint psql \
|
||||
-e PGPASSWORD="$old_bootstrap_password" vector-reconcile \
|
||||
--host vector-db --username "$THT_VECTOR_BOOTSTRAP_USER" --dbname thoth --command 'SELECT 1' \
|
||||
>/dev/null 2>&1; then
|
||||
echo "old bootstrap credential still works after rotation" >&2
|
||||
exit 1
|
||||
fi
|
||||
compose run --rm --no-deps --entrypoint psql \
|
||||
-e PGPASSWORD="$new_bootstrap_password" vector-reconcile \
|
||||
--host vector-db --username "$THT_VECTOR_BOOTSTRAP_USER" --dbname thoth --command 'SELECT 1' \
|
||||
>/dev/null
|
||||
compose run --rm vector-reconcile
|
||||
bootstrap_rotation_status=$(compose run --rm --no-deps vector-migrate)
|
||||
printf '%s\n' "$bootstrap_rotation_status" | grep -q '"pending": \[\]'
|
||||
compose up --force-recreate --no-deps --wait core
|
||||
probe_vector read
|
||||
|
||||
compose restart vector-db core
|
||||
compose up --wait vector-db core
|
||||
probe_vector read
|
||||
|
||||
if [ "$mode" = "--backup-restore" ]; then
|
||||
image=$(compose images -q vector-db)
|
||||
network="${smoke_project}_default"
|
||||
docker volume create \
|
||||
--label "com.docker.compose.project=$smoke_project" \
|
||||
--label "io.thothii.smoke-owner=$smoke_owner" "$restore_volume" >/dev/null
|
||||
docker run -d --name "$restore_container" \
|
||||
--label "com.docker.compose.project=$smoke_project" \
|
||||
--label "io.thothii.smoke-owner=$smoke_owner" \
|
||||
--network "$network" --network-alias vector-db-restore \
|
||||
--mount "type=volume,source=$restore_volume,target=/var/lib/postgresql/data" \
|
||||
--mount "type=bind,source=$bundle,target=/run/secrets/thothii.secrets,readonly" \
|
||||
--mount "type=bind,source=$(pwd)/deploy/vector/vector-db-entrypoint.sh,target=/opt/thoth/vector-db-entrypoint.sh,readonly" \
|
||||
--mount "type=bind,source=$(pwd)/deploy/vector/secret-policy.sh,target=/opt/thoth/secret-policy.sh,readonly" \
|
||||
-e POSTGRES_DB=thoth -e POSTGRES_USER="$THT_VECTOR_BOOTSTRAP_USER" \
|
||||
-e THT_SECRETS_FILE=/run/secrets/thothii.secrets \
|
||||
--entrypoint /opt/thoth/vector-db-entrypoint.sh "$image" >/dev/null
|
||||
attempts=0
|
||||
until docker exec "$restore_container" pg_isready \
|
||||
-U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth >/dev/null 2>&1; do
|
||||
attempts=$((attempts + 1))
|
||||
[ "$attempts" -lt 30 ] || { echo "restore database did not become ready" >&2; exit 1; }
|
||||
sleep 1
|
||||
done
|
||||
docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql -X \
|
||||
-U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth -v ON_ERROR_STOP=1 --command \
|
||||
"CREATE SCHEMA vectors; CREATE EXTENSION vector WITH SCHEMA vectors;
|
||||
CREATE TABLE vectors.memory (
|
||||
id bigserial PRIMARY KEY, record_key text UNIQUE NOT NULL, kind text NOT NULL,
|
||||
content_hash text NOT NULL, metadata jsonb NOT NULL,
|
||||
embedding vectors.vector(768) NOT NULL, indexed_at timestamptz NOT NULL DEFAULT now());
|
||||
INSERT INTO vectors.memory (record_key, kind, content_hash, metadata, embedding)
|
||||
VALUES ('restore-sentinel', 'memory', 'sentinel-original', '{}',
|
||||
('[' || '1,' || repeat('0,', 766) || '0]')::vectors.vector);" >/dev/null
|
||||
|
||||
docker run --rm --network "$network" \
|
||||
--mount "type=bind,source=$(pwd),target=/repo,readonly" \
|
||||
--mount "type=bind,source=$secret_dir,target=/scratch" "$image" \
|
||||
/repo/scripts/vector-backup.sh --host vector-db --database thoth \
|
||||
--user "$THT_VECTOR_BOOTSTRAP_USER" --password-file /scratch/bootstrap \
|
||||
--output /scratch/vector.dump
|
||||
|
||||
compose exec -T vector-db sh -ec '
|
||||
. /opt/thoth/secret-policy.sh
|
||||
export PGPASSWORD=$(read_bundle_secret /run/secrets/thothii.secrets THT_VECTOR_BOOTSTRAP_PASSWORD)
|
||||
psql -X -U "$POSTGRES_USER" -d thoth -v ON_ERROR_STOP=1 --command \
|
||||
"UPDATE vectors.memory SET content_hash = '\''mutated-after-backup'\'' WHERE record_key = '\''$1'\''"' \
|
||||
sh "$marker" >/dev/null
|
||||
|
||||
if docker run --rm --network "$network" \
|
||||
--mount "type=bind,source=$(pwd),target=/repo,readonly" \
|
||||
--mount "type=bind,source=$secret_dir,target=/scratch" "$image" \
|
||||
/repo/scripts/vector-restore.sh \
|
||||
--active-host vector-db --active-database thoth --active-user "$THT_VECTOR_BOOTSTRAP_USER" \
|
||||
--active-password-file /scratch/bootstrap \
|
||||
--target-host vector-db-restore --target-database thoth \
|
||||
--target-user "$THT_VECTOR_BOOTSTRAP_USER" --target-password-file /scratch/bootstrap \
|
||||
--input /scratch/vector.dump --force-nonempty >/dev/null 2>&1; then
|
||||
echo "forced restore unexpectedly succeeded without archived ACL roles" >&2
|
||||
exit 1
|
||||
fi
|
||||
sentinel=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \
|
||||
-XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \
|
||||
"SELECT content_hash FROM vectors.memory WHERE record_key='restore-sentinel'")
|
||||
test "$sentinel" = sentinel-original
|
||||
docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql -X \
|
||||
-U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth -v ON_ERROR_STOP=1 --command \
|
||||
"DROP TABLE vectors.memory; CREATE ROLE vector_reader NOLOGIN; CREATE ROLE vector_writer NOLOGIN;" \
|
||||
>/dev/null
|
||||
|
||||
docker run --rm --network "$network" \
|
||||
--mount "type=bind,source=$(pwd),target=/repo,readonly" \
|
||||
--mount "type=bind,source=$secret_dir,target=/scratch" "$image" \
|
||||
/repo/scripts/vector-restore.sh \
|
||||
--active-host vector-db --active-database thoth --active-user "$THT_VECTOR_BOOTSTRAP_USER" \
|
||||
--active-password-file /scratch/bootstrap \
|
||||
--target-host vector-db-restore --target-database thoth \
|
||||
--target-user "$THT_VECTOR_BOOTSTRAP_USER" --target-password-file /scratch/bootstrap \
|
||||
--input /scratch/vector.dump
|
||||
|
||||
docker run --rm --network "$network" \
|
||||
--mount "type=bind,source=$(pwd)/deploy/vector/reconcile-roles.sh,target=/opt/thoth/reconcile-roles.sh,readonly" \
|
||||
--mount "type=bind,source=$(pwd)/deploy/vector/secret-policy.sh,target=/opt/thoth/secret-policy.sh,readonly" \
|
||||
--mount "type=bind,source=$bundle,target=/run/secrets/thothii.secrets,readonly" \
|
||||
-e PGHOST=vector-db-restore -e PGDATABASE=thoth \
|
||||
-e PGUSER="$THT_VECTOR_BOOTSTRAP_USER" \
|
||||
-e THT_SECRETS_FILE=/run/secrets/thothii.secrets \
|
||||
-e THT_VECTOR_MIGRATOR_USER=thoth_vector_migrator \
|
||||
-e THT_VECTOR_READER_USER=thoth_vector_reader \
|
||||
-e THT_VECTOR_WRITER_USER=thoth_vector_writer \
|
||||
--entrypoint /opt/thoth/reconcile-roles.sh "$image" >/dev/null
|
||||
|
||||
compose exec -T core sh -ec '
|
||||
. /app/docker/secret-policy.sh
|
||||
tmp=$(mktemp -d); trap "rm -rf \"$tmp\"" EXIT
|
||||
for role in READER WRITER; do
|
||||
file="$tmp/$role"
|
||||
read_bundle_secret /run/secrets/thothii.secrets "THT_VECTOR_${role}_PASSWORD" >"$file"
|
||||
export "THT_VECTOR_${role}_PASSWORD_FILE=$file"
|
||||
done
|
||||
exec /opt/venv/bin/python - "$1"
|
||||
' sh "$marker" <<'PY'
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
|
||||
from tht.adapters.vector.pgvector import PgVectorStore
|
||||
from tht.config import DatabaseConfig
|
||||
from tht.ports.vector import VectorWriteRecord
|
||||
from tht.vectorstore.records import VectorRecord
|
||||
|
||||
def config(role):
|
||||
return DatabaseConfig(
|
||||
host="vector-db-restore", port=5432, database="thoth", schema="vectors",
|
||||
user=f"thoth_vector_{role}",
|
||||
password=open(os.environ[f"THT_VECTOR_{role.upper()}_PASSWORD_FILE"]).read(),
|
||||
)
|
||||
|
||||
store = PgVectorStore(config("reader"), config("writer"), expected_dimension=768)
|
||||
assert store.health().ok, store.health()
|
||||
embedding = [1.0] + [0.0] * 767
|
||||
marker = sys.argv[1]
|
||||
assert store.search(["memory"], embedding, limit=1, kinds=["memory"])[0].id == marker
|
||||
write_id = marker + "-restore-write"
|
||||
record = VectorRecord(
|
||||
id=write_id, kind="memory", ref=write_id, title="restore writer",
|
||||
content=write_id, metadata={},
|
||||
)
|
||||
store.upsert("memory", [VectorWriteRecord(record, embedding, hashlib.sha256(write_id.encode()).hexdigest())])
|
||||
assert store.existing_hashes("memory", ["memory"])[write_id]
|
||||
PY
|
||||
|
||||
restored=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \
|
||||
-XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \
|
||||
"SELECT content_hash <> 'mutated-after-backup' FROM vectors.memory WHERE record_key = '$marker'")
|
||||
test "$restored" = t
|
||||
expected_migrations=$(find harness/tht/migrations/vector -type f -name '[0-9][0-9][0-9]_*.sql' \
|
||||
-exec basename {} \; | sed 's/_.*//' | sort | paste -sd, -)
|
||||
applied_migrations=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \
|
||||
-XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \
|
||||
"SELECT string_agg(version, ',' ORDER BY version) FROM public.tht_vector_migrations")
|
||||
test "$applied_migrations" = "$expected_migrations"
|
||||
dimensions=$(docker exec -e PGPASSWORD="$new_bootstrap_password" "$restore_container" psql \
|
||||
-XAt -U "$THT_VECTOR_BOOTSTRAP_USER" -d thoth --command \
|
||||
"SELECT count(*) = 3 FROM pg_attribute a JOIN pg_class c ON c.oid=a.attrelid
|
||||
JOIN pg_namespace n ON n.oid=c.relnamespace
|
||||
WHERE n.nspname='vectors' AND a.attname='embedding' AND format_type(a.atttypid,a.atttypmod)='vectors.vector(768)'")
|
||||
test "$dimensions" = t
|
||||
echo "Transactional rollback and disposable-volume restore adapter parity passed."
|
||||
fi
|
||||
|
||||
echo "Local pgvector runtime/bootstrap rotation, least-privilege roles, and persistence passed."
|
||||
Executable
+126
@@ -0,0 +1,126 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
if [ "${1:-}" = "--cleanup-failure" ] && [ -z "${PREPROCESS_SMOKE_CHILD:-}" ]; then
|
||||
child_project="thoth-preprocess-failure-$$"
|
||||
child_tmp=$(mktemp -d "${TMPDIR:-/tmp}/thoth-preprocess-failure.XXXXXX")
|
||||
set +e
|
||||
PREPROCESS_SMOKE_CHILD=1 PREPROCESS_SMOKE_INJECT_FAILURE=1 \
|
||||
PREPROCESS_SMOKE_PROJECT="$child_project" PREPROCESS_SMOKE_TMP="$child_tmp" "$0"
|
||||
child_status=$?
|
||||
set -e
|
||||
test "$child_status" -eq 97
|
||||
test ! -e "$child_tmp"
|
||||
test -z "$(docker ps -aq --filter "label=com.docker.compose.project=$child_project")"
|
||||
test -z "$(docker volume ls -q --filter "label=com.docker.compose.project=$child_project")"
|
||||
test -z "$(docker network ls -q --filter "label=com.docker.compose.project=$child_project")"
|
||||
echo "injected preprocessing failure preserved status and cleaned every owned resource."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
tmp=${PREPROCESS_SMOKE_TMP:-$(mktemp -d "${TMPDIR:-/tmp}/thoth-preprocess.XXXXXX")}
|
||||
project=${PREPROCESS_SMOKE_PROJECT:-thoth-preprocess-$$}
|
||||
compose=""
|
||||
cleanup() {
|
||||
original_status=$?
|
||||
cleanup_failed=0
|
||||
set +e
|
||||
if [ -n "$compose" ]; then
|
||||
$compose down --volumes >/dev/null
|
||||
test "$?" -eq 0 || cleanup_failed=1
|
||||
fi
|
||||
test -z "$(docker ps -aq --filter "label=com.docker.compose.project=$project")" || cleanup_failed=1
|
||||
test -z "$(docker volume ls -q --filter "label=com.docker.compose.project=$project")" || cleanup_failed=1
|
||||
test -z "$(docker network ls -q --filter "label=com.docker.compose.project=$project")" || cleanup_failed=1
|
||||
rm -rf "$tmp"
|
||||
test ! -e "$tmp" || cleanup_failed=1
|
||||
if [ "$original_status" -ne 0 ]; then
|
||||
exit "$original_status"
|
||||
fi
|
||||
if [ "$cleanup_failed" -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 129' HUP
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
mkdir -p "$tmp/source/evidence"
|
||||
printf '%s\n' '# Evidence' 'generation one' >"$tmp/source/evidence/a.md"
|
||||
bundle="$tmp/thothii.secrets"
|
||||
{
|
||||
printf 'THT_VECTOR_BOOTSTRAP_PASSWORD=smoke-bootstrap-%s\n' "$project"
|
||||
printf 'THT_VECTOR_MIGRATOR_PASSWORD=smoke-migrator-%s\n' "$project"
|
||||
printf 'THT_VECTOR_READER_PASSWORD=smoke-reader-%s\n' "$project"
|
||||
printf 'THT_VECTOR_WRITER_PASSWORD=smoke-writer-%s\n' "$project"
|
||||
} >"$bundle"
|
||||
chmod 0600 "$bundle"
|
||||
export THT_SECRETS_FILE="$bundle"
|
||||
export THT_OLLAMA_URL=http://mock-embeddings:8081
|
||||
|
||||
cat >"$tmp/smoke.yaml" <<YAML
|
||||
services:
|
||||
mock-embeddings:
|
||||
image: thothii-core:local
|
||||
profiles: [preprocess]
|
||||
entrypoint: [/opt/venv/bin/python, -c]
|
||||
command:
|
||||
- |
|
||||
import json
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
class H(BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
n=len(json.loads(self.rfile.read(int(self.headers['Content-Length'])))['input'])
|
||||
body=json.dumps({'embeddings': [[1.0]+[0.0]*767 for _ in range(n)]}).encode()
|
||||
self.send_response(200); self.send_header('Content-Length', str(len(body))); self.end_headers(); self.wfile.write(body)
|
||||
def log_message(self, *args): pass
|
||||
HTTPServer(('0.0.0.0',8081),H).serve_forever()
|
||||
preprocess-evidence:
|
||||
volumes:
|
||||
- $tmp/source:/data/source:ro
|
||||
depends_on:
|
||||
mock-embeddings: {condition: service_started}
|
||||
YAML
|
||||
|
||||
compose="docker compose -f compose.yaml -f deploy/compose.local-vector.yaml -f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml -f $tmp/smoke.yaml --project-name $project --profile local-vector --profile preprocess"
|
||||
$compose build preprocess-evidence
|
||||
if [ "${PREPROCESS_SMOKE_INJECT_FAILURE:-0}" = "1" ]; then
|
||||
sh -c 'exit 97'
|
||||
fi
|
||||
generation_count() {
|
||||
$compose run --rm --no-deps --entrypoint /opt/venv/bin/python preprocess-evidence -c \
|
||||
'import pathlib,re; root=pathlib.Path("/data/workspaces/preprocess-evidence/corpus"); print(sum(1 for p in root.iterdir() if p.is_dir() and re.fullmatch(r"gen-[0-9a-f]{32}",p.name)) if root.exists() else 0)'
|
||||
}
|
||||
before=0
|
||||
first=$($compose run --rm preprocess-evidence)
|
||||
after_first=$(generation_count)
|
||||
second=$($compose run --rm preprocess-evidence)
|
||||
after_second=$(generation_count)
|
||||
printf '%s' 'generation two' >>"$tmp/source/evidence/a.md"
|
||||
third=$($compose run --rm preprocess-evidence)
|
||||
after_third=$(generation_count)
|
||||
dwh=$($compose run --rm preprocess-dwh)
|
||||
python3 - "$first" "$second" "$third" "$before" "$after_first" "$after_second" "$after_third" <<'PY'
|
||||
import json, sys
|
||||
a, b, c = map(json.loads, sys.argv[1:4])
|
||||
before, first_count, second_count, third_count = map(int, sys.argv[4:])
|
||||
assert len(a["changed"]) == 1 and not a["unchanged"]
|
||||
assert len(b["unchanged"]) == 1 and not b["changed"]
|
||||
assert len(c["changed"]) == 1 and c["generation"] != a["generation"]
|
||||
assert a["published"] and not b["published"] and c["published"]
|
||||
assert first_count == before + 1
|
||||
assert second_count == first_count
|
||||
assert third_count == second_count + 1
|
||||
PY
|
||||
python3 - "$dwh" <<'PY'
|
||||
import json, sys
|
||||
assert json.loads(sys.argv[1])["status"] == "succeeded"
|
||||
PY
|
||||
active=$($compose run --rm --no-deps --entrypoint sh preprocess-evidence -c \
|
||||
'cat /data/workspaces/preprocess-evidence/corpus/ACTIVE')
|
||||
python3 - "$third" "$active" <<'PY'
|
||||
import json, sys
|
||||
assert json.loads(sys.argv[1])["generation"] == sys.argv[2].strip()
|
||||
PY
|
||||
echo "real Compose preprocessing unchanged rerun, mutation, DWH job, and ACTIVE publish passed."
|
||||
Executable
+23
@@ -0,0 +1,23 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
policy=frontend/src/api/backend-url-policy.json
|
||||
corpus=frontend/src/api/backend-url-cases.json
|
||||
|
||||
jq -c '.[]' "$corpus" | while IFS= read -r case_json; do
|
||||
value=$(printf '%s' "$case_json" | jq -r '.value')
|
||||
valid=$(printf '%s' "$case_json" | jq -r '.valid')
|
||||
if BACKEND_URL_POLICY_FILE="$policy" ./docker/validate-backend-url.sh "$value"; then
|
||||
actual=true
|
||||
else
|
||||
actual=false
|
||||
fi
|
||||
if [ "$actual" != "$valid" ]; then
|
||||
echo "shell policy mismatch for BACKEND_BASE_URL=$value: expected $valid" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "shell canonical URL corpus: ok"
|
||||
Executable
+112
@@ -0,0 +1,112 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT TERM
|
||||
|
||||
bundle="$tmp/thothii.secrets"
|
||||
cat >"$bundle" <<'EOF'
|
||||
# disposable deployment-contract bundle
|
||||
THT_MODEL_API_KEY=test-model
|
||||
THT_VECTOR_BOOTSTRAP_PASSWORD=contract-bootstrap
|
||||
THT_VECTOR_MIGRATOR_PASSWORD=contract-migrator
|
||||
THT_VECTOR_READER_PASSWORD=contract-reader
|
||||
THT_VECTOR_WRITER_PASSWORD=contract-writer
|
||||
EOF
|
||||
chmod 0600 "$bundle"
|
||||
export THT_SECRETS_FILE="$bundle"
|
||||
|
||||
docker compose config >"$tmp/base.yaml"
|
||||
grep -q '^ core:' "$tmp/base.yaml"
|
||||
grep -q '^ frontend:' "$tmp/base.yaml"
|
||||
grep -q 'host_ip: 127.0.0.1' "$tmp/base.yaml"
|
||||
grep -q 'AUTH_MODE: none' "$tmp/base.yaml"
|
||||
grep -q 'THOTH_PUBLIC_EXPOSURE: "false"' "$tmp/base.yaml"
|
||||
grep -q 'THT_SECRETS_FILE: /run/secrets/thothii.secrets' "$tmp/base.yaml"
|
||||
grep -q 'target: /home/thoth/.pi/agent/models.json' "$tmp/base.yaml"
|
||||
grep -q 'source: .*/deploy/pi/models.json' "$tmp/base.yaml"
|
||||
grep -q 'target: /home/thoth/.pi/agent/settings.json' "$tmp/base.yaml"
|
||||
if grep -q 'THT_[A-Z0-9_]*_SECRET_FILE:' "$tmp/base.yaml"; then
|
||||
echo "base Compose must not require legacy secret-file variables" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker compose -f compose.yaml -f deploy/compose.local-vector.yaml \
|
||||
--profile local-vector config >"$tmp/local-vector.yaml"
|
||||
grep -q 'target: thothii.secrets' "$tmp/local-vector.yaml"
|
||||
if grep -Eq 'vector_(bootstrap|migrator|reader|writer)_password|THT_[A-Z0-9_]+_SECRET_FILE' "$tmp/local-vector.yaml"; then
|
||||
echo "rendered local-vector config contains legacy per-secret references" >&2
|
||||
exit 1
|
||||
fi
|
||||
if grep -q 'contract-' "$tmp/local-vector.yaml"; then
|
||||
echo "rendered local-vector config leaked a bundle secret value" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker compose -f compose.yaml -f deploy/compose.local.yaml \
|
||||
config >"$tmp/local.yaml"
|
||||
if grep -q 'env_file:' "$tmp/local.yaml"; then
|
||||
echo "local Compose must use the root .env interpolation file" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf '%s\n' 'THT_MODEL_API_KEY=test-model' >"$tmp/thothii.secrets"
|
||||
chmod 0600 "$tmp/thothii.secrets"
|
||||
THT_SECRETS_FILE="$tmp/thothii.secrets" \
|
||||
THT_DB_NAME=test THT_DWH_REST_URL=https://dwh.example.test \
|
||||
THT_VEC_REST_URL=https://vector.example.test THT_OLLAMA_URL=https://embed.example.test \
|
||||
docker compose -f compose.yaml -f deploy/compose.production.yaml \
|
||||
config >"$tmp/production.yaml"
|
||||
grep -q 'AUTH_MODE: upstream' "$tmp/production.yaml"
|
||||
grep -q 'THOTH_PUBLIC_EXPOSURE: "true"' "$tmp/production.yaml"
|
||||
grep -q 'THT_SECRETS_FILE: /run/secrets/thothii.secrets' "$tmp/production.yaml"
|
||||
grep -q 'target: thothii.secrets' "$tmp/production.yaml"
|
||||
if grep -q 'test-model' "$tmp/production.yaml"; then
|
||||
echo "rendered production config leaked the model API key" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if PI_PROVIDER_API_KEY='must-not-leak' ./docker/core-entrypoint.sh doctor 2>"$tmp/legacy-model.err"; then
|
||||
echo "legacy generic model credential was accepted" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q 'PI_PROVIDER_API_KEY is unsupported' "$tmp/legacy-model.err"
|
||||
if grep -q 'must-not-leak' "$tmp/legacy-model.err"; then
|
||||
echo "legacy model credential leaked through entrypoint diagnostics" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf 'THT_VECTOR_READER_PASSWORD=one\nTHT_VECTOR_READER_PASSWORD=two\n' >"$tmp/invalid-bundle"
|
||||
chmod 0600 "$tmp/invalid-bundle"
|
||||
if THT_SECRETS_FILE="$tmp/invalid-bundle" ./docker/core-entrypoint.sh doctor \
|
||||
>"$tmp/invalid-bundle.out" 2>"$tmp/invalid-bundle.err"; then
|
||||
echo "entrypoint accepted an invalid secret bundle" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q 'THT_SECRETS_FILE points to an invalid secret bundle' "$tmp/invalid-bundle.err"
|
||||
if grep -q 'THT_VECTOR_READER_PASSWORD' "$tmp/invalid-bundle.err"; then
|
||||
echo "invalid bundle diagnostics leaked key material" >&2
|
||||
exit 1
|
||||
fi
|
||||
before_tmp=$(find "${TMPDIR:-/tmp}" -maxdepth 1 -type d -name 'thothii-secrets.*' -print | sort)
|
||||
THT_SECRETS_FILE="$bundle" ./docker/core-entrypoint.sh doctor >/dev/null 2>&1 || true
|
||||
after_tmp=$(find "${TMPDIR:-/tmp}" -maxdepth 1 -type d -name 'thothii-secrets.*' -print | sort)
|
||||
test "$before_tmp" = "$after_tmp"
|
||||
if grep -Eq 'THT_VECTOR_(BOOTSTRAP|MIGRATOR|READER|WRITER)_PASSWORD_FILE|target: vector_(bootstrap|migrator|reader|writer)_password|dwh_api_key|model_api_key|THT_[A-Z0-9_]+_SECRET_FILE' "$tmp/production.yaml"; then
|
||||
echo "production external config contains local direct vector secrets" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if awk '/^FROM / && $2 !~ /@sha256:/ { found=1 } END { exit !found }' \
|
||||
docker/core.Dockerfile docker/frontend.Dockerfile; then
|
||||
echo "every Dockerfile base must include an immutable digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
grep -qx 'deploy/\*' .dockerignore
|
||||
grep -qx '!deploy/vector/' .dockerignore
|
||||
grep -qx 'deploy/vector/\*' .dockerignore
|
||||
grep -qx '!deploy/vector/secret-policy.sh' .dockerignore
|
||||
|
||||
echo "container deployment security contract passed."
|
||||
Executable
+43
@@ -0,0 +1,43 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
test -f .env.example
|
||||
test -f deploy/secrets/thothii.secrets.example
|
||||
grep -q '^docker compose up --build -d$' docs/installazione-docker-4-contesti.md
|
||||
if grep -q 'cp deploy/env.example deploy/.env\|THT_[A-Z0-9_]*_SECRET_FILE=' docs/installazione-docker-4-contesti.md; then
|
||||
echo "installation guide still presents the legacy per-file secret setup" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT TERM
|
||||
|
||||
mkdir -p "$tmp/deploy/secrets" "$tmp/deploy/workspaces"
|
||||
cp compose.yaml "$tmp/compose.yaml"
|
||||
cp .env.example "$tmp/.env"
|
||||
cp deploy/secrets/thothii.secrets.example "$tmp/deploy/secrets/thothii.secrets"
|
||||
printf '%s\n' 'THT_MODEL_API_KEY=example-secret' >>"$tmp/deploy/secrets/thothii.secrets"
|
||||
chmod 0600 "$tmp/deploy/secrets/thothii.secrets"
|
||||
|
||||
services=$(docker compose --project-directory "$tmp" config --services)
|
||||
[ "$services" = "core
|
||||
frontend" ] || {
|
||||
echo "default Compose services must be core and frontend (got: $services)" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
rendered=$(docker compose --project-directory "$tmp" config)
|
||||
printf '%s\n' "$rendered" | grep -q 'target: thothii.secrets'
|
||||
if printf '%s\n' "$rendered" | grep -Eq 'dwh_api_key|vector_reader_api_key|vector_writer_api_key|model_api_key|thoth_ca'; then
|
||||
echo "default Compose must not declare legacy per-secret mounts" >&2
|
||||
exit 1
|
||||
fi
|
||||
if printf '%s\n' "$rendered" | grep -Eq 'THT_[A-Z0-9_]+_SECRET_FILE:'; then
|
||||
echo "default Compose must not require legacy secret-file variables" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s\n' "$rendered" | grep -q 'THT_SECRETS_FILE: /run/secrets/thothii.secrets'
|
||||
|
||||
echo "default Compose contract passed."
|
||||
Executable
+86
@@ -0,0 +1,86 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT TERM
|
||||
log="$tmp/docker.log"
|
||||
marker_file="$tmp/marker"
|
||||
|
||||
mkdir -p "$tmp/bin"
|
||||
cat >"$tmp/bin/docker" <<'EOF'
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
printf '%s\n' "$*" >>"$FAKE_DOCKER_LOG"
|
||||
|
||||
case " $* " in
|
||||
*" port frontend 8080 "*) printf '%s\n' '127.0.0.1:49152' ;;
|
||||
*" exec -T core sh -c "*"printf"*)
|
||||
for last do :; done
|
||||
printf '%s\n' "$last" >"$FAKE_MARKER_FILE"
|
||||
;;
|
||||
*" exec -T core sh -c "*"cat /data/.compose-smoke-marker"*)
|
||||
cat "$FAKE_MARKER_FILE"
|
||||
;;
|
||||
esac
|
||||
EOF
|
||||
cat >"$tmp/bin/curl" <<'EOF'
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
header_file=""
|
||||
for arg do
|
||||
if [ "${previous:-}" = "--dump-header" ]; then header_file=$arg; fi
|
||||
previous=$arg
|
||||
done
|
||||
if [ -n "$header_file" ]; then
|
||||
case "$*" in
|
||||
*"/events"*) printf 'HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nCache-Control: no-cache\r\n\r\n' >"$header_file" ;;
|
||||
*) printf 'HTTP/1.1 200 OK\r\nContent-Type: application/json; charset=utf-8\r\n\r\n' >"$header_file" ;;
|
||||
esac
|
||||
fi
|
||||
case "$*" in
|
||||
*"/health"*) printf '%s\n' '{"status":"ok"}' ;;
|
||||
esac
|
||||
EOF
|
||||
chmod +x "$tmp/bin/docker" "$tmp/bin/curl"
|
||||
|
||||
run_smoke() {
|
||||
PATH="$tmp/bin:$PATH" \
|
||||
FAKE_DOCKER_LOG="$log" \
|
||||
FAKE_MARKER_FILE="$marker_file" \
|
||||
SMOKE_PROJECT="$1" \
|
||||
KEEP_SMOKE_RESOURCES="${2:-0}" \
|
||||
./scripts/docker-smoke.sh
|
||||
}
|
||||
|
||||
run_smoke thothii-smoke-dynamic
|
||||
|
||||
while IFS= read -r invocation; do
|
||||
case "$invocation" in
|
||||
"compose --project-name thothii-smoke-dynamic "*) ;;
|
||||
*) echo "Compose invocation escaped the smoke project: $invocation" >&2; exit 1 ;;
|
||||
esac
|
||||
done <"$log"
|
||||
grep -q ' down --volumes$' "$log"
|
||||
if grep -q -- '--remove-orphans' "$log"; then
|
||||
echo "smoke cleanup must not remove operator orphans" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
: >"$log"
|
||||
run_smoke thothii-smoke-kept 1
|
||||
if grep -q ' down ' "$log"; then
|
||||
echo "KEEP_SMOKE_RESOURCES=1 unexpectedly cleaned the project" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
: >"$log"
|
||||
if PATH="$tmp/bin:$PATH" FAKE_DOCKER_LOG="$log" FAKE_MARKER_FILE="$marker_file" \
|
||||
SMOKE_PROJECT=thothii ./scripts/docker-smoke.sh >/dev/null 2>&1; then
|
||||
echo "reserved operator project was accepted" >&2
|
||||
exit 1
|
||||
fi
|
||||
test ! -s "$log"
|
||||
|
||||
echo "docker-smoke dynamic isolation contract passed."
|
||||
Executable
+23
@@ -0,0 +1,23 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
project="thothii-external-lifecycle-$$"
|
||||
cleanup() { docker compose --project-name "$project" --profile external down --volumes >/dev/null 2>&1 || true; }
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
unset THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE
|
||||
unset THT_VECTOR_READER_PASSWORD_SECRET_FILE THT_VECTOR_WRITER_PASSWORD_SECRET_FILE
|
||||
rendered=$(docker compose --project-name "$project" --profile external config)
|
||||
if printf '%s' "$rendered" | grep -q 'THT_VECTOR_.*PASSWORD_FILE\|vector_.*password'; then
|
||||
echo "external config contains local vector secret references" >&2
|
||||
exit 1
|
||||
fi
|
||||
docker compose --project-name "$project" --profile external up --build --wait core
|
||||
core=$(docker compose --project-name "$project" --profile external ps -q core)
|
||||
inspect=$(docker inspect "$core")
|
||||
if printf '%s' "$inspect" | grep -q 'THT_VECTOR_.*PASSWORD_FILE\|/run/secrets/vector_.*password'; then
|
||||
echo "external core inspect contains local vector secret references" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "external core lifecycle without local vector secrets passed."
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
./scripts/local-vector-smoke.sh --live-collision-test
|
||||
Executable
+71
@@ -0,0 +1,71 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT TERM
|
||||
|
||||
fake="$tmp/docker"
|
||||
log="$tmp/docker.log"
|
||||
state="$tmp/state"
|
||||
|
||||
cat >"$fake" <<'SH'
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
printf '%s\n' "$*" >>"$FAKE_DOCKER_LOG"
|
||||
|
||||
if [ "${FAKE_COLLISION:-0}" = 1 ] && [ "$1 $2" = "ps -aq" ]; then
|
||||
printf '%s\n' collision-container
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$1 $2" = "ps -aq" ] || [ "$1 $2" = "volume ls" ] || [ "$1 $2" = "network ls" ]; then
|
||||
if [ "${FAKE_MISMATCH_ON_CLEANUP:-0}" = 1 ] && [ -f "$FAKE_DOCKER_STATE" ]; then
|
||||
printf '%s\n' foreign-resource
|
||||
fi
|
||||
: >"$FAKE_DOCKER_STATE"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$1" = inspect ] || [ "$1 $2" = "volume inspect" ] || [ "$1 $2" = "network inspect" ]; then
|
||||
printf '%s\n' foreign-owner
|
||||
exit 0
|
||||
fi
|
||||
|
||||
case "$*" in
|
||||
*"config --services"*) printf '%s\n' vector-db vector-reconcile vector-migrate core frontend ;;
|
||||
*"run --rm --no-deps vector-migrate"*) printf '%s\n' '{"applied":["001","002","003"],"drifted":[],"pending":[]}' ;;
|
||||
esac
|
||||
exit 0
|
||||
SH
|
||||
chmod 0755 "$fake"
|
||||
|
||||
if PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" FAKE_DOCKER_STATE="$state" \
|
||||
SMOKE_PROJECT=operator-owned ./scripts/local-vector-smoke.sh >"$tmp/out" 2>"$tmp/err"; then
|
||||
echo "smoke accepted caller-controlled SMOKE_PROJECT" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q 'SMOKE_PROJECT is not accepted' "$tmp/err"
|
||||
test ! -s "$log"
|
||||
|
||||
: >"$log"
|
||||
rm -f "$state"
|
||||
PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" FAKE_DOCKER_STATE="$state" \
|
||||
FAKE_COLLISION=1 ./scripts/local-vector-smoke.sh >"$tmp/out" 2>"$tmp/err" || true
|
||||
grep -q 'refusing existing Compose project resources' "$tmp/err"
|
||||
if grep -q 'compose.*up' "$log"; then
|
||||
echo "smoke started after detecting a project collision" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
: >"$log"
|
||||
rm -f "$state"
|
||||
PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" FAKE_DOCKER_STATE="$state" \
|
||||
FAKE_MISMATCH_ON_CLEANUP=1 ./scripts/local-vector-smoke.sh >"$tmp/out" 2>"$tmp/err" || true
|
||||
grep -q 'refusing cleanup of resource not owned by this smoke' "$tmp/err"
|
||||
if grep -q 'down --volumes' "$log"; then
|
||||
echo "smoke removed resources after ownership mismatch" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "local-vector smoke collision and cleanup ownership contracts passed."
|
||||
Executable
+84
@@ -0,0 +1,84 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
tmp_bundle=$(mktemp)
|
||||
trap 'rm -f "$tmp_bundle"' EXIT HUP INT TERM
|
||||
cat >"$tmp_bundle" <<'EOF'
|
||||
THT_VECTOR_BOOTSTRAP_PASSWORD=test-bootstrap
|
||||
THT_VECTOR_MIGRATOR_PASSWORD=test-migrator
|
||||
THT_VECTOR_READER_PASSWORD=test-reader
|
||||
THT_VECTOR_WRITER_PASSWORD=test-writer
|
||||
EOF
|
||||
chmod 0600 "$tmp_bundle"
|
||||
export THT_SECRETS_FILE="$tmp_bundle"
|
||||
|
||||
local_files="-f compose.yaml -f deploy/compose.local-vector.yaml -f deploy/compose.preprocess.yaml -f deploy/compose.preprocess-local-vector.yaml"
|
||||
local_json=$(docker compose $local_files --profile local-vector --profile preprocess config --format json)
|
||||
|
||||
printf '%s' "$local_json" | python3 -c '
|
||||
import json, sys
|
||||
|
||||
config = json.load(sys.stdin)
|
||||
services = config["services"]
|
||||
assert "thothii_secrets" in config.get("secrets", {}), config.get("secrets")
|
||||
assert "vector_bootstrap_password" not in config.get("secrets", {})
|
||||
assert "vector_migrator_password" not in config.get("secrets", {})
|
||||
assert "vector_reader_password" not in config.get("secrets", {})
|
||||
assert "vector_writer_password" not in config.get("secrets", {})
|
||||
for name, service in services.items():
|
||||
if name.startswith("vector-") or name.startswith("preprocess-") or name == "core":
|
||||
assert any(item.get("target") == "thothii.secrets" for item in service.get("secrets", []) if isinstance(item, dict)), (name, service.get("secrets"))
|
||||
assert "vector_reader_password" not in str(service)
|
||||
assert "vector_writer_password" not in str(service)
|
||||
for name in ("preprocess-evidence", "preprocess-dwh"):
|
||||
dependency = services[name].get("depends_on", {}).get("vector-migrate")
|
||||
assert dependency is not None, f"{name} does not depend on vector-migrate"
|
||||
assert dependency["condition"] == "service_completed_successfully", dependency
|
||||
'
|
||||
|
||||
external_json=$(docker compose \
|
||||
-f compose.yaml -f deploy/compose.preprocess.yaml \
|
||||
--profile preprocess config --format json)
|
||||
|
||||
printf '%s' "$external_json" | python3 -c '
|
||||
import json, sys
|
||||
|
||||
config = json.load(sys.stdin)
|
||||
services = config["services"]
|
||||
assert "vector-db" not in services
|
||||
assert "vector-migrate" not in services
|
||||
assert "vector-reconcile" not in services
|
||||
for name in ("preprocess-evidence", "preprocess-dwh"):
|
||||
service = services[name]
|
||||
assert "depends_on" not in service
|
||||
assert all(item.get("target") == "thothii.secrets" for item in service.get("secrets", []) if isinstance(item, dict)), service.get("secrets")
|
||||
assert "vector_reader_password" not in str(service)
|
||||
assert "vector_writer_password" not in str(service)
|
||||
'
|
||||
|
||||
python3 - <<'PY'
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
os.environ.update({
|
||||
"THT_DB_NAME": "thoth",
|
||||
"THT_DWH_REST_URL": "http://dwh.invalid",
|
||||
"THT_DWH_API_KEY": "dwh",
|
||||
"THT_VECTOR_DATABASE": "thoth",
|
||||
"THT_VECTOR_READER_USER": "reader",
|
||||
"THT_VECTOR_WRITER_USER": "writer",
|
||||
"THT_VECTOR_READER_PASSWORD_FILE": "/tmp/generated-reader",
|
||||
"THT_VECTOR_WRITER_PASSWORD_FILE": "/tmp/generated-writer",
|
||||
"THT_DOCS_ROOT": "/data/source",
|
||||
"THT_OLLAMA_URL": "http://ollama.invalid",
|
||||
})
|
||||
text = Path("deploy/workspaces/local-vector.yaml").read_text()
|
||||
assert "password_file: ${THT_VECTOR_READER_PASSWORD_FILE}" in text
|
||||
assert "password_file: ${THT_VECTOR_WRITER_PASSWORD_FILE}" in text
|
||||
assert "${THT_SECRETS_FILE}" not in text
|
||||
print("local-vector workspace resolution contract: ok")
|
||||
PY
|
||||
|
||||
echo "preprocess compose config: ok"
|
||||
Executable
+106
@@ -0,0 +1,106 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT TERM
|
||||
fakebin="$tmp/bin"
|
||||
mkdir "$fakebin"
|
||||
printf '%s' secret >"$tmp/password"
|
||||
chmod 0600 "$tmp/password"
|
||||
|
||||
cat >"$fakebin/pg_dump" <<'SH'
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
for arg in "$@"; do case "$arg" in --file=*) output=${arg#--file=} ;; esac; done
|
||||
printf 'custom dump' >"$output"
|
||||
if [ -n "${RACE_OUTPUT:-}" ]; then
|
||||
printf 'concurrent owner' >"$RACE_OUTPUT"
|
||||
fi
|
||||
SH
|
||||
chmod 0755 "$fakebin/pg_dump"
|
||||
|
||||
victim="$tmp/victim"
|
||||
output="$tmp/vector.dump"
|
||||
printf 'sentinel' >"$victim"
|
||||
ln -s "$victim" "$output.partial"
|
||||
PATH="$fakebin:$PATH" ./scripts/vector-backup.sh --host source --database thoth --user admin \
|
||||
--password-file "$tmp/password" --output "$output" >/dev/null
|
||||
test "$(cat "$victim")" = sentinel
|
||||
test "$(cat "$output")" = 'custom dump'
|
||||
test -L "$output.partial"
|
||||
|
||||
race_output="$tmp/raced.dump"
|
||||
if PATH="$fakebin:$PATH" RACE_OUTPUT="$race_output" ./scripts/vector-backup.sh \
|
||||
--host source --database thoth --user admin --password-file "$tmp/password" \
|
||||
--output "$race_output" >"$tmp/race.out" 2>"$tmp/race.err"; then
|
||||
echo "backup replaced a destination created concurrently" >&2
|
||||
exit 1
|
||||
fi
|
||||
test "$(cat "$race_output")" = 'concurrent owner'
|
||||
if find "$tmp" -name '.raced.dump.tmp.*' -print | grep -q .; then
|
||||
echo "backup left its owned temporary archive after publication failure" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cat >"$fakebin/psql" <<'SH'
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
case "$*" in
|
||||
*pg_control_system*)
|
||||
echo same-cluster ;;
|
||||
*) echo 0 ;;
|
||||
esac
|
||||
SH
|
||||
cat >"$fakebin/pg_restore" <<'SH'
|
||||
#!/bin/sh
|
||||
printf '%s\n' "$*" >"$RESTORE_LOG"
|
||||
SH
|
||||
chmod 0755 "$fakebin/psql" "$fakebin/pg_restore"
|
||||
printf 'archive' >"$tmp/input"
|
||||
if PATH="$fakebin:$PATH" RESTORE_LOG="$tmp/restore.log" ./scripts/vector-restore.sh \
|
||||
--active-host source --active-database active --active-user admin \
|
||||
--active-password-file "$tmp/password" --target-host target --target-database restore \
|
||||
--target-user admin --target-password-file "$tmp/password" --input "$tmp/input" \
|
||||
>"$tmp/out" 2>"$tmp/err"; then
|
||||
echo "restore accepted a target on the active PostgreSQL cluster" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q 'same PostgreSQL cluster' "$tmp/err"
|
||||
test ! -e "$tmp/restore.log"
|
||||
|
||||
cat >"$fakebin/psql" <<'SH'
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
case "$*" in
|
||||
*pg_control_system*)
|
||||
case "$*" in *--host=source*) echo same-cluster ;; *) echo other-cluster ;; esac ;;
|
||||
*) echo 0 ;;
|
||||
esac
|
||||
SH
|
||||
chmod 0755 "$fakebin/psql"
|
||||
PATH="$fakebin:$PATH" RESTORE_LOG="$tmp/restore.log" ./scripts/vector-restore.sh \
|
||||
--active-host source --active-database active --active-user admin \
|
||||
--active-password-file "$tmp/password" --target-host target --target-database restore \
|
||||
--target-user admin --target-password-file "$tmp/password" --input "$tmp/input" >/dev/null
|
||||
grep -q -- '--single-transaction' "$tmp/restore.log"
|
||||
grep -q -- '--exit-on-error' "$tmp/restore.log"
|
||||
|
||||
# The live restore smoke must follow the packaged migration set instead of a stale
|
||||
# hard-coded count when a new migration is added.
|
||||
if grep -Eq 'vector_(bootstrap|migrator|reader|writer)_password' \
|
||||
deploy/compose.local-vector.yaml deploy/compose.preprocess-local-vector.yaml; then
|
||||
echo "local-vector Compose still declares legacy per-password secrets" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq 'thothii_secrets' deploy/compose.local-vector.yaml
|
||||
grep -Fq 'thothii_secrets' deploy/compose.preprocess-local-vector.yaml
|
||||
if grep -Fq 'SELECT count(*) = 3 FROM public.tht_vector_migrations' \
|
||||
scripts/local-vector-smoke.sh; then
|
||||
echo "local vector smoke hard-codes the pre-004 migration count" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq 'expected_migrations=' scripts/local-vector-smoke.sh
|
||||
grep -Fq 'applied_migrations=' scripts/local-vector-smoke.sh
|
||||
|
||||
echo "vector backup/restore filesystem, identity, and transaction contracts passed."
|
||||
Executable
+56
@@ -0,0 +1,56 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT TERM
|
||||
|
||||
fake="$tmp/docker"
|
||||
log="$tmp/docker.log"
|
||||
cat >"$fake" <<'SH'
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
printf '%s:%s\n' "${THT_VECTOR_BOOTSTRAP_USER:-unset}" "$*" >>"$FAKE_DOCKER_LOG"
|
||||
exit "${FAKE_DOCKER_EXIT:-0}"
|
||||
SH
|
||||
chmod 0755 "$fake"
|
||||
|
||||
printf '%s' old-password >"$tmp/old"
|
||||
printf '%s' "new-'quoted-\$-password" >"$tmp/new"
|
||||
cp "$tmp/old" "$tmp/original"
|
||||
|
||||
printf 'invalid password\n' >"$tmp/whitespace"
|
||||
chmod 0600 "$tmp/old" "$tmp/new" "$tmp/original" "$tmp/whitespace"
|
||||
: >"$log"
|
||||
if PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" THT_VECTOR_BOOTSTRAP_USER=custom_admin \
|
||||
./scripts/vector-rotate-bootstrap-password.sh "$tmp/old" "$tmp/whitespace" \
|
||||
>"$tmp/out" 2>"$tmp/err"; then
|
||||
echo "rotation accepted a whitespace-containing secret" >&2
|
||||
exit 1
|
||||
fi
|
||||
cmp "$tmp/old" "$tmp/original"
|
||||
test ! -s "$log"
|
||||
if find "$tmp" -name 'old.rotate.*' -print | grep -q .; then
|
||||
echo "rotation staged a deployment file before secret validation" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" FAKE_DOCKER_EXIT=1 \
|
||||
./scripts/vector-rotate-bootstrap-password.sh "$tmp/old" "$tmp/new" \
|
||||
>"$tmp/out" 2>"$tmp/err"; then
|
||||
echo "rotation unexpectedly succeeded when database verification failed" >&2
|
||||
exit 1
|
||||
fi
|
||||
cmp "$tmp/old" "$tmp/original"
|
||||
|
||||
: >"$log"
|
||||
PATH="$tmp:$PATH" FAKE_DOCKER_LOG="$log" THT_VECTOR_BOOTSTRAP_USER=custom_admin \
|
||||
./scripts/vector-rotate-bootstrap-password.sh "$tmp/old" "$tmp/new" \
|
||||
>"$tmp/out" 2>"$tmp/err"
|
||||
cmp "$tmp/old" "$tmp/new"
|
||||
grep -q '/run/secrets/bootstrap-old:ro' "$log"
|
||||
grep -q '/run/secrets/bootstrap-new:ro' "$log"
|
||||
grep -q '^custom_admin:' "$log"
|
||||
grep -q 'atomically replaced only after verified database login' "$tmp/out"
|
||||
|
||||
echo "bootstrap rotation ordering and no-config-change failure contracts passed."
|
||||
Executable
+42
@@ -0,0 +1,42 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
image=${1:?usage: test-vector-migration-image.sh IMAGE [PLATFORM]}
|
||||
platform=${2:-${PLATFORM:-linux/arm64}}
|
||||
slug=$$
|
||||
network="thoth-vector-migration-$slug"
|
||||
database="thoth-vector-db-$slug"
|
||||
|
||||
cleanup() {
|
||||
docker rm --force "$database" >/dev/null 2>&1 || true
|
||||
docker network rm "$network" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
docker network create "$network" >/dev/null
|
||||
docker run --detach --rm --platform "$platform" --name "$database" --network "$network" \
|
||||
-e POSTGRES_DB=thoth -e POSTGRES_USER=thoth_admin -e POSTGRES_PASSWORD=test-only \
|
||||
pgvector/pgvector:pg16 >/dev/null
|
||||
|
||||
attempt=0
|
||||
until docker exec "$database" pg_isready -U thoth_admin -d thoth >/dev/null 2>&1; do
|
||||
attempt=$((attempt + 1))
|
||||
if [ "$attempt" -ge 30 ]; then
|
||||
echo "pgvector test database did not become ready" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
database_url="postgresql+psycopg2://thoth_admin:test-only@$database:5432/thoth"
|
||||
applied=$(docker run --rm --platform "$platform" --network "$network" \
|
||||
--entrypoint /opt/venv/bin/tht -e THT_VECTOR_ADMIN_URL="$database_url" \
|
||||
"$image" vector migrate --json)
|
||||
status=$(docker run --rm --platform "$platform" --network "$network" \
|
||||
--entrypoint /opt/venv/bin/tht -e THT_VECTOR_ADMIN_URL="$database_url" \
|
||||
"$image" vector migrate --status --json)
|
||||
|
||||
expected='{"applied": ["001", "002", "003"], "drifted": [], "pending": []}'
|
||||
test "$applied" = "$expected"
|
||||
test "$status" = "$expected"
|
||||
echo "core image vector migration discovery/status smoke passed"
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT TERM
|
||||
|
||||
. ./deploy/vector/secret-policy.sh
|
||||
|
||||
: >"$tmp/empty"
|
||||
printf 'has newline\n' >"$tmp/newline"
|
||||
printf 'has space' >"$tmp/space"
|
||||
printf 'safe-quoted-\047-dollar-$' >"$tmp/valid"
|
||||
printf 'docker-secret' >"$tmp/docker"
|
||||
printf 'owner-readonly' >"$tmp/readonly"
|
||||
printf 'too-open' >"$tmp/open"
|
||||
printf '# comment\n\nTHT_VECTOR_READER_PASSWORD=reader\nTHT_VECTOR_WRITER_PASSWORD=writer\n' >"$tmp/bundle"
|
||||
printf 'THT_VECTOR_READER_PASSWORD=reader\nTHT_VECTOR_WRITER_PASSWORD=writer\nTHT_DWH_API_KEY=one\nTHT_DWH_API_KEY=two\n' >"$tmp/duplicate-bundle"
|
||||
printf 'THT_VECTOR_READER_PASSWORD=reader\r\nTHT_VECTOR_WRITER_PASSWORD=writer\r\n' >"$tmp/crlf-bundle"
|
||||
awk 'BEGIN { printf "THT_VECTOR_READER_PASSWORD="; for (i = 1; i <= 16385; i++) printf "x"; print "" }' >"$tmp/long-line-bundle"
|
||||
awk 'BEGIN { for (i = 1; i <= 70000; i++) print "# filler" }' >"$tmp/large-bundle"
|
||||
chmod 0600 "$tmp/valid"
|
||||
chmod 0444 "$tmp/docker"
|
||||
chmod 0400 "$tmp/readonly"
|
||||
chmod 0640 "$tmp/open"
|
||||
chmod 0600 "$tmp/bundle" "$tmp/duplicate-bundle" "$tmp/crlf-bundle" "$tmp/long-line-bundle" "$tmp/large-bundle"
|
||||
|
||||
for invalid in empty newline space; do
|
||||
if validate_secret_file "$tmp/$invalid" "$invalid" >/dev/null 2>&1; then
|
||||
echo "secret policy accepted $invalid" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
validate_secret_file "$tmp/valid" valid
|
||||
validate_secret_file "$tmp/readonly" readonly
|
||||
if validate_secret_file "$tmp/docker" docker >/dev/null 2>&1; then
|
||||
echo "secret policy accepted world-readable host secret" >&2
|
||||
exit 1
|
||||
fi
|
||||
if validate_secret_file "$tmp/open" open >/dev/null 2>&1; then
|
||||
echo "secret policy accepted group-readable host secret" >&2
|
||||
exit 1
|
||||
fi
|
||||
test "$(read_secret_file "$tmp/valid" valid)" = "safe-quoted-'-dollar-$"
|
||||
test "$(read_bundle_secret "$tmp/bundle" THT_VECTOR_READER_PASSWORD)" = reader
|
||||
test "$(read_bundle_secret "$tmp/crlf-bundle" THT_VECTOR_READER_PASSWORD)" = reader
|
||||
if read_bundle_secret "$tmp/duplicate-bundle" THT_VECTOR_READER_PASSWORD >/dev/null 2>&1; then
|
||||
echo "secret policy accepted a duplicate unrelated bundle key" >&2
|
||||
exit 1
|
||||
fi
|
||||
for invalid_bundle in long-line-bundle large-bundle; do
|
||||
if read_bundle_secret "$tmp/$invalid_bundle" THT_VECTOR_READER_PASSWORD >/dev/null 2>&1; then
|
||||
echo "secret policy accepted oversized $invalid_bundle" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "shared vector secret policy contracts passed."
|
||||
Executable
+53
@@ -0,0 +1,53 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)
|
||||
. "$root/deploy/vector/secret-policy.sh"
|
||||
|
||||
usage() {
|
||||
echo "usage: $0 --host HOST --database DB --user USER --password-file FILE --output FILE [--port PORT]" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
host= database= user= password_file= output= port=5432
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--host) host=${2-}; shift 2 ;;
|
||||
--port) port=${2-}; shift 2 ;;
|
||||
--database) database=${2-}; shift 2 ;;
|
||||
--user) user=${2-}; shift 2 ;;
|
||||
--password-file) password_file=${2-}; shift 2 ;;
|
||||
--output) output=${2-}; shift 2 ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
done
|
||||
[ -n "$host" ] && [ -n "$database" ] && [ -n "$user" ] || usage
|
||||
[ -n "$password_file" ] && [ -n "$output" ] || usage
|
||||
validate_secret_file "$password_file" "backup password file"
|
||||
[ ! -e "$output" ] || { echo "refusing to overwrite existing backup: $output" >&2; exit 2; }
|
||||
output_dir=$(dirname "$output")
|
||||
output_name=$(basename "$output")
|
||||
[ -d "$output_dir" ] || { echo "backup destination directory does not exist" >&2; exit 2; }
|
||||
|
||||
password=$(read_secret_file "$password_file" "backup password file")
|
||||
|
||||
umask 077
|
||||
passfile=$(mktemp "${TMPDIR:-/tmp}/thoth-vector-pgpass.XXXXXX")
|
||||
temporary_output=$(mktemp "$output_dir/.${output_name}.tmp.XXXXXX")
|
||||
cleanup() { rm -f "$passfile" "$temporary_output"; }
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
escaped=$(printf '%s' "$password" | sed 's/\\/\\\\/g; s/:/\\:/g')
|
||||
printf '%s:%s:%s:%s:%s\n' "$host" "$port" "$database" "$user" "$escaped" >"$passfile"
|
||||
chmod 0600 "$passfile"
|
||||
|
||||
PGPASSFILE=$passfile pg_dump \
|
||||
--host="$host" --port="$port" --username="$user" --dbname="$database" \
|
||||
--format=custom --compress=9 \
|
||||
--table=vectors.schema_records --table=vectors.evidence --table=vectors.memory \
|
||||
--table=public.tht_vector_migrations --file="$temporary_output"
|
||||
if ! ln "$temporary_output" "$output"; then
|
||||
echo "refusing to replace backup destination created concurrently: $output" >&2
|
||||
exit 2
|
||||
fi
|
||||
rm -f "$temporary_output"
|
||||
echo "Vector backup written: $output"
|
||||
Executable
+80
@@ -0,0 +1,80 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)
|
||||
. "$root/deploy/vector/secret-policy.sh"
|
||||
|
||||
usage() {
|
||||
echo "usage: $0 --active-host HOST --active-database DB --active-user USER --active-password-file FILE --target-host HOST --target-database DB --target-user USER --target-password-file FILE --input FILE [--active-port PORT] [--target-port PORT] [--force-nonempty]" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
active_host= active_database= active_user= active_password_file= active_port=5432
|
||||
target_host= target_database= target_user= target_password_file= target_port=5432
|
||||
input= force=0
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--active-host) active_host=${2-}; shift 2 ;;
|
||||
--active-port) active_port=${2-}; shift 2 ;;
|
||||
--active-database) active_database=${2-}; shift 2 ;;
|
||||
--active-user) active_user=${2-}; shift 2 ;;
|
||||
--active-password-file) active_password_file=${2-}; shift 2 ;;
|
||||
--target-host) target_host=${2-}; shift 2 ;;
|
||||
--target-port) target_port=${2-}; shift 2 ;;
|
||||
--target-database) target_database=${2-}; shift 2 ;;
|
||||
--target-user) target_user=${2-}; shift 2 ;;
|
||||
--target-password-file) target_password_file=${2-}; shift 2 ;;
|
||||
--input) input=${2-}; shift 2 ;;
|
||||
--force-nonempty) force=1; shift ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
done
|
||||
for value in "$active_host" "$active_database" "$active_user" "$active_password_file" \
|
||||
"$target_host" "$target_database" "$target_user" "$target_password_file" "$input"; do
|
||||
[ -n "$value" ] || usage
|
||||
done
|
||||
[ -r "$input" ] || { echo "backup input is not readable" >&2; exit 2; }
|
||||
validate_secret_file "$active_password_file" "active source password file"
|
||||
validate_secret_file "$target_password_file" "target password file"
|
||||
|
||||
umask 077
|
||||
active_pass=$(mktemp "${TMPDIR:-/tmp}/thoth-vector-active-pgpass.XXXXXX")
|
||||
target_pass=$(mktemp "${TMPDIR:-/tmp}/thoth-vector-target-pgpass.XXXXXX")
|
||||
cleanup() { rm -f "$active_pass" "$target_pass"; }
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
make_passfile() {
|
||||
secret=$(read_secret_file "$5" "database password file")
|
||||
escaped=$(printf '%s' "$secret" | sed 's/\\/\\\\/g; s/:/\\:/g')
|
||||
printf '%s:%s:%s:%s:%s\n' "$1" "$2" "$3" "$4" "$escaped" >"$6"
|
||||
chmod 0600 "$6"
|
||||
}
|
||||
make_passfile "$active_host" "$active_port" "$active_database" "$active_user" \
|
||||
"$active_password_file" "$active_pass"
|
||||
make_passfile "$target_host" "$target_port" "$target_database" "$target_user" \
|
||||
"$target_password_file" "$target_pass"
|
||||
|
||||
identity_sql="SELECT system_identifier::text FROM pg_control_system()"
|
||||
active_identity=$(PGPASSFILE=$active_pass psql -XAt --host="$active_host" --port="$active_port" \
|
||||
--username="$active_user" --dbname="$active_database" --command="$identity_sql")
|
||||
target_identity=$(PGPASSFILE=$target_pass psql -XAt --host="$target_host" --port="$target_port" \
|
||||
--username="$target_user" --dbname="$target_database" --command="$identity_sql")
|
||||
[ "$active_identity" != "$target_identity" ] || {
|
||||
echo "refusing restore: active source and target are on the same PostgreSQL cluster" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
object_count=$(PGPASSFILE=$target_pass psql -XAt --host="$target_host" --port="$target_port" \
|
||||
--username="$target_user" --dbname="$target_database" --command="
|
||||
SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace
|
||||
WHERE (n.nspname='vectors' OR (n.nspname='public' AND c.relname='tht_vector_migrations'))
|
||||
AND c.relkind IN ('r','p','S','v','m');")
|
||||
if [ "$object_count" != 0 ] && [ "$force" != 1 ]; then
|
||||
echo "refusing restore into non-empty target; use --force-nonempty explicitly" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
PGPASSFILE=$target_pass pg_restore --exit-on-error --single-transaction \
|
||||
--clean --if-exists --no-owner \
|
||||
--host="$target_host" --port="$target_port" --username="$target_user" \
|
||||
--dbname="$target_database" "$input"
|
||||
echo "Vector restore completed into explicit target $target_host:$target_port/$target_database"
|
||||
Executable
+46
@@ -0,0 +1,46 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
. ./deploy/vector/secret-policy.sh
|
||||
|
||||
if [ "$#" -ne 2 ]; then
|
||||
echo "usage: $0 OLD_SECRET_FILE NEW_SECRET_FILE" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
absolute_file() {
|
||||
directory=$(CDPATH= cd -- "$(dirname -- "$1")" && pwd)
|
||||
printf '%s/%s\n' "$directory" "$(basename -- "$1")"
|
||||
}
|
||||
|
||||
old_secret=$(absolute_file "$1")
|
||||
new_secret=$(absolute_file "$2")
|
||||
validate_secret_file "$old_secret" old_bootstrap_secret
|
||||
validate_secret_file "$new_secret" new_bootstrap_secret
|
||||
if [ "$old_secret" -ef "$new_secret" ]; then
|
||||
echo "old and new secret files must be distinct" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
project=${COMPOSE_PROJECT_NAME:-thothii}
|
||||
replacement=$(mktemp "${old_secret}.rotate.XXXXXX")
|
||||
trap 'rm -f "$replacement"' EXIT HUP INT TERM
|
||||
cp "$new_secret" "$replacement"
|
||||
chmod 0600 "$replacement"
|
||||
|
||||
docker compose -f compose.yaml -f deploy/compose.local-vector.yaml \
|
||||
--project-name "$project" --profile local-vector run --rm --no-deps \
|
||||
--user 0:0 \
|
||||
--entrypoint /opt/venv/bin/python \
|
||||
--volume "$old_secret:/run/secrets/bootstrap-old:ro" \
|
||||
--volume "$new_secret:/run/secrets/bootstrap-new:ro" \
|
||||
--volume "$(pwd)/deploy/vector/rotate-bootstrap-password.py:/opt/thoth/rotate-bootstrap-password.py:ro" \
|
||||
core /opt/thoth/rotate-bootstrap-password.py \
|
||||
/run/secrets/bootstrap-old /run/secrets/bootstrap-new
|
||||
|
||||
mv -f "$replacement" "$old_secret"
|
||||
trap - EXIT HUP INT TERM
|
||||
|
||||
echo "Deployment bootstrap secret atomically replaced only after verified database login."
|
||||
echo "Re-run: docker compose -f compose.yaml -f deploy/compose.local-vector.yaml --project-name $project --profile local-vector up --wait vector-reconcile vector-migrate core"
|
||||
Executable
+52
@@ -0,0 +1,52 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
platform=${PLATFORM:-linux/arm64}
|
||||
slug=$(printf '%s' "$platform" | tr '/:' '--')
|
||||
core_image="thothii-core:verify-$slug"
|
||||
frontend_image="thothii-frontend:verify-$slug"
|
||||
inventory_dir=${CONTAINER_INVENTORY_DIR:-.artifacts/container-images/$slug}
|
||||
|
||||
mkdir -p "$inventory_dir"
|
||||
|
||||
docker buildx build --platform "$platform" --load \
|
||||
-f docker/core.Dockerfile -t "$core_image" .
|
||||
docker buildx build --platform "$platform" --load \
|
||||
-f docker/frontend.Dockerfile -t "$frontend_image" .
|
||||
|
||||
docker run --rm --platform "$platform" --entrypoint /app/docker/smoke/core-smoke.sh \
|
||||
"$core_image"
|
||||
./scripts/test-vector-migration-image.sh "$core_image" "$platform"
|
||||
docker run --rm --platform "$platform" -e BACKEND_BASE_URL=/api \
|
||||
"$frontend_image" frontend-config-smoke
|
||||
docker run --rm --platform "$platform" -e BACKEND_BASE_URL= \
|
||||
"$frontend_image" frontend-config-smoke
|
||||
docker run --rm --platform "$platform" --entrypoint frontend-policy-smoke "$frontend_image"
|
||||
|
||||
if docker run --rm --platform "$platform" -e BACKEND_BASE_URL=/backend \
|
||||
"$frontend_image" frontend-config-smoke >/dev/null 2>&1; then
|
||||
echo "frontend accepted an unsupported BACKEND_BASE_URL" >&2
|
||||
exit 1
|
||||
fi
|
||||
if docker run --rm --platform "$platform" -e THOTH_PUBLIC_EXPOSURE=true -e AUTH_MODE=none \
|
||||
"$core_image" server >/dev/null 2>&1; then
|
||||
echo "core accepted public exposure without upstream authentication" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker image inspect "$core_image" >"$inventory_dir/core-image-inspect.json"
|
||||
docker image inspect "$frontend_image" >"$inventory_dir/frontend-image-inspect.json"
|
||||
docker run --rm --platform "$platform" --entrypoint sh "$core_image" -c \
|
||||
'dpkg-query -W; /opt/venv/bin/pip freeze; /opt/venv/bin/python -c '"'"'import glob,json; rows=set();
|
||||
for path in glob.glob("/app/backend/node_modules/**/package.json", recursive=True):
|
||||
try:
|
||||
package=json.load(open(path)); rows.add((package.get("name","?"), package.get("version","?")))
|
||||
except (OSError, ValueError): pass
|
||||
print("\n".join(f"{name}=={version}" for name,version in sorted(rows)))'"'"'' \
|
||||
>"$inventory_dir/core-packages.txt"
|
||||
docker run --rm --platform "$platform" --entrypoint sh "$frontend_image" -c 'apk info -vv' \
|
||||
>"$inventory_dir/frontend-packages.txt"
|
||||
|
||||
echo "container verification and inventory complete for $platform"
|
||||
Reference in New Issue
Block a user