mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(deploy): generalize connector secret overrides

Browse Source
This commit is contained in:
marcopan
2026-08-04 15:21:49 +02:00
parent b5071f1494
commit eb01979072
9 changed files with 384 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -35,6 +35,7 @@ config/ca-chain.pem
# ThothII deployment configuration and secret values (keep only the README tracked)
deploy/.env
deploy/compose.connector-secrets.local.yaml
deploy/compose.psd-local.yaml
deploy/workspaces/psd.yaml
deploy/secrets/*
deploy/compose.connector-secrets.yaml
-15
View File
@@ -1,15 +0,0 @@
# Selected direct PostgreSQL/pgvector connector secrets. Each target must match a corresponding
# THT_WS_*_FILE=/run/secrets/<target> binding; source paths are host-only operator configuration.
services:
core:
secrets:
- source: psd_clinical_dwh_password
target: psd-clinical-dwh-password
- source: psd_clinical_vector_password
target: psd-clinical-vector-password
secrets:
psd_clinical_dwh_password:
file: ${THT_WS_PSD_CLINICAL_DWH_PASSWORD_SOURCE:?set THT_WS_PSD_CLINICAL_DWH_PASSWORD_SOURCE}
psd_clinical_vector_password:
file: ${THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_SOURCE:?set THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_SOURCE}
deploy/compose.git-https.yaml
+2
View File
@@ -1,5 +1,7 @@
# Select this override only for an HTTPS Git remote. The separate CA mount keeps TLS validation
# explicit; neither host-only source file nor its contents belongs in the base Compose contract.
x-thoth-git-transport: https
services:
core:
environment:
deploy/compose.git-ssh.yaml
+2
View File
@@ -1,5 +1,7 @@
# Select this override only for an SSH Git remote. The host-only source files must be absolute,
# normalized paths; strict host-key checking is mandatory for registry pull and publish.
x-thoth-git-transport: ssh
services:
core:
environment:
deploy/workspace-registry.env.example
+8 -6
View File
@@ -1,6 +1,5 @@
# Copy these non-secret registry settings into the installation environment.
# Select at most one Git transport override and only the connector-secret entries whose matching
# THT_WS_*_FILE bindings are declared. Every host path below must be absolute and normalized.
# Select at most one Git transport override. Every host path below must be absolute and normalized.
# Their contents are never committed, emitted by the API, or stored in the registry.
THT_WORKSPACE_REGISTRY_ROOT=/data/workspace-registry
THT_WORKSPACE_GIT_BRANCH=main
@@ -15,7 +14,10 @@ THT_WORKSPACE_GIT_AUTHOR_EMAIL=thoth-workspace-registry@localhost
# THT_WORKSPACE_GIT_SSH_KEY_FILE=/absolute/path/to/git-ssh-key
# THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=/absolute/path/to/git-known-hosts
# Host-only connector sources consumed only by deploy/compose.connector-secrets.yaml. The matching
# THT_WS_*_FILE values belong in the separate workspace bindings env file and target /run/secrets.
# THT_WS_PSD_CLINICAL_DWH_PASSWORD_SOURCE=/absolute/path/to/psd-clinical-dwh-password
# THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_SOURCE=/absolute/path/to/psd-clinical-vector-password
# Generate an untracked connector override from arbitrary THT_WS_*_FILE bindings and their
# matching host-only THT_WS_*_SOURCE paths. The generator records paths and variable names only;
# it never writes secret values into the generated Compose file.
# scripts/generate-connector-secrets-override.sh --bindings-env /absolute/path/workspace-bindings.env \
# --operator-env /absolute/path/operator.env --output deploy/compose.connector-secrets.local.yaml
# Run Compose through scripts/compose-with-preflight.sh so relative, non-normalized, and mixed
# SSH/HTTPS selections are rejected before Docker receives the invocation.
scripts/compose-with-preflight.sh
Executable
+127
View File
@@ -0,0 +1,127 @@
#!/usr/bin/env bash
# Validate operator-managed Compose inputs before delegating to Docker Compose.
set -euo pipefail
usage() {
echo "usage: $0 --env-file <operator.env> -f <compose.yaml> ... <docker-compose-command>" >&2
exit 2
}
trim() {
local value="$1"
value="${value#"${value%%[![:space:]]*}"}"
value="${value%"${value##*[![:space:]]}"}"
printf '%s' "$value"
}
is_safe_absolute_path() {
local value="$1" segment
local -a segments
[[ "$value" == /* && "$value" != *//* ]] || return 1
IFS=/ read -r -a segments <<<"$value"
for segment in "${segments[@]}"; do
[[ "$segment" != . && "$segment" != .. ]] || return 1
done
}
read_env_value() {
local source="$1" wanted="$2" line trimmed name value result=""
while IFS= read -r line || [[ -n "$line" ]]; do
trimmed="$(trim "$line")"
[[ -n "$trimmed" && "$trimmed" != \#* && "$trimmed" == *=* ]] || continue
name="$(trim "${trimmed%%=*}")"
[[ "$name" == "$wanted" ]] || continue
value="$(trim "${trimmed#*=}")"
value="$(trim "${value%%#*}")"
value="${value#\"}"; value="${value%\"}"
value="${value#\'}"; value="${value%\'}"
result="$value"
done <"$source"
printf '%s' "$result"
}
source_names() {
local source="$1" line trimmed name
{
while IFS= read -r line || [[ -n "$line" ]]; do
trimmed="$(trim "$line")"
[[ -n "$trimmed" && "$trimmed" != \#* && "$trimmed" == *=* ]] || continue
name="$(trim "${trimmed%%=*}")"
[[ "$name" =~ ^[A-Za-z_][A-Za-z0-9_]*_SOURCE$ ]] && printf '%s\n' "$name"
done <"$source"
while IFS= read -r line; do
name="${line%%=*}"
[[ "$name" =~ ^[A-Za-z_][A-Za-z0-9_]*_SOURCE$ ]] && printf '%s\n' "$name"
done < <(env)
} | sort -u
}
record_git_transport() {
local compose_file="$1" transport=""
[[ -f "$compose_file" ]] || return 0
transport="$(awk '
/^[[:space:]]*x-thoth-git-transport:[[:space:]]*/ {
value = $0
sub(/^[[:space:]]*x-thoth-git-transport:[[:space:]]*/, "", value)
sub(/[[:space:]]*#.*/, "", value)
print value
exit
}
' "$compose_file")"
case "$transport" in
ssh) ssh_override=1 ;;
https) https_override=1 ;;
"")
case "$(basename "$compose_file")" in
*git-ssh*.yaml|*git-ssh*.yml) ssh_override=1 ;;
*git-https*.yaml|*git-https*.yml) https_override=1 ;;
esac
;;
*) echo "invalid x-thoth-git-transport in $compose_file" >&2; exit 2 ;;
esac
}
env_file=""
ssh_override=0
https_override=0
arguments=("$@")
for ((index = 0; index < ${#arguments[@]}; index += 1)); do
argument="${arguments[index]}"
case "$argument" in
--env-file)
((index + 1 < ${#arguments[@]})) || usage
((index += 1))
env_file="${arguments[index]}"
;;
--env-file=*) env_file="${argument#--env-file=}" ;;
-f|--file)
((index + 1 < ${#arguments[@]})) || usage
((index += 1))
compose_file="${arguments[index]}"
record_git_transport "$compose_file"
;;
--file=*)
compose_file="${argument#--file=}"
record_git_transport "$compose_file"
;;
esac
done
[[ -n "$env_file" && -f "$env_file" ]] || { echo "Compose preflight requires an existing --env-file" >&2; exit 2; }
if ((ssh_override && https_override)); then
echo "SSH and HTTPS Git overrides are mutually exclusive" >&2
exit 2
fi
while IFS= read -r name; do
value="$(read_env_value "$env_file" "$name")"
if [[ -v "$name" ]]; then
value="${!name}"
fi
if [[ -n "$value" ]] && ! is_safe_absolute_path "$value"; then
echo "unsafe source path for $name in $env_file" >&2
exit 2
fi
done < <(source_names "$env_file")
exec docker compose "${arguments[@]}"
scripts/generate-connector-secrets-override.sh
Executable
+116
View File
@@ -0,0 +1,116 @@
#!/usr/bin/env bash
# Generate one untracked, installation-specific Compose override from explicit THT_WS_* bindings.
set -euo pipefail
usage() {
echo "usage: $0 --bindings-env <workspace-bindings.env> --operator-env <operator.env> --output <override.yaml>" >&2
exit 2
}
trim() {
local value="$1"
value="${value#"${value%%[![:space:]]*}"}"
value="${value%"${value##*[![:space:]]}"}"
printf '%s' "$value"
}
is_safe_absolute_path() {
local value="$1" segment
local -a segments
[[ "$value" == /* && "$value" != *//* ]] || return 1
IFS=/ read -r -a segments <<<"$value"
for segment in "${segments[@]}"; do
[[ "$segment" != . && "$segment" != .. ]] || return 1
done
}
read_env_value() {
local source="$1" wanted="$2" line trimmed name value result=""
while IFS= read -r line || [[ -n "$line" ]]; do
trimmed="$(trim "$line")"
[[ -n "$trimmed" && "$trimmed" != \#* && "$trimmed" == *=* ]] || continue
name="$(trim "${trimmed%%=*}")"
[[ "$name" == "$wanted" ]] || continue
value="$(trim "${trimmed#*=}")"
value="$(trim "${value%%#*}")"
value="${value#\"}"; value="${value%\"}"
value="${value#\'}"; value="${value%\'}"
result="$value"
done <"$source"
printf '%s' "$result"
}
bindings_env=""
operator_env=""
output=""
while (($#)); do
case "$1" in
--bindings-env) bindings_env="${2:-}"; shift 2 ;;
--operator-env) operator_env="${2:-}"; shift 2 ;;
--output) output="${2:-}"; shift 2 ;;
*) usage ;;
esac
done
[[ -f "$bindings_env" && -f "$operator_env" && -n "$output" ]] || usage
[[ ! -e "$output" ]] || { echo "refusing to overwrite connector override: $output" >&2; exit 2; }
names=()
targets=()
sources=()
while IFS=$'\t' read -r name target; do
[[ "$name" =~ ^THT_WS_[A-Za-z0-9_]+_FILE$ ]] || continue
[[ "$target" =~ ^/run/secrets/[A-Za-z0-9][A-Za-z0-9_.-]*$ && "$target" != *..* ]] || {
echo "invalid connector secret target for $name: $target" >&2
exit 2
}
source_name="${name%_FILE}_SOURCE"
source_path="$(read_env_value "$operator_env" "$source_name")"
if [[ -v "$source_name" ]]; then
source_path="${!source_name}"
fi
if [[ -z "$source_path" ]]; then
echo "set $source_name in $operator_env" >&2
exit 2
fi
if ! is_safe_absolute_path "$source_path"; then
echo "unsafe source path for $source_name in $operator_env" >&2
exit 2
fi
names+=("$name")
targets+=("${target#/run/secrets/}")
sources+=("$source_name")
done < <(
awk '
function trim(value) { sub(/^[[:space:]]+/, "", value); sub(/[[:space:]]+$/, "", value); return value }
{
line = trim($0)
if (line == "" || line ~ /^#/) next
equals = index(line, "=")
if (!equals) next
name = trim(substr(line, 1, equals - 1))
value = trim(substr(line, equals + 1))
sub(/[[:space:]]+#.*/, "", value)
if (value ~ /^".*"$/ || value ~ /^\047.*\047$/) value = substr(value, 2, length(value) - 2)
print name "\t" value
}
' "$bindings_env"
)
((${#names[@]})) || { echo "no THT_WS_*_FILE connector bindings found in $bindings_env" >&2; exit 2; }
{
printf '%s\n' '# Generated by scripts/generate-connector-secrets-override.sh; keep this file untracked.'
printf '%s\n' 'services:' ' core:' ' secrets:'
for ((index = 0; index < ${#names[@]}; index += 1)); do
printf ' - source: connector_secret_%d\n' "$((index + 1))"
printf ' target: %s\n' "${targets[index]}"
done
printf '%s\n' '' 'secrets:'
for ((index = 0; index < ${#names[@]}; index += 1)); do
printf ' connector_secret_%d:\n' "$((index + 1))"
printf ' file: ${%s:?set %s}\n' "${sources[index]}" "${sources[index]}"
done
} >"$output"
printf 'generated %s connector secret mount(s) at %s\n' "${#names[@]}" "$output"
scripts/test-compose-secret-policy.sh
+95 -46
View File
@@ -1,9 +1,9 @@
#!/usr/bin/env bash
# Render optional secret overrides with disposable sources and enforce their isolation contract.
# Render selected secret contracts through the real Compose preflight wrapper.
set -euo pipefail
root="$(cd "$(dirname "$0")/.." && pwd -P)"
fixture_root="$(mktemp -d "${TMPDIR:-/tmp}/thoth-compose-secret-policy.XXXXXX")"
fixture_root="$(mktemp -d "${TMPDIR%/}/thoth-compose-secret-policy.XXXXXX")"
trap 'rm -rf "$fixture_root"' EXIT HUP INT TERM
write_secret() {
@@ -14,19 +14,16 @@ write_secret() {
render() {
local name="$1"; shift
docker compose --env-file "$fixture_root/.env" -f "$root/compose.yaml" "$@" config --format json \
>"$fixture_root/$name.json"
"$root/scripts/compose-with-preflight.sh" --env-file "$fixture_root/operator.env" \
-f "$root/compose.yaml" "$@" config --format json >"$fixture_root/$name.json"
}
assert_render_contract() {
local name="$1" expected_targets="$2"
node - "$fixture_root/$name.json" "$name" "$expected_targets" \
"$fixture_root/ssh-private-key" "$fixture_root/ssh-known-hosts" \
"$fixture_root/https-credentials" "$fixture_root/https-ca.pem" \
"$fixture_root/dwh-password" "$fixture_root/vector-password" <<'NODE'
local name="$1" expected_bind_targets="$2" expected_secret_targets="$3"
node - "$fixture_root/$name.json" "$name" "$expected_bind_targets" "$expected_secret_targets" <<'NODE'
const fs = require("fs");
const [configPath, name, expectedTargets, ...sourcePaths] = process.argv.slice(2);
const [configPath, name, expectedBindTargets, expectedSecretTargets] = process.argv.slice(2);
const config = JSON.parse(fs.readFileSync(configPath, "utf8"));
const core = config.services?.core;
if (!core) throw new Error(`${name}: missing core service`);
@@ -34,13 +31,13 @@ if (name === "base" && (core.volumes || []).some((mount) => mount.source === "/d
throw new Error("base: /dev/null must never be used as a secret mount source");
}
const mountTargets = (core.volumes || [])
const bindTargets = (core.volumes || [])
.filter((mount) => mount.target?.startsWith("/run/secrets/"))
.map((mount) => mount.target)
.sort();
const expectedMountTargets = expectedTargets ? expectedTargets.split(",").filter(Boolean).sort() : [];
if (mountTargets.join(",") !== expectedMountTargets.join(",")) {
throw new Error(`${name}: unexpected /run/secrets bind targets: ${mountTargets.join(",")}`);
const expectedBinds = expectedBindTargets ? expectedBindTargets.split(",").filter(Boolean).sort() : [];
if (bindTargets.join(",") !== expectedBinds.join(",")) {
throw new Error(`${name}: unexpected /run/secrets bind targets: ${bindTargets.join(",")}`);
}
for (const mount of (core.volumes || []).filter((item) => item.target?.startsWith("/run/secrets/"))) {
if (mount.type !== "bind" || !mount.read_only) {
@@ -49,11 +46,9 @@ for (const mount of (core.volumes || []).filter((item) => item.target?.startsWit
}
const secretTargets = (core.secrets || []).map((secret) => secret.target).sort();
if (name === "connector" && secretTargets.join(",") !== "psd-clinical-dwh-password,psd-clinical-vector-password") {
throw new Error(`${name}: connector secret targets do not match declared THT_WS_*_FILE paths`);
}
if (name !== "connector" && secretTargets.length !== 0) {
throw new Error(`${name}: unexpected Docker secrets`);
const expectedSecrets = expectedSecretTargets ? expectedSecretTargets.split(",").filter(Boolean).sort() : [];
if (secretTargets.join(",") !== expectedSecrets.join(",")) {
throw new Error(`${name}: connector targets do not match generated THT_WS_*_FILE bindings`);
}
if (name === "ssh") {
@@ -62,41 +57,52 @@ if (name === "ssh") {
if (!command.includes(option)) throw new Error(`ssh: missing strict SSH option ${option}`);
}
}
if (name === "https") {
if (core.environment?.GIT_CONFIG_VALUE_1 !== "/run/secrets/workspace-registry-git-ca") {
throw new Error("https: HTTPS CA verification is not configured");
}
if (name === "https" && core.environment?.GIT_CONFIG_VALUE_1 !== "/run/secrets/workspace-registry-git-ca") {
throw new Error("https: HTTPS CA verification is not configured");
}
const rendered = JSON.stringify(config);
for (const secret of ["fixture-ssh-private-key", "fixture-ssh-known-hosts", "fixture-https-credentials", "fixture-https-ca", "fixture-dwh-password", "fixture-vector-password"]) {
for (const secret of ["fixture-ssh-private-key", "fixture-ssh-known-hosts", "fixture-https-credentials", "fixture-https-ca", "fixture-dwh-password", "fixture-vector-api-key"]) {
if (rendered.includes(secret)) throw new Error(`${name}: rendered Compose leaked fixture secret value`);
}
for (const sourcePath of sourcePaths) {
if (!sourcePath.startsWith("/")) throw new Error(`${name}: fixture source must be absolute`);
}
NODE
}
assert_required_source() {
local variable="$1" override="$2"
assert_missing_source_rejected() {
local variable="$1"
local generated="$fixture_root/missing-$variable.yaml"
local missing_env="$fixture_root/missing-$variable.env"
grep -v "^$variable=" "$fixture_root/.env" >"$missing_env"
if env -u "$variable" docker compose --env-file "$missing_env" -f "$root/compose.yaml" -f "$override" config --quiet \
grep -v "^$variable=" "$fixture_root/operator.env" >"$missing_env"
if env -u "$variable" "$root/scripts/generate-connector-secrets-override.sh" \
--bindings-env "$fixture_root/workspace-bindings.env" --operator-env "$missing_env" --output "$generated" \
>"$fixture_root/missing-$variable.out" 2>"$fixture_root/missing-$variable.err"; then
echo "selected override accepted missing $variable" >&2
echo "connector generator accepted missing $variable" >&2
exit 1
fi
grep -Fq "$variable" "$fixture_root/missing-$variable.err"
}
assert_unsafe_source_rejected() {
local value="$1" label="$2"
local unsafe_env="$fixture_root/$label.env"
sed "s|^THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=.*|THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=$value|" \
"$fixture_root/operator.env" >"$unsafe_env"
if env -u THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE \
"$root/scripts/compose-with-preflight.sh" --env-file "$unsafe_env" -f "$root/compose.yaml" config --quiet \
>"$fixture_root/$label.out" 2>"$fixture_root/$label.err"; then
echo "Compose preflight accepted $label source path" >&2
exit 1
fi
grep -Fq 'unsafe source path' "$fixture_root/$label.err"
}
write_secret "$fixture_root/pi-auth.json" 'fixture-pi-auth'
write_secret "$fixture_root/ssh-private-key" 'fixture-ssh-private-key'
write_secret "$fixture_root/ssh-known-hosts" 'fixture-ssh-known-hosts'
write_secret "$fixture_root/https-credentials" 'fixture-https-credentials'
write_secret "$fixture_root/https-ca.pem" 'fixture-https-ca'
write_secret "$fixture_root/dwh-password" 'fixture-dwh-password'
write_secret "$fixture_root/vector-password" 'fixture-vector-password'
write_secret "$fixture_root/vector-api-key" 'fixture-vector-api-key'
printf '%s\n' \
'THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git' \
@@ -105,23 +111,66 @@ printf '%s\n' \
"THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=$fixture_root/ssh-known-hosts" \
"THT_WORKSPACE_GIT_CREDENTIALS_FILE=$fixture_root/https-credentials" \
"THT_WORKSPACE_GIT_CA_FILE=$fixture_root/https-ca.pem" \
"THT_WS_PSD_CLINICAL_DWH_PASSWORD_SOURCE=$fixture_root/dwh-password" \
"THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_SOURCE=$fixture_root/vector-password" >"$fixture_root/.env"
"THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=$fixture_root/dwh-password" \
"THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_SOURCE=$fixture_root/vector-api-key" >"$fixture_root/operator.env"
printf '%s\n' \
'THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=postgres_direct' \
'THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password' \
'THT_WS_NORTH_STAR_RESEARCH_VECTOR_TRANSPORT=rest_api' \
'THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_FILE=/run/secrets/north-star-research-vector-api-key' \
>"$fixture_root/workspace-bindings.env"
connector_override="$fixture_root/compose.connector-secrets.local.yaml"
"$root/scripts/generate-connector-secrets-override.sh" \
--bindings-env "$fixture_root/workspace-bindings.env" --operator-env "$fixture_root/operator.env" \
--output "$connector_override"
render base
assert_render_contract base ''
assert_render_contract base '' ''
render ssh -f "$root/deploy/compose.git-ssh.yaml"
assert_render_contract ssh '/run/secrets/workspace-registry-git-known-hosts,/run/secrets/workspace-registry-git-ssh-key'
assert_render_contract ssh '/run/secrets/workspace-registry-git-known-hosts,/run/secrets/workspace-registry-git-ssh-key' ''
render https -f "$root/deploy/compose.git-https.yaml"
assert_render_contract https '/run/secrets/workspace-registry-git-ca,/run/secrets/workspace-registry-git-credentials'
render connector -f "$root/deploy/compose.connector-secrets.yaml"
assert_render_contract connector ''
assert_render_contract https '/run/secrets/workspace-registry-git-ca,/run/secrets/workspace-registry-git-credentials' ''
render connector -f "$connector_override"
assert_render_contract connector '' 'north-star-research-dwh-password,north-star-research-vector-api-key'
assert_required_source THT_WORKSPACE_GIT_SSH_KEY_FILE "$root/deploy/compose.git-ssh.yaml"
assert_required_source THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE "$root/deploy/compose.git-ssh.yaml"
assert_required_source THT_WORKSPACE_GIT_CREDENTIALS_FILE "$root/deploy/compose.git-https.yaml"
assert_required_source THT_WORKSPACE_GIT_CA_FILE "$root/deploy/compose.git-https.yaml"
assert_required_source THT_WS_PSD_CLINICAL_DWH_PASSWORD_SOURCE "$root/deploy/compose.connector-secrets.yaml"
assert_required_source THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_SOURCE "$root/deploy/compose.connector-secrets.yaml"
assert_missing_source_rejected THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE
assert_missing_source_rejected THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_SOURCE
assert_unsafe_source_rejected 'relative/secret' relative-source
assert_unsafe_source_rejected '/private/secrets/../secret' non-normalized-source
if THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE='relative/host-override' \
"$root/scripts/compose-with-preflight.sh" --env-file "$fixture_root/operator.env" -f "$root/compose.yaml" config --quiet \
>"$fixture_root/host-override.out" 2>"$fixture_root/host-override.err"; then
echo "Compose preflight accepted a relative exported source path" >&2
exit 1
fi
grep -Fq 'unsafe source path' "$fixture_root/host-override.err"
if THT_WS_HOST_ONLY_PASSWORD_SOURCE='relative/host-only' \
"$root/scripts/compose-with-preflight.sh" --env-file "$fixture_root/operator.env" -f "$root/compose.yaml" config --quiet \
>"$fixture_root/host-only.out" 2>"$fixture_root/host-only.err"; then
echo "Compose preflight accepted a relative host-only source path" >&2
exit 1
fi
grep -Fq 'unsafe source path' "$fixture_root/host-only.err"
if "$root/scripts/compose-with-preflight.sh" --env-file "$fixture_root/operator.env" \
-f "$root/compose.yaml" -f "$root/deploy/compose.git-ssh.yaml" -f "$root/deploy/compose.git-https.yaml" config --quiet \
>"$fixture_root/combined.out" 2>"$fixture_root/combined.err"; then
echo "Compose preflight accepted combined SSH and HTTPS overrides" >&2
exit 1
fi
grep -Fq 'mutually exclusive' "$fixture_root/combined.err"
cp "$root/deploy/compose.git-ssh.yaml" "$fixture_root/transport-a.yaml"
cp "$root/deploy/compose.git-https.yaml" "$fixture_root/transport-b.yaml"
if "$root/scripts/compose-with-preflight.sh" --env-file "$fixture_root/operator.env" \
-f "$root/compose.yaml" -f "$fixture_root/transport-a.yaml" -f "$fixture_root/transport-b.yaml" config --quiet \
>"$fixture_root/renamed-combined.out" 2>"$fixture_root/renamed-combined.err"; then
echo "Compose preflight accepted renamed combined Git overrides" >&2
exit 1
fi
grep -Fq 'mutually exclusive' "$fixture_root/renamed-combined.err"
echo "Compose secret policy passed."
scripts/verify-workspace-install-docs.sh
+33 -24
View File
@@ -64,30 +64,40 @@ compose_fixture() {
local name="$1" directory="$2"; shift 2
(
cd "$directory"
docker compose --env-file .env "$@" config --quiet
"$root/scripts/compose-with-preflight.sh" --env-file .env "$@" config --quiet
)
echo "$name passed"
}
prepare_binding_fixture() {
local directory="$1"
cp "$root/docs/install/examples/workspace-bindings.env.example" "$directory/workspace-bindings.env"
printf '%s\n' \
'THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=postgres_direct' \
'THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password' \
'THT_WS_NORTH_STAR_RESEARCH_VECTOR_TRANSPORT=rest_api' \
'THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_FILE=/run/secrets/north-star-research-vector-api-key' \
>"$directory/workspace-bindings.env"
printf 'THT_WORKSPACE_BINDINGS_ENV_FILE=%s\n' "$directory/workspace-bindings.env" >>"$directory/.env"
}
verify_connector_fixture() {
local directory="$1" rendered project
local directory="$1" rendered project connector_override
project="thoth-install-connector-fixture-$$"
connector_override="$directory/connector-secrets.local.yaml"
"$root/scripts/generate-connector-secrets-override.sh" \
--bindings-env "$directory/workspace-bindings.env" --operator-env "$directory/.env" \
--output "$connector_override" >/dev/null
rendered="$(
cd "$directory"
docker compose --env-file .env -f compose.workspace-registry.yaml -f connector-secrets.yaml config
"$root/scripts/compose-with-preflight.sh" --env-file .env \
-f compose.workspace-registry.yaml -f connector-secrets.local.yaml config
)"
for expected in \
'THT_WS_PSD_CLINICAL_DWH_TRANSPORT: postgres_direct' \
'THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: /run/secrets/psd-clinical-dwh-password' \
'THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_FILE: /run/secrets/psd-clinical-vector-password' \
'target: psd-clinical-dwh-password' \
'target: psd-clinical-vector-password'; do
'THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT: postgres_direct' \
'THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE: /run/secrets/north-star-research-dwh-password' \
'THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_FILE: /run/secrets/north-star-research-vector-api-key' \
'target: north-star-research-dwh-password' \
'target: north-star-research-vector-api-key'; do
grep -Fq "$expected" <<<"$rendered" || {
echo "connector fixture does not give core required binding or secret target: $expected" >&2
return 1
@@ -96,33 +106,33 @@ verify_connector_fixture() {
echo "copied connector binding/secret fixture passed"
if ! (
cd "$directory"
docker compose --project-name "$project" --env-file .env -f compose.workspace-registry.yaml -f connector-secrets.yaml \
run --rm --no-deps --build --entrypoint sh core -c '
test "$THT_WS_PSD_CLINICAL_DWH_TRANSPORT" = postgres_direct
test "$THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE" = /run/secrets/psd-clinical-dwh-password
test "$THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_FILE" = /run/secrets/psd-clinical-vector-password
test -f "$THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE"
test -f "$THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_FILE"
"$root/scripts/compose-with-preflight.sh" --project-name "$project" --env-file .env \
-f compose.workspace-registry.yaml -f connector-secrets.local.yaml run --rm --no-deps --build --entrypoint sh core -c '
test "$THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT" = postgres_direct
test "$THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE" = /run/secrets/north-star-research-dwh-password
test "$THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_FILE" = /run/secrets/north-star-research-vector-api-key
test -f "$THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE"
test -f "$THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_FILE"
'
); then
(
cd "$directory"
docker compose --project-name "$project" --env-file .env -f compose.workspace-registry.yaml -f connector-secrets.yaml \
down --volumes --remove-orphans
"$root/scripts/compose-with-preflight.sh" --project-name "$project" --env-file .env \
-f compose.workspace-registry.yaml -f connector-secrets.local.yaml down --volumes --remove-orphans
) || true
return 1
fi
(
cd "$directory"
docker compose --project-name "$project" --env-file .env -f compose.workspace-registry.yaml -f connector-secrets.yaml \
down --volumes --remove-orphans
"$root/scripts/compose-with-preflight.sh" --project-name "$project" --env-file .env \
-f compose.workspace-registry.yaml -f connector-secrets.local.yaml down --volumes --remove-orphans
)
echo "core process sees connector bindings and secret files passed"
}
verify_copied_operator_fixtures() {
local fixture_root local_dir server_dir https_dir ssh_dir connector_dir
fixture_root="$(mktemp -d "${TMPDIR:-/tmp}/thoth-install-fixtures.XXXXXX")"
fixture_root="$(mktemp -d "${TMPDIR%/}/thoth-install-fixtures.XXXXXX")"
trap 'rm -rf "$fixture_root"' RETURN
local_dir="$fixture_root/local"; server_dir="$fixture_root/server"
https_dir="$fixture_root/https"; ssh_dir="$fixture_root/ssh"; connector_dir="$fixture_root/connector"
@@ -168,12 +178,11 @@ verify_copied_operator_fixtures() {
compose_fixture "copied SSH Git override fixture" "$ssh_dir" -f compose.workspace-registry.yaml -f git-ssh.yaml
cp "$root/docs/install/examples/local-compose.workspace-registry.yaml" "$connector_dir/compose.workspace-registry.yaml"
cp "$root/docs/install/examples/connector-secrets.workspace-registry.yaml" "$connector_dir/connector-secrets.yaml"
: >"$connector_dir/dwh-password"; : >"$connector_dir/vector-password"
printf '%s\n' \
"THT_SOURCE_ROOT=$root" \
"THT_WS_PSD_CLINICAL_DWH_PASSWORD_SOURCE=$connector_dir/dwh-password" \
"THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_SOURCE=$connector_dir/vector-password" >"$connector_dir/.env"
"THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=$connector_dir/dwh-password" \
"THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_SOURCE=$connector_dir/vector-password" >"$connector_dir/.env"
prepare_binding_fixture "$connector_dir"
verify_connector_fixture "$connector_dir"