fix: enforce server trust boundaries

This commit is contained in:
2026-08-05 11:42:35 +02:00
parent 96fe5bfa79
commit a94affd6ac
10 changed files with 511 additions and 40 deletions
+8 -1
View File
@@ -2,7 +2,14 @@
set -euo pipefail
repository_root=$(cd "$(dirname "$0")/.." && pwd)
output_directory="$repository_root/dist/thothctl"
output_directory="${THT_THOTHCTL_OUTPUT_DIRECTORY:-$repository_root/dist/thothctl}"
if [[ "$output_directory" != /* || "$output_directory" == / || "$output_directory" == */ ||
"$output_directory" == *//* || "/$output_directory/" == */../* ||
"/$output_directory/" == */./* ]]; then
echo "THT_THOTHCTL_OUTPUT_DIRECTORY must be an absolute canonical path" >&2
exit 2
fi
mkdir -p "$output_directory"
docker build --file "$repository_root/docker/thothctl.Dockerfile" --output "type=local,dest=$output_directory" "$repository_root"
+81
View File
@@ -0,0 +1,81 @@
#!/usr/bin/env bash
# Execute the documented server ownership model with distinct runtime and human operator IDs.
set -euo pipefail
root="$(cd "$(dirname "$0")/.." && pwd -P)"
image='golang:1.26.5-bookworm@sha256:1ecb7edf62a0408027bd5729dfd6b1b8766e578e8df93995b225dfd0944eb651'
docker run --rm --volume "$root:/repository:ro" "$image" /bin/bash -ceu '
groupadd --gid 10001 thothii
useradd --uid 10001 --gid 10001 --no-create-home --shell /usr/sbin/nologin thothii
groupadd --gid 20001 operator-primary
groupadd --gid 20002 thothii-ops
groupadd --gid 20003 docker
useradd --uid 20001 --gid 20001 --groups 20002,20003 --create-home --shell /bin/bash operator
install -d -o 10001 -g 20002 -m 2750 /srv/thothii/source
install -d -o 10001 -g 20002 -m 2770 /srv/thothii/operator
install -d -o 10001 -g 20002 -m 2750 /srv/thothii/secrets
install -d -o 10001 -g 10001 -m 0750 /srv/thothii/data /srv/thothii/pi-state /srv/thothii/workspace-registry
install -d -o 10001 -g 20002 -m 2750 /srv/thothii/source/ThothII /srv/thothii/source/ThothII/scripts
install -o 10001 -g 20002 -m 0750 /repository/scripts/build-thothctl.sh /srv/thothii/source/ThothII/scripts/build-thothctl.sh
install -o 10001 -g 20002 -m 0750 /repository/scripts/generate-connector-secrets-override.sh /srv/thothii/source/ThothII/scripts/generate-connector-secrets-override.sh
printf "%s\n" "PLACEHOLDER=replace-me" "THT_WS_TEST_DWH_PASSWORD_SOURCE=/srv/thothii/secrets/dwh-password" > /srv/thothii/operator/server.env
printf "%s\n" "projectDirectory: replace-me" > /srv/thothii/operator/thothii-installation.yaml
printf "%s\n" "THT_WS_TEST_DWH_PASSWORD_FILE=/run/secrets/test-dwh-password" > /srv/thothii/operator/workspace-bindings.env
printf "%s\n" "operator-readable-secret" > /srv/thothii/secrets/dwh-password
chown 10001:20002 /srv/thothii/operator/server.env /srv/thothii/operator/thothii-installation.yaml /srv/thothii/operator/workspace-bindings.env /srv/thothii/secrets/dwh-password
chmod 0660 /srv/thothii/operator/server.env /srv/thothii/operator/thothii-installation.yaml /srv/thothii/operator/workspace-bindings.env
chmod 0640 /srv/thothii/secrets/dwh-password
printf "%s\n" \
"#!/bin/bash" \
"set -euo pipefail" \
"if [[ \"\${1:-}\" == build ]]; then" \
" destination=; for argument in \"\$@\"; do case \"\$argument\" in type=local,dest=*) destination=\"\${argument#type=local,dest=}\" ;; esac; done" \
" test -n \"\$destination\"; mkdir -p \"\$destination\"" \
" printf \"%s\\n\" \"#!/bin/bash\" \"set -euo pipefail\" \"test -r \\\"\\\$2\\\"\" \"test -r /srv/thothii/secrets/dwh-password\" \"docker compose up --detach\" > \"\$destination/thothctl-linux-amd64\"" \
" chmod 0750 \"\$destination/thothctl-linux-amd64\"; exit 0" \
"fi" \
"test \"\${1:-}\" = compose; : > /srv/thothii/operator/start.marker" \
> /usr/local/bin/docker
chmod 0755 /usr/local/bin/docker
runuser --user operator -- /bin/bash -ceu '\''
umask 0007
sed -i "s/replace-me/ready/" /srv/thothii/operator/server.env
sed -i "s#replace-me#/srv/thothii/source/ThothII#" /srv/thothii/operator/thothii-installation.yaml
/srv/thothii/source/ThothII/scripts/generate-connector-secrets-override.sh \
--bindings-env /srv/thothii/operator/workspace-bindings.env \
--operator-env /srv/thothii/operator/server.env \
--output /srv/thothii/operator/connector-secrets.server.yaml
test -r /srv/thothii/secrets/dwh-password
if (printf tamper >> /srv/thothii/secrets/dwh-password) 2>/dev/null; then exit 41; fi
if touch /srv/thothii/source/operator-must-not-write 2>/dev/null; then exit 42; fi
if touch /srv/thothii/data/operator-must-not-write 2>/dev/null; then exit 43; fi
THT_THOTHCTL_OUTPUT_DIRECTORY=/srv/thothii/operator/build-output \
/srv/thothii/source/ThothII/scripts/build-thothctl.sh
if THT_THOTHCTL_OUTPUT_DIRECTORY=relative-output \
/srv/thothii/source/ThothII/scripts/build-thothctl.sh 2>/dev/null; then exit 44; fi
root_output_error=/srv/thothii/operator/root-output.error
if THT_THOTHCTL_OUTPUT_DIRECTORY=/ \
/srv/thothii/source/ThothII/scripts/build-thothctl.sh 2>"$root_output_error"; then exit 45; fi
grep -Fq "THT_THOTHCTL_OUTPUT_DIRECTORY must be an absolute canonical path" \
"$root_output_error" || exit 46
rm -f "$root_output_error"
/srv/thothii/operator/build-output/thothctl-linux-amd64 \
--installation /srv/thothii/operator/thothii-installation.yaml start
'\''
test "$(stat -c %u:%g /srv/thothii/operator/connector-secrets.server.yaml)" = 20001:20002
test "$(stat -c %a /srv/thothii/operator/connector-secrets.server.yaml)" = 660
test "$(stat -c %u:%g /srv/thothii/operator/build-output/thothctl-linux-amd64)" = 20001:20002
test "$(stat -c %a /srv/thothii/operator/build-output/thothctl-linux-amd64)" = 750
test -f /srv/thothii/operator/start.marker
test ! -e /srv/thothii/source/operator-must-not-write
test ! -e /srv/thothii/data/operator-must-not-write
test "$(cat /srv/thothii/secrets/dwh-password)" = operator-readable-secret
'
echo "distinct server operator UID/GID fixture passed"
+62 -4
View File
@@ -100,6 +100,41 @@ sed '/^case "\$mode" in/,$d' "$root/scripts/verify-workspace-install-docs.sh" >"
# shellcheck source=/dev/null
source "$verifier_functions"
adapted_reorder="$negative_root/caddy-adapted-reorder.json"
node - "$adapted_reorder" <<'NODE'
const fs = require("fs");
const publicHeaders = [
"X-Thoth-Principal-Issuer", "X-Thoth-Principal-Subject",
"X-Thoth-Principal-Display-Name", "X-Thoth-Is-Admin",
];
const trustedHeaders = [
"X-Thoth-Trusted-Principal-Issuer", "X-Thoth-Trusted-Principal-Subject",
"X-Thoth-Trusted-Principal-Display-Name", "X-Thoth-Trusted-Is-Admin",
];
const clear = (name) => ({handler: "headers", request: {delete: [name]}});
const auth = {
handler: "reverse_proxy", upstreams: [{dial: "auth-gateway:4180"}],
handle_response: [{match: {status_code: [2]}, routes: [{handle: trustedHeaders.map((name, index) => ({
handler: "headers", request: {set: {[name]: [`{http.reverse_proxy.header.${publicHeaders[index]}}`]}},
}))}]}],
};
const document = {routes: [{handle: [
...publicHeaders.map(clear), auth, ...trustedHeaders.map(clear),
{handler: "reverse_proxy", upstreams: [{dial: "127.0.0.1:8080"}]},
]}]};
fs.writeFileSync(process.argv[2], JSON.stringify(document));
NODE
adapted_output="$negative_root/caddy-adapted-output"
set +e
verify_caddy_adapted_identity_order "$adapted_reorder" >"$adapted_output" 2>&1
adapted_status=$?
set -e
if [[ $adapted_status -eq 0 ]] || ! grep -Fq "Caddy adapted identity clears must execute before authentication" "$adapted_output"; then
echo "Caddy reordered adapted-handler fixture was not rejected correctly" >&2
cat "$adapted_output" >&2
exit 1
fi
negative_failures=0
expect_guide_rejected() {
local label="$1" validator="$2" source_guide="$3" relative_path="$4"
@@ -175,6 +210,14 @@ switch (mutation) {
case "nginx-no-admin-map":
changed = original.replace("proxy_set_header X-Thoth-Trusted-Is-Admin $thoth_is_admin;", "proxy_set_header X-Thoth-Trusted-Is-Admin \"\";");
break;
case "nginx-admin-clear-wrong-scope": {
const clear = ' proxy_set_header X-Thoth-Is-Admin "";';
const authAt = original.indexOf(clear);
changed = original.slice(0, authAt) + original.slice(authAt + clear.length + 1);
const frontendAt = changed.indexOf(clear);
changed = changed.slice(0, frontendAt) + clear + "\n" + clear + changed.slice(frontendAt + clear.length);
break;
}
case "caddy-no-auth":
changed = original.replace("forward_auth auth-gateway:4180 {", "# forward authentication omitted");
break;
@@ -196,6 +239,13 @@ switch (mutation) {
case "caddy-no-admin-map":
changed = original.replace("X-Thoth-Is-Admin>X-Thoth-Trusted-Is-Admin", "X-Thoth-Is-Admin");
break;
case "caddy-clears-after-auth": {
const clearPattern = /(?:\t\trequest_header -X-(?:Authenticated-User|Thoth-[^\n]+)\n)+/;
const clears = original.match(clearPattern)?.[0] || "";
changed = original.replace(clearPattern, "");
changed = changed.replace("\n\t\treverse_proxy 127.0.0.1:8080 {", "\n" + clears + "\n\t\treverse_proxy 127.0.0.1:8080 {");
break;
}
case "dirty-source":
changed = original.replaceAll("git status --porcelain --untracked-files=all", "git status --short");
break;
@@ -310,19 +360,23 @@ expect_guide_rejected \
expect_guide_rejected \
"Nginx issuer inbound claim not cleared" verify_reverse_proxy_nginx_guide \
"$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-issuer-clear \
"Nginx proxy does not clear inbound issuer identity"
"Nginx auth location does not clear inbound issuer identity"
expect_guide_rejected \
"Nginx subject auth response not captured" verify_reverse_proxy_nginx_guide \
"$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-subject-capture \
"Nginx proxy does not capture authenticated subject identity"
"Nginx frontend location does not capture authenticated subject identity"
expect_guide_rejected \
"Nginx display identity not mapped to private hop" verify_reverse_proxy_nginx_guide \
"$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-display-map \
"Nginx proxy does not map authenticated display identity"
"Nginx frontend location does not map authenticated display identity"
expect_guide_rejected \
"Nginx admin identity not mapped to private hop" verify_reverse_proxy_nginx_guide \
"$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-admin-map \
"Nginx proxy does not map authenticated admin identity"
"Nginx frontend location does not map authenticated admin identity"
expect_guide_rejected \
"Nginx admin clear moved out of auth scope" verify_reverse_proxy_nginx_guide \
"$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-admin-clear-wrong-scope \
"Nginx auth location does not clear inbound admin identity"
expect_guide_rejected \
"Caddy identity without authentication" verify_reverse_proxy_caddy_guide \
"$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-auth \
@@ -351,6 +405,10 @@ expect_guide_rejected \
"Caddy admin identity not mapped to private hop" verify_reverse_proxy_caddy_guide \
"$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-admin-map \
"Caddy proxy does not map authenticated admin identity"
expect_guide_rejected \
"Caddy identity clears reordered after auth" verify_reverse_proxy_caddy_guide \
"$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-clears-after-auth \
"Caddy identity clears must precede forward_auth"
expect_guide_rejected \
"dirty or untracked source tree" verify_local_guide \
+166 -11
View File
@@ -513,6 +513,9 @@ verify_server_guide() {
"core" \
"UID/GID 10001" \
"thothii-ops" \
"-m 2770 /srv/thothii/operator" \
"chmod 0660 /srv/thothii/operator/server.env" \
"THT_THOTHCTL_OUTPUT_DIRECTORY=/srv/thothii/operator/build-output" \
"/srv/thothii" \
"example operator root" \
"/run/secrets" \
@@ -655,19 +658,47 @@ const identities = [
["admin", "Is-Admin", "thoth_is_admin", "x_thoth_is_admin"],
];
function escaped(value) { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); }
function directiveBlock(text, marker) {
const start = text.indexOf(marker);
if (start < 0) throw new Error(`Nginx proxy lacks scoped block: ${marker}`);
const opening = text.indexOf("{", start);
let depth = 0;
for (let index = opening; index < text.length; index++) {
if (text[index] === "{") depth++;
if (text[index] === "}" && --depth === 0) return text.slice(opening + 1, index);
}
throw new Error(`Nginx proxy has unterminated scoped block: ${marker}`);
}
const authLocation = directiveBlock(block, "location = /_authenticate {");
const frontendLocation = directiveBlock(block, "location / {");
for (const [label, publicName, variable, upstream] of identities) {
const trustedName = publicName === "Is-Admin" ? "Is-Admin" : publicName;
const publicClears = block.match(new RegExp(`proxy_set_header\\s+X-Thoth-${escaped(publicName)}\\s+"";`, "g")) || [];
if (publicClears.length < 2) throw new Error(`Nginx proxy does not clear inbound ${label} identity`);
const publicClear = new RegExp(`proxy_set_header\\s+X-Thoth-${escaped(publicName)}\\s+"";`);
const trustedHeader = `X-Thoth-Trusted-${trustedName}`;
if (!new RegExp(`proxy_set_header\\s+${escaped(trustedHeader)}\\s+"";`).test(block)) {
throw new Error(`Nginx proxy does not clear inbound trusted ${label} identity`);
const trustedClear = new RegExp(`proxy_set_header\\s+${escaped(trustedHeader)}\\s+"";`);
const authPublicAt = authLocation.search(publicClear);
const authTrustedAt = authLocation.search(trustedClear);
if (authPublicAt < 0) {
throw new Error(`Nginx auth location does not clear inbound ${label} identity`);
}
if (!new RegExp(`auth_request_set\\s+\\$${variable}\\s+\\$upstream_http_${upstream};`, "m").test(block.replace(/\s+/g, " "))) {
throw new Error(`Nginx proxy does not capture authenticated ${label} identity`);
if (authTrustedAt < 0) {
throw new Error(`Nginx auth location does not clear inbound trusted ${label} identity`);
}
if (!new RegExp(`proxy_set_header\\s+${escaped(trustedHeader)}\\s+\\$${variable};`).test(block)) {
throw new Error(`Nginx proxy does not map authenticated ${label} identity`);
const frontendPublicAt = frontendLocation.search(publicClear);
if (frontendPublicAt < 0) {
throw new Error(`Nginx frontend location does not clear inbound ${label} identity`);
}
const normalizedFrontend = frontendLocation.replace(/\s+/g, " ");
const captureAt = normalizedFrontend.search(new RegExp(`auth_request_set\\s+\\$${variable}\\s+\\$upstream_http_${upstream};`));
if (captureAt < 0) {
throw new Error(`Nginx frontend location does not capture authenticated ${label} identity`);
}
const mapAt = normalizedFrontend.search(new RegExp(`proxy_set_header\\s+${escaped(trustedHeader)}\\s+\\$${variable};`));
if (mapAt < 0) {
throw new Error(`Nginx frontend location does not map authenticated ${label} identity`);
}
if (new RegExp(`auth_request_set\\s+\\$${variable}|proxy_set_header\\s+${escaped(trustedHeader)}\\s+\\$${variable};`).test(authLocation)) {
throw new Error(`Nginx auth location performs a forbidden ${label} capture or mapping`);
}
}
NODE
@@ -706,26 +737,148 @@ if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(block) || !block.includes("127.0.0.1:
for (const token of tokens) {
if (!block.includes(token)) throw new Error(`Caddy proxy lacks structural token: ${token}`);
}
function directiveBlock(text, marker) {
const start = text.indexOf(marker);
if (start < 0) throw new Error(`Caddy proxy lacks scoped block: ${marker}`);
const opening = text.indexOf("{", start);
let depth = 0;
for (let index = opening; index < text.length; index++) {
if (text[index] === "{") depth++;
if (text[index] === "}" && --depth === 0) return {start, end: index, body: text.slice(opening + 1, index)};
}
throw new Error(`Caddy proxy has unterminated scoped block: ${marker}`);
}
const route = directiveBlock(block, "route {");
const forward = directiveBlock(route.body, "forward_auth auth-gateway:4180 {");
const forwardAt = route.body.indexOf("forward_auth auth-gateway:4180 {");
for (const [label, publicName, trustedName] of [
["issuer", "X-Thoth-Principal-Issuer", "X-Thoth-Trusted-Principal-Issuer"],
["subject", "X-Thoth-Principal-Subject", "X-Thoth-Trusted-Principal-Subject"],
["display", "X-Thoth-Principal-Display-Name", "X-Thoth-Trusted-Principal-Display-Name"],
["admin", "X-Thoth-Is-Admin", "X-Thoth-Trusted-Is-Admin"],
]) {
if (!block.includes(`request_header -${publicName}`)) {
const publicClearAt = route.body.indexOf(`request_header -${publicName}`);
if (publicClearAt < 0) {
throw new Error(`Caddy proxy does not clear inbound ${label} identity`);
}
if (!block.includes(`request_header -${trustedName}`)) {
const trustedClearAt = route.body.indexOf(`request_header -${trustedName}`);
if (trustedClearAt < 0) {
throw new Error(`Caddy proxy does not clear inbound trusted ${label} identity`);
}
if (!block.includes(`${publicName}>${trustedName}`)) {
if (publicClearAt > forwardAt || trustedClearAt > forwardAt) {
throw new Error("Caddy identity clears must precede forward_auth");
}
if (!forward.body.includes(`${publicName}>${trustedName}`)) {
throw new Error(`Caddy proxy does not map authenticated ${label} identity`);
}
}
const outsideForward = route.body.slice(0, forward.start) + route.body.slice(forward.end + 1);
if (/X-Thoth-(?:Principal-[^\s>]+|Is-Admin)>X-Thoth-Trusted-/.test(outsideForward)) {
throw new Error("Caddy maps identity outside the authenticated response stage");
}
NODE
echo "Caddy reverse-proxy guide contract passed"
}
verify_caddy_adapted_identity_order() {
local adapted="$1"
node - "$adapted" <<'NODE'
const fs = require("fs");
const document = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
const publicHeaders = [
"X-Thoth-Principal-Issuer", "X-Thoth-Principal-Subject",
"X-Thoth-Principal-Display-Name", "X-Thoth-Is-Admin",
];
const trustedHeaders = [
"X-Thoth-Trusted-Principal-Issuer", "X-Thoth-Trusted-Principal-Subject",
"X-Thoth-Trusted-Principal-Display-Name", "X-Thoth-Trusted-Is-Admin",
];
function authUpstream(handler) {
return handler?.handler === "reverse_proxy" &&
(handler.upstreams || []).some((upstream) => upstream.dial === "auth-gateway:4180");
}
function frontendUpstream(handler) {
return handler?.handler === "reverse_proxy" &&
(handler.upstreams || []).some((upstream) => upstream.dial === "127.0.0.1:8080");
}
function findHandlerArray(value) {
if (!value || typeof value !== "object") return null;
if (Array.isArray(value)) {
if (value.some(authUpstream) && value.some(frontendUpstream)) return value;
for (const child of value) {
const found = findHandlerArray(child);
if (found) return found;
}
return null;
}
for (const child of Object.values(value)) {
const found = findHandlerArray(child);
if (found) return found;
}
return null;
}
function collectTrustedSets(value, collected = new Map()) {
if (!value || typeof value !== "object") return collected;
if (value.handler === "headers") {
for (const [name, replacement] of Object.entries(value.request?.set || {})) {
if (trustedHeaders.includes(name)) collected.set(name, replacement);
}
}
for (const child of Object.values(value)) collectTrustedSets(child, collected);
return collected;
}
const handlers = findHandlerArray(document);
if (!handlers) throw new Error("Caddy adapted config lacks the ordered auth/frontend handler chain");
const authAt = handlers.findIndex(authUpstream);
const frontendAt = handlers.findIndex(frontendUpstream);
if (authAt < 0 || frontendAt <= authAt) throw new Error("Caddy adapted auth/frontend handler order is invalid");
const expectedClears = [...publicHeaders, ...trustedHeaders];
for (const header of expectedClears) {
const clearAt = handlers.findIndex((handler) =>
handler?.handler === "headers" && (handler.request?.delete || []).includes(header));
if (clearAt < 0 || clearAt >= authAt) {
throw new Error("Caddy adapted identity clears must execute before authentication");
}
}
const auth = handlers[authAt];
const successResponse = (auth.handle_response || []).find((response) =>
(response.match?.status_code || []).map(Number).includes(2));
if (!successResponse) throw new Error("Caddy adapted identity mapping is not restricted to auth 2xx");
const mappings = collectTrustedSets(successResponse);
for (let index = 0; index < trustedHeaders.length; index++) {
const replacement = mappings.get(trustedHeaders[index]);
const expected = `{http.reverse_proxy.header.${publicHeaders[index]}}`;
if (!Array.isArray(replacement) || replacement.length !== 1 || replacement[0] !== expected) {
throw new Error(`Caddy adapted authenticated mapping is invalid for ${trustedHeaders[index]}`);
}
}
for (let index = 0; index < handlers.length; index++) {
if (index === authAt) continue;
if (collectTrustedSets(handlers[index]).size !== 0) {
throw new Error("Caddy adapted config maps trusted identity outside auth success");
}
}
NODE
}
verify_caddy_effective_proxy_guide() {
local adapted
adapted="$(mktemp "${TMPDIR:-/tmp}/thoth-caddy-adapted.XXXXXX")"
if ! awk '
/^```caddyfile$/ { code=1; next }
code && /^```$/ { exit }
code { print }
' "$root/docs/install/reverse-proxy-caddy.md" \
| docker run --rm -i caddy:2.10.2-alpine caddy adapt --config - --adapter caddyfile >"$adapted"; then
rm -f "$adapted"
echo "Caddy documented configuration could not be adapted" >&2
return 1
fi
verify_caddy_adapted_identity_order "$adapted"
rm -f "$adapted"
echo "Caddy adapted trust-stage contract passed"
}
verify_manual() {
local profile="$1" manual
manual="$root/docs/install/$profile-workspace-registry.md"
@@ -1236,6 +1389,8 @@ case "$mode" in
verify_server_guide
verify_reverse_proxy_nginx_guide
verify_reverse_proxy_caddy_guide
verify_caddy_effective_proxy_guide
"$root/scripts/test-server-operator-permissions.sh"
verify_server_installation_example
fi
verify_manual "$profile"