mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix: make server operations executable

Browse Source
This commit is contained in:
marcopan
2026-08-05 11:06:34 +02:00
parent a707fb442c
commit 96fe5bfa79
11 changed files with 1299 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/test-verify-workspace-install-docs.sh
+113 -1
View File
@@ -20,6 +20,8 @@ for fixture in \
"Caddy reverse-proxy guide contract" \
"local installation example rendered from path with spaces" \
"server installation example rendered from path with spaces" \
"server pinned migration image fixture" \
"server backup checksum root-only fixture" \
"local manual canonical base+override references" \
"server manual canonical base+override references" \
"canonical local base+override fixture" \
@@ -33,6 +35,32 @@ for fixture in \
}
done
server_guide="$root/docs/install/server.md"
for required in \
'thothii-ops' \
'THT_BACKUP_ROOT=/srv/thothii-backups' \
'sessions migrate --yes' \
'"pending":[]' \
'"drifted":[]' \
'remove --yes' \
'sha256sum --check SHA256SUMS' \
'DOCKER-USER' \
'iptables -I INPUT' \
'com.docker.network.bridge.name'; do
grep -Fq -- "$required" "$server_guide" || {
echo "server operations guide lacks executable contract: $required" >&2
exit 1
}
done
grep -Fq '"$THTCTL" --help' "$server_guide" || {
echo "server guide lacks plain thothctl --help" >&2
exit 1
}
if grep -Fq '"$THTCTL" --installation "$INSTALLATION" --help' "$server_guide"; then
echo "server guide still uses installation-scoped --help" >&2
exit 1
fi
for manual in "$root/docs/install/local-workspace-registry.md"; do
grep -Fq 'export THT_SOURCE_ROOT=/absolute/path/to/ThothII' "$manual" || {
echo "installation manual does not publish a self-contained THT_SOURCE_ROOT export: $manual" >&2
@@ -114,6 +142,18 @@ switch (mutation) {
case "server-coupling":
changed += "\nAttach core to the omics_portal application network.\n";
break;
case "server-host-loopback":
changed += "\nFor host-gateway, keep the external service listening on 127.0.0.1.\n";
break;
case "server-raw-remove":
changed += "\n```sh\ndocker rm thothii-core thothii-frontend\n```\n";
break;
case "server-pinned-migrator-mismatch":
changed += "\n```yaml\nservices:\n core:\n image: registry.invalid/core@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n session-migrate:\n image: thothii-core:local\n```\n";
break;
case "server-pinned-frontend-missing":
changed = original.replace(' frontend:\n build: !reset null\n image: registry.example.com/thothii/frontend@sha256:<64-lowercase-hex-digits>\n', '');
break;
case "nginx-no-auth":
changed = original.replace(" auth_request /_authenticate;", " # authentication omitted");
break;
@@ -123,6 +163,18 @@ switch (mutation) {
case "nginx-no-sse":
changed = original.replace(" proxy_buffering off;", " proxy_buffering on;");
break;
case "nginx-no-issuer-clear":
changed = original.replaceAll('proxy_set_header X-Thoth-Principal-Issuer "";', 'proxy_set_header X-Thoth-Principal-Issuer $http_x_thoth_principal_issuer;');
break;
case "nginx-no-subject-capture":
changed = original.replace("auth_request_set $thoth_principal_subject", "# missing auth capture $thoth_principal_subject");
break;
case "nginx-no-display-map":
changed = original.replace("proxy_set_header X-Thoth-Trusted-Principal-Display-Name $thoth_principal_display_name;", "proxy_set_header X-Thoth-Trusted-Principal-Display-Name \"\";");
break;
case "nginx-no-admin-map":
changed = original.replace("proxy_set_header X-Thoth-Trusted-Is-Admin $thoth_is_admin;", "proxy_set_header X-Thoth-Trusted-Is-Admin \"\";");
break;
case "caddy-no-auth":
changed = original.replace("forward_auth auth-gateway:4180 {", "# forward authentication omitted");
break;
@@ -132,6 +184,18 @@ switch (mutation) {
case "caddy-core-upstream":
changed = original.replaceAll("127.0.0.1:8080", "127.0.0.1:8787");
break;
case "caddy-no-issuer-public-clear":
changed = original.replace("request_header -X-Thoth-Principal-Issuer", "request_header X-Thoth-Principal-Issuer {header.X-Thoth-Principal-Issuer}");
break;
case "caddy-no-subject-trusted-clear":
changed = original.replace("request_header -X-Thoth-Trusted-Principal-Subject", "request_header X-Thoth-Trusted-Principal-Subject {header.X-Thoth-Trusted-Principal-Subject}");
break;
case "caddy-no-display-map":
changed = original.replace("X-Thoth-Principal-Display-Name>X-Thoth-Trusted-Principal-Display-Name", "X-Thoth-Principal-Display-Name");
break;
case "caddy-no-admin-map":
changed = original.replace("X-Thoth-Is-Admin>X-Thoth-Trusted-Is-Admin", "X-Thoth-Is-Admin");
break;
case "dirty-source":
changed = original.replaceAll("git status --porcelain --untracked-files=all", "git status --short");
break;
@@ -215,6 +279,22 @@ expect_guide_rejected \
"server application coupling" verify_server_guide \
"$root/docs/install/server.md" docs/install/server.md server-coupling \
"server installation guide introduces forbidden application coupling"
expect_guide_rejected \
"server host-gateway loopback listener" verify_server_guide \
"$root/docs/install/server.md" docs/install/server.md server-host-loopback \
"server host-gateway guidance assumes a host loopback listener"
expect_guide_rejected \
"server raw container removal" verify_server_guide \
"$root/docs/install/server.md" docs/install/server.md server-raw-remove \
"server uninstall bypasses installation-aware removal"
expect_guide_rejected \
"server pinned migrator differs from core" verify_server_guide \
"$root/docs/install/server.md" docs/install/server.md server-pinned-migrator-mismatch \
"server pinned migration image must equal the pinned core image"
expect_guide_rejected \
"server pinned frontend is missing" verify_server_guide \
"$root/docs/install/server.md" docs/install/server.md server-pinned-frontend-missing \
"server pinned image override must pin core, session-migrate, and frontend without builds"
expect_guide_rejected \
"Nginx identity without authentication" verify_reverse_proxy_nginx_guide \
"$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-auth \
@@ -227,6 +307,22 @@ expect_guide_rejected \
"Nginx buffered SSE" verify_reverse_proxy_nginx_guide \
"$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-sse \
"Nginx proxy lacks structural token: proxy_buffering off;"
expect_guide_rejected \
"Nginx issuer inbound claim not cleared" verify_reverse_proxy_nginx_guide \
"$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-issuer-clear \
"Nginx proxy does not clear inbound issuer identity"
expect_guide_rejected \
"Nginx subject auth response not captured" verify_reverse_proxy_nginx_guide \
"$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-subject-capture \
"Nginx proxy does not capture authenticated subject identity"
expect_guide_rejected \
"Nginx display identity not mapped to private hop" verify_reverse_proxy_nginx_guide \
"$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-display-map \
"Nginx proxy does not map authenticated display identity"
expect_guide_rejected \
"Nginx admin identity not mapped to private hop" verify_reverse_proxy_nginx_guide \
"$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-admin-map \
"Nginx proxy does not map authenticated admin identity"
expect_guide_rejected \
"Caddy identity without authentication" verify_reverse_proxy_caddy_guide \
"$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-auth \
@@ -234,11 +330,27 @@ expect_guide_rejected \
expect_guide_rejected \
"Caddy untrusted identity forwarding" verify_reverse_proxy_caddy_guide \
"$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-client-identity \
"Caddy proxy lacks structural token: X-Thoth-Principal-Subject>X-Thoth-Trusted-Principal-Subject"
"Caddy proxy does not map authenticated subject identity"
expect_guide_rejected \
"Caddy direct core exposure" verify_reverse_proxy_caddy_guide \
"$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-core-upstream \
"Caddy proxy must forward only to frontend on 127.0.0.1:8080"
expect_guide_rejected \
"Caddy issuer inbound claim not cleared" verify_reverse_proxy_caddy_guide \
"$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-issuer-public-clear \
"Caddy proxy does not clear inbound issuer identity"
expect_guide_rejected \
"Caddy subject private-hop claim not cleared" verify_reverse_proxy_caddy_guide \
"$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-subject-trusted-clear \
"Caddy proxy does not clear inbound trusted subject identity"
expect_guide_rejected \
"Caddy display identity not mapped to private hop" verify_reverse_proxy_caddy_guide \
"$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-display-map \
"Caddy proxy does not map authenticated display identity"
expect_guide_rejected \
"Caddy admin identity not mapped to private hop" verify_reverse_proxy_caddy_guide \
"$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-admin-map \
"Caddy proxy does not map authenticated admin identity"
expect_guide_rejected \
"dirty or untracked source tree" verify_local_guide \
scripts/verify-workspace-install-docs.sh
+140 -9
View File
@@ -512,12 +512,16 @@ verify_server_guide() {
"frontend" \
"core" \
"UID/GID 10001" \
"thothii-ops" \
"/srv/thothii" \
"example operator root" \
"/run/secrets" \
"Git-backed workspace registry is the source of truth" \
"host.docker.internal" \
"host-gateway" \
"com.docker.network.bridge.name" \
"DOCKER-USER" \
"iptables -I INPUT" \
"container 127.0.0.1" \
"collection" \
"embedding" \
@@ -525,6 +529,12 @@ verify_server_guide() {
"@sha256:" \
"bash scripts/build-thothctl.sh" \
"thothctl --installation" \
"sessions migrate --yes" \
'"pending":[]' \
'"drifted":[]' \
"remove --yes" \
"THT_BACKUP_ROOT=/srv/thothii-backups" \
"sha256sum --check SHA256SUMS" \
"curl --fail http://127.0.0.1:8080/health" \
"https://thoth.example.com" \
"pi update" \
@@ -564,6 +574,36 @@ for (const line of source.split(/\n/)) {
throw new Error("server docker compose down --volumes must appear only in an explicit prose prohibition");
}
}
if (/```(?:sh|bash)\n[\s\S]*?\bdocker\s+rm\b[\s\S]*?```/i.test(source)) {
throw new Error("server uninstall bypasses installation-aware removal");
}
if (/host-gateway[^\n]{0,120}(?:listen|listening|bound)[^\n]{0,80}127\.0\.0\.1|(?:listen|listening|bound)[^\n]{0,80}127\.0\.0\.1[^\n]{0,120}host-gateway/i.test(source)) {
throw new Error("server host-gateway guidance assumes a host loopback listener");
}
const pinnedStart = source.indexOf("## Build locally or select pinned images");
const pinnedEnd = source.indexOf("\n## ", pinnedStart + 3);
const pinnedSection = source.slice(pinnedStart, pinnedEnd < 0 ? source.length : pinnedEnd);
const pinnedBlock = [...pinnedSection.matchAll(/```yaml\n([\s\S]*?)```/g)].map((match) => match[1])
.find((block) => block.includes("session-migrate:")) || "";
function pinnedService(name) {
const match = pinnedBlock.match(new RegExp(`^ ${name}:\\n((?: [^\\n]*\\n)+)`, "m"));
return match ? match[1] : "";
}
const pinnedCore = pinnedService("core");
const pinnedMigrator = pinnedService("session-migrate");
const pinnedFrontend = pinnedService("frontend");
const coreImage = pinnedCore.match(/image:\s*(\S+)/)?.[1];
const migratorImage = pinnedMigrator.match(/image:\s*(\S+)/)?.[1];
const frontendImage = pinnedFrontend.match(/image:\s*(\S+)/)?.[1];
if (![pinnedCore, pinnedMigrator, pinnedFrontend].every((block) => block.includes("build: !reset null")) ||
!coreImage || coreImage !== migratorImage || !/@sha256:<64-lowercase-hex-digits>$/.test(coreImage) ||
!frontendImage || !/@sha256:<64-lowercase-hex-digits>$/.test(frontendImage)) {
throw new Error("server pinned image override must pin core, session-migrate, and frontend without builds");
}
if (/session-migrate:[\s\S]{0,180}image:\s*thothii-core:local/.test(source) &&
/core:[\s\S]{0,180}image:\s*registry\.[^\n]+@sha256:[a-f0-9]{64}/.test(source)) {
throw new Error("server pinned migration image must equal the pinned core image");
}
if (/```(?:sh|bash)\n[\s\S]*?\bdocker compose\s+(?:up|stop|down|restart|pull|build)\b[\s\S]*?```/i.test(source)) {
throw new Error("server lifecycle must use thothctl, not raw Docker Compose");
}
@@ -595,11 +635,9 @@ const block = [...source.matchAll(/```nginx\n([\s\S]*?)```/g)].map((match) => ma
const tokens = [
"listen 443 ssl;", "ssl_certificate ", "ssl_certificate_key ",
"location = /_authenticate {", "internal;", "proxy_pass http://auth-gateway:4180/verify;",
"auth_request /_authenticate;", "auth_request_set $thoth_principal_subject",
"$upstream_http_x_thoth_principal_subject", "proxy_pass http://127.0.0.1:8080;",
"auth_request /_authenticate;", "proxy_pass http://127.0.0.1:8080;",
"proxy_http_version 1.1;", "proxy_buffering off;", "proxy_cache off;",
"proxy_read_timeout 3600s;", "proxy_set_header X-Thoth-Principal-Subject \"\";",
"proxy_set_header X-Thoth-Trusted-Principal-Subject $thoth_principal_subject;",
"proxy_read_timeout 3600s;",
];
if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(block) || !block.includes("http://127.0.0.1:8080")) {
throw new Error("Nginx proxy must forward only to frontend on 127.0.0.1:8080");
@@ -610,6 +648,28 @@ for (const token of tokens) {
if (/proxy_set_header\s+X-Thoth-Trusted-[^;]+\$http_/i.test(block)) {
throw new Error("Nginx proxy trusts a client-supplied identity header");
}
const identities = [
["issuer", "Principal-Issuer", "thoth_principal_issuer", "x_thoth_principal_issuer"],
["subject", "Principal-Subject", "thoth_principal_subject", "x_thoth_principal_subject"],
["display", "Principal-Display-Name", "thoth_principal_display_name", "x_thoth_principal_display_name"],
["admin", "Is-Admin", "thoth_is_admin", "x_thoth_is_admin"],
];
function escaped(value) { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); }
for (const [label, publicName, variable, upstream] of identities) {
const trustedName = publicName === "Is-Admin" ? "Is-Admin" : publicName;
const publicClears = block.match(new RegExp(`proxy_set_header\\s+X-Thoth-${escaped(publicName)}\\s+"";`, "g")) || [];
if (publicClears.length < 2) throw new Error(`Nginx proxy does not clear inbound ${label} identity`);
const trustedHeader = `X-Thoth-Trusted-${trustedName}`;
if (!new RegExp(`proxy_set_header\\s+${escaped(trustedHeader)}\\s+"";`).test(block)) {
throw new Error(`Nginx proxy does not clear inbound trusted ${label} identity`);
}
if (!new RegExp(`auth_request_set\\s+\\$${variable}\\s+\\$upstream_http_${upstream};`, "m").test(block.replace(/\s+/g, " "))) {
throw new Error(`Nginx proxy does not capture authenticated ${label} identity`);
}
if (!new RegExp(`proxy_set_header\\s+${escaped(trustedHeader)}\\s+\\$${variable};`).test(block)) {
throw new Error(`Nginx proxy does not map authenticated ${label} identity`);
}
}
NODE
echo "Nginx reverse-proxy guide contract passed"
}
@@ -637,10 +697,7 @@ const source = fs.readFileSync(process.argv[2], "utf8");
const block = [...source.matchAll(/```caddyfile\n([\s\S]*?)```/g)].map((match) => match[1]).join("\n");
const tokens = [
"thoth.example.com {", "route {",
"request_header -X-Thoth-Principal-Subject",
"request_header -X-Thoth-Trusted-Principal-Subject",
"forward_auth auth-gateway:4180 {", "uri /verify", "copy_headers {",
"X-Thoth-Principal-Subject>X-Thoth-Trusted-Principal-Subject",
"reverse_proxy 127.0.0.1:8080 {", "flush_interval -1",
];
if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(block) || !block.includes("127.0.0.1:8080")) {
@@ -649,6 +706,22 @@ if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(block) || !block.includes("127.0.0.1:
for (const token of tokens) {
if (!block.includes(token)) throw new Error(`Caddy proxy lacks structural token: ${token}`);
}
for (const [label, publicName, trustedName] of [
["issuer", "X-Thoth-Principal-Issuer", "X-Thoth-Trusted-Principal-Issuer"],
["subject", "X-Thoth-Principal-Subject", "X-Thoth-Trusted-Principal-Subject"],
["display", "X-Thoth-Principal-Display-Name", "X-Thoth-Trusted-Principal-Display-Name"],
["admin", "X-Thoth-Is-Admin", "X-Thoth-Trusted-Is-Admin"],
]) {
if (!block.includes(`request_header -${publicName}`)) {
throw new Error(`Caddy proxy does not clear inbound ${label} identity`);
}
if (!block.includes(`request_header -${trustedName}`)) {
throw new Error(`Caddy proxy does not clear inbound trusted ${label} identity`);
}
if (!block.includes(`${publicName}>${trustedName}`)) {
throw new Error(`Caddy proxy does not map authenticated ${label} identity`);
}
}
NODE
echo "Caddy reverse-proxy guide contract passed"
}
@@ -821,7 +894,7 @@ verify_server_installation_example() {
return 1
}
local fixture source_copy operator_dir copied_example connector_override env_file
local fixture source_copy operator_dir copied_example connector_override env_file backup_root
fixture="$(mktemp -d "${TMPDIR%/}/thoth server install.XXXXXX")"
trap 'rm -rf "$fixture"' RETURN
[[ "$fixture" == *" "* ]] || {
@@ -830,8 +903,9 @@ verify_server_installation_example() {
}
source_copy="$fixture/ThothII server source"
operator_dir="$fixture/server operator files"
backup_root="$fixture/server backups"
mkdir -p "$source_copy/deploy/pi" "$source_copy/deploy/workspaces" \
"$operator_dir/data" "$operator_dir/pi-state" "$operator_dir/workspace-registry"
"$operator_dir/data" "$operator_dir/pi-state" "$operator_dir/workspace-registry" "$backup_root"
cp "$root/compose.yaml" "$source_copy/compose.yaml"
cp "$root/deploy/compose.server.yaml" "$source_copy/deploy/compose.server.yaml"
cp "$root/deploy/compose.session-server.yaml.example" \
@@ -869,6 +943,7 @@ verify_server_installation_example() {
"THT_DATA_ROOT=$operator_dir/data" \
"THT_PI_STATE_ROOT=$operator_dir/pi-state" \
"THT_WORKSPACE_REGISTRY_ROOT=$operator_dir/workspace-registry" \
"THT_BACKUP_ROOT=$backup_root" \
"THT_SERVER_WORKSPACE_CONFIG=$source_copy/deploy/workspaces/server-sessions.yaml.example" \
'THT_LLM_URL=https://llm.example.invalid' \
'THT_SESSION_DB_HOST=sessions.example.invalid' \
@@ -942,6 +1017,62 @@ for (const secret of [
}
NODE
echo "server installation example rendered from path with spaces passed"
local migration_rendered="$fixture/server-migration.json"
"$root/scripts/compose-with-preflight.sh" --env-file "$descriptor_env" \
"${files[@]}" --profile session-migrate config --format json >"$migration_rendered"
node - "$migration_rendered" <<'NODE'
const fs = require("fs");
const config = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
const services = config.services || {};
if (!services.core || !services["session-migrate"]) throw new Error("server migration profile is missing core or session-migrate");
if (services.core.image !== services["session-migrate"].image) throw new Error("source migration image differs from core");
if (services["session-migrate"].build) throw new Error("source migration service unexpectedly declares a build");
NODE
local pinned_template="$fixture/pinned-template.yaml" pinned_override="$operator_dir/pinned-images.yaml"
awk '
/^## Build locally or select pinned images$/ { section=1; next }
section && /^```yaml$/ { code=1; next }
code && /^```$/ { exit }
code { print }
' "$root/docs/install/server.md" >"$pinned_template"
sed \
-e "s#registry.example.com/thothii/core@sha256:<64-lowercase-hex-digits>#registry.example.com/thothii/core@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa#g" \
-e "s#registry.example.com/thothii/frontend@sha256:<64-lowercase-hex-digits>#registry.example.com/thothii/frontend@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb#g" \
"$pinned_template" >"$pinned_override"
chmod 0600 "$pinned_override"
local pinned_rendered="$fixture/server-pinned-migration.json"
"$root/scripts/compose-with-preflight.sh" --env-file "$descriptor_env" \
"${files[@]}" -f "$pinned_override" --profile session-migrate config --format json >"$pinned_rendered"
node - "$pinned_rendered" <<'NODE'
const fs = require("fs");
const config = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
const core = config.services?.core;
const frontend = config.services?.frontend;
const migrator = config.services?.["session-migrate"];
if (!core || !frontend || !migrator) throw new Error("pinned migration profile lacks core, frontend, or session-migrate");
if (core.image !== migrator.image || !/@sha256:[a-f0-9]{64}$/.test(core.image)) {
throw new Error("pinned migration image does not equal the exact core digest");
}
if (!/@sha256:[a-f0-9]{64}$/.test(frontend.image)) throw new Error("frontend is not pinned by exact digest");
for (const [name, service] of Object.entries({core, frontend, migrator})) {
if (service.build) throw new Error(name + " retained a local build in pinned mode");
if (/:local$/.test(service.image || "")) throw new Error(name + " retained a local image in pinned mode");
}
NODE
echo "server pinned migration image fixture passed"
local checksum_root="$fixture/root-only-checksum"
mkdir -m 0700 "$checksum_root"
printf 'fixture backup bytes\n' >"$checksum_root/runtime-data.tgz"
/bin/sh -ceu 'cd "$1"; sha256sum runtime-data.tgz > SHA256SUMS; sha256sum --check SHA256SUMS' sh "$checksum_root" >/dev/null
printf 'corruption\n' >>"$checksum_root/runtime-data.tgz"
if (cd "$checksum_root" && sha256sum --check SHA256SUMS) >/dev/null 2>&1; then
echo "corrupted server backup checksum fixture was accepted" >&2
return 1
fi
echo "server backup checksum root-only fixture passed"
}
write_private() {