mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

docs: add autonomous server installation guide

Browse Source
This commit is contained in:
marcopan
2026-08-05 10:22:02 +02:00
parent df21046472
commit a707fb442c
7 changed files with 1056 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/test-verify-workspace-install-docs.sh
+79 -3
View File
@@ -15,7 +15,11 @@ for fixture in \
"source update fail-closed semantics" \
"Windows line-ending recovery guide contract" \
"Pi management guide contract" \
"server installation guide contract" \
"Nginx reverse-proxy guide contract" \
"Caddy reverse-proxy guide contract" \
"local installation example rendered from path with spaces" \
"server installation example rendered from path with spaces" \
"local manual canonical base+override references" \
"server manual canonical base+override references" \
"canonical local base+override fixture" \
@@ -29,9 +33,7 @@ for fixture in \
}
done
for manual in \
"$root/docs/install/local-workspace-registry.md" \
"$root/docs/install/server-workspace-registry.md"; do
for manual in "$root/docs/install/local-workspace-registry.md"; do
grep -Fq 'export THT_SOURCE_ROOT=/absolute/path/to/ThothII' "$manual" || {
echo "installation manual does not publish a self-contained THT_SOURCE_ROOT export: $manual" >&2
exit 1
@@ -46,6 +48,17 @@ for manual in \
fi
done
grep -Fq 'THTCTL=/srv/thothii/operator/thothctl' \
"$root/docs/install/server-workspace-registry.md" || {
echo "server installation manual does not use the installation-aware operator CLI" >&2
exit 1
}
grep -Fq 'INSTALLATION=/srv/thothii/operator/thothii-installation.yaml' \
"$root/docs/install/server-workspace-registry.md" || {
echo "server installation manual does not identify the server installation descriptor" >&2
exit 1
}
if rg -n 'local-compose\.workspace-registry|server-compose\.workspace-registry|connector-secrets\.workspace-registry|docker compose' \
"$root/docs/install/local-workspace-registry.md" \
"$root/docs/install/server-workspace-registry.md"; then
@@ -92,6 +105,33 @@ switch (mutation) {
case "raw-pi":
changed += "\n```sh\ndocker compose exec core pi --version\n```\n";
break;
case "server-secret-env":
changed += "\n```dotenv\nTHT_MODEL_API_KEY=unsafe-secret-value\n```\n";
break;
case "server-docker-socket":
changed += "\nMount /var/run/docker.sock into core for management.\n";
break;
case "server-coupling":
changed += "\nAttach core to the omics_portal application network.\n";
break;
case "nginx-no-auth":
changed = original.replace(" auth_request /_authenticate;", " # authentication omitted");
break;
case "nginx-core-upstream":
changed = original.replaceAll("http://127.0.0.1:8080", "http://127.0.0.1:8787");
break;
case "nginx-no-sse":
changed = original.replace(" proxy_buffering off;", " proxy_buffering on;");
break;
case "caddy-no-auth":
changed = original.replace("forward_auth auth-gateway:4180 {", "# forward authentication omitted");
break;
case "caddy-client-identity":
changed = original.replace("X-Thoth-Principal-Subject>X-Thoth-Trusted-Principal-Subject", "X-Thoth-Principal-Subject");
break;
case "caddy-core-upstream":
changed = original.replaceAll("127.0.0.1:8080", "127.0.0.1:8787");
break;
case "dirty-source":
changed = original.replaceAll("git status --porcelain --untracked-files=all", "git status --short");
break;
@@ -163,6 +203,42 @@ expect_guide_rejected \
"raw non-installation-aware Pi access" verify_pi_management_guide \
"$root/docs/install/pi-management.md" docs/install/pi-management.md raw-pi \
"raw non-installation-aware Compose Pi access is forbidden"
expect_guide_rejected \
"server secret in environment" verify_server_guide \
"$root/docs/install/server.md" docs/install/server.md server-secret-env \
"server installation guide embeds a secret value"
expect_guide_rejected \
"server Docker socket mount" verify_server_guide \
"$root/docs/install/server.md" docs/install/server.md server-docker-socket \
"server installation guide introduces a Docker socket dependency"
expect_guide_rejected \
"server application coupling" verify_server_guide \
"$root/docs/install/server.md" docs/install/server.md server-coupling \
"server installation guide introduces forbidden application coupling"
expect_guide_rejected \
"Nginx identity without authentication" verify_reverse_proxy_nginx_guide \
"$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-auth \
"Nginx proxy lacks structural token: auth_request /_authenticate;"
expect_guide_rejected \
"Nginx direct core exposure" verify_reverse_proxy_nginx_guide \
"$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-core-upstream \
"Nginx proxy must forward only to frontend on 127.0.0.1:8080"
expect_guide_rejected \
"Nginx buffered SSE" verify_reverse_proxy_nginx_guide \
"$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-sse \
"Nginx proxy lacks structural token: proxy_buffering off;"
expect_guide_rejected \
"Caddy identity without authentication" verify_reverse_proxy_caddy_guide \
"$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-auth \
"Caddy proxy lacks structural token: forward_auth auth-gateway:4180 {"
expect_guide_rejected \
"Caddy untrusted identity forwarding" verify_reverse_proxy_caddy_guide \
"$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-client-identity \
"Caddy proxy lacks structural token: X-Thoth-Principal-Subject>X-Thoth-Trusted-Principal-Subject"
expect_guide_rejected \
"Caddy direct core exposure" verify_reverse_proxy_caddy_guide \
"$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-core-upstream \
"Caddy proxy must forward only to frontend on 127.0.0.1:8080"
expect_guide_rejected \
"dirty or untracked source tree" verify_local_guide \
scripts/verify-workspace-install-docs.sh
+325 -5
View File
@@ -488,6 +488,171 @@ NODE
echo "Pi management guide contract passed"
}
verify_server_guide() {
local guide="$root/docs/install/server.md"
[[ -f "$guide" ]] || {
echo "missing server installation guide sections: docs/install/server.md" >&2
return 1
}
require_headings "$guide" "server installation guide" \
"Deployment contract" \
"Service account and directories" \
"Firewall and network boundaries" \
"Address co-resident external services" \
"Prepare operator files and secrets" \
"Build locally or select pinned images" \
"Install thothctl" \
"Start and verify readiness" \
"Configure TLS and upstream authentication" \
"Operate Pi, drain, and roll back" \
"Back up and restore" \
"Diagnostics" \
"Data-preserving uninstall"
require_text "$guide" "server installation guide" \
"frontend" \
"core" \
"UID/GID 10001" \
"/srv/thothii" \
"example operator root" \
"/run/secrets" \
"Git-backed workspace registry is the source of truth" \
"host.docker.internal" \
"host-gateway" \
"container 127.0.0.1" \
"collection" \
"embedding" \
"bash scripts/build-local.sh" \
"@sha256:" \
"bash scripts/build-thothctl.sh" \
"thothctl --installation" \
"curl --fail http://127.0.0.1:8080/health" \
"https://thoth.example.com" \
"pi update" \
"--drain" \
"pi rollback --yes" \
"pi maintenance recover --yes" \
"docker compose down --volumes" \
"reverse-proxy-nginx.md" \
"reverse-proxy-caddy.md"
node - "$guide" <<'NODE'
const fs = require("fs");
const source = fs.readFileSync(process.argv[2], "utf8");
if (/omics_portal|chirone|localllm_default|datamart-builder|compose\.production|compose\.psd-local/i.test(source)) {
throw new Error("server installation guide introduces forbidden application coupling");
}
if (/\/var\/run\/docker\.sock|docker\.sock/i.test(source)) {
throw new Error("server installation guide introduces a Docker socket dependency");
}
for (const line of source.split(/\n/)) {
const match = line.match(/^\s*([A-Z][A-Z0-9_]*(?:PASSWORD|TOKEN|API_KEY|SECRET)[A-Z0-9_]*)\s*=\s*(\S.*)$/);
if (!match) continue;
const [, name, rawValue] = match;
const value = rawValue.trim();
if (!/(?:_FILE|_SOURCE)$/.test(name) && value && !/^\$\{?[A-Z_][A-Z0-9_]*\}?$/.test(value)) {
throw new Error("server installation guide embeds a secret value");
}
}
let inCodeFence = false;
for (const line of source.split(/\n/)) {
if (line.trimStart().startsWith("```")) {
inCodeFence = !inCodeFence;
continue;
}
if (!line.includes("docker compose down --volumes")) continue;
const normalized = line.toLowerCase().replaceAll("*", "");
if (inCodeFence || !/(do not|never)/.test(normalized) || /^\s*(docker|&?\s*docker)/.test(normalized)) {
throw new Error("server docker compose down --volumes must appear only in an explicit prose prohibition");
}
}
if (/```(?:sh|bash)\n[\s\S]*?\bdocker compose\s+(?:up|stop|down|restart|pull|build)\b[\s\S]*?```/i.test(source)) {
throw new Error("server lifecycle must use thothctl, not raw Docker Compose");
}
NODE
echo "server installation guide contract passed"
}
verify_reverse_proxy_nginx_guide() {
local guide="$root/docs/install/reverse-proxy-nginx.md"
[[ -f "$guide" ]] || {
echo "missing Nginx reverse-proxy guide: docs/install/reverse-proxy-nginx.md" >&2
return 1
}
require_headings "$guide" "Nginx reverse-proxy guide" \
"Trust boundary" \
"Example configuration" \
"Validate and reload" \
"Test authentication and SSE"
require_text "$guide" "Nginx reverse-proxy guide" \
"Forwarding identity headers alone does not authenticate a user" \
"authentication gateway" \
"2xx" \
"TLS" \
"frontend"
node - "$guide" <<'NODE'
const fs = require("fs");
const source = fs.readFileSync(process.argv[2], "utf8");
const block = [...source.matchAll(/```nginx\n([\s\S]*?)```/g)].map((match) => match[1]).join("\n");
const tokens = [
"listen 443 ssl;", "ssl_certificate ", "ssl_certificate_key ",
"location = /_authenticate {", "internal;", "proxy_pass http://auth-gateway:4180/verify;",
"auth_request /_authenticate;", "auth_request_set $thoth_principal_subject",
"$upstream_http_x_thoth_principal_subject", "proxy_pass http://127.0.0.1:8080;",
"proxy_http_version 1.1;", "proxy_buffering off;", "proxy_cache off;",
"proxy_read_timeout 3600s;", "proxy_set_header X-Thoth-Principal-Subject \"\";",
"proxy_set_header X-Thoth-Trusted-Principal-Subject $thoth_principal_subject;",
];
if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(block) || !block.includes("http://127.0.0.1:8080")) {
throw new Error("Nginx proxy must forward only to frontend on 127.0.0.1:8080");
}
for (const token of tokens) {
if (!block.includes(token)) throw new Error(`Nginx proxy lacks structural token: ${token}`);
}
if (/proxy_set_header\s+X-Thoth-Trusted-[^;]+\$http_/i.test(block)) {
throw new Error("Nginx proxy trusts a client-supplied identity header");
}
NODE
echo "Nginx reverse-proxy guide contract passed"
}
verify_reverse_proxy_caddy_guide() {
local guide="$root/docs/install/reverse-proxy-caddy.md"
[[ -f "$guide" ]] || {
echo "missing Caddy reverse-proxy guide: docs/install/reverse-proxy-caddy.md" >&2
return 1
}
require_headings "$guide" "Caddy reverse-proxy guide" \
"Trust boundary" \
"Example configuration" \
"Validate and reload" \
"Test authentication and SSE"
require_text "$guide" "Caddy reverse-proxy guide" \
"Forwarding identity headers alone does not authenticate a user" \
"authentication gateway" \
"2xx" \
"automatic HTTPS" \
"frontend"
node - "$guide" <<'NODE'
const fs = require("fs");
const source = fs.readFileSync(process.argv[2], "utf8");
const block = [...source.matchAll(/```caddyfile\n([\s\S]*?)```/g)].map((match) => match[1]).join("\n");
const tokens = [
"thoth.example.com {", "route {",
"request_header -X-Thoth-Principal-Subject",
"request_header -X-Thoth-Trusted-Principal-Subject",
"forward_auth auth-gateway:4180 {", "uri /verify", "copy_headers {",
"X-Thoth-Principal-Subject>X-Thoth-Trusted-Principal-Subject",
"reverse_proxy 127.0.0.1:8080 {", "flush_interval -1",
];
if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(block) || !block.includes("127.0.0.1:8080")) {
throw new Error("Caddy proxy must forward only to frontend on 127.0.0.1:8080");
}
for (const token of tokens) {
if (!block.includes(token)) throw new Error(`Caddy proxy lacks structural token: ${token}`);
}
NODE
echo "Caddy reverse-proxy guide contract passed"
}
verify_manual() {
local profile="$1" manual
manual="$root/docs/install/$profile-workspace-registry.md"
@@ -520,11 +685,27 @@ verify_manual() {
return 1
}
done
for expected in \
'export THT_SOURCE_ROOT=/absolute/path/to/ThothII' \
'--env-file "$THT_OPERATOR_ENV"' \
"-f \"\$THT_SOURCE_ROOT/compose.yaml\" -f \"\$THT_SOURCE_ROOT/deploy/compose.$profile.yaml\"" \
'"$THT_SOURCE_ROOT/scripts/generate-connector-secrets-override.sh"'; do
local -a expected_steps
if [[ "$profile" == local ]]; then
expected_steps=(
'export THT_SOURCE_ROOT=/absolute/path/to/ThothII'
'--env-file "$THT_OPERATOR_ENV"'
"-f \"\$THT_SOURCE_ROOT/compose.yaml\" -f \"\$THT_SOURCE_ROOT/deploy/compose.$profile.yaml\""
'"$THT_SOURCE_ROOT/scripts/generate-connector-secrets-override.sh"'
)
else
expected_steps=(
'THTCTL=/srv/thothii/operator/thothctl'
'INSTALLATION=/srv/thothii/operator/thothii-installation.yaml'
'"$THTCTL" --installation "$INSTALLATION" start'
'"$THTCTL" --installation "$INSTALLATION" doctor'
'docs/install/examples/thothii-installation.server.yaml'
'compose.yaml'
'deploy/compose.server.yaml'
'server.md'
)
fi
for expected in "${expected_steps[@]}"; do
grep -Fq -- "$expected" "$manual" || {
echo "$profile manual lacks canonical operator step: $expected" >&2
return 1
@@ -633,6 +814,136 @@ NODE
echo "local installation example rendered from path with spaces passed"
}
verify_server_installation_example() {
local example="$root/docs/install/examples/thothii-installation.server.yaml"
[[ -f "$example" ]] || {
echo "missing server installation example: docs/install/examples/thothii-installation.server.yaml" >&2
return 1
}
local fixture source_copy operator_dir copied_example connector_override env_file
fixture="$(mktemp -d "${TMPDIR%/}/thoth server install.XXXXXX")"
trap 'rm -rf "$fixture"' RETURN
[[ "$fixture" == *" "* ]] || {
echo "server installation fixture path does not contain spaces" >&2
return 1
}
source_copy="$fixture/ThothII server source"
operator_dir="$fixture/server operator files"
mkdir -p "$source_copy/deploy/pi" "$source_copy/deploy/workspaces" \
"$operator_dir/data" "$operator_dir/pi-state" "$operator_dir/workspace-registry"
cp "$root/compose.yaml" "$source_copy/compose.yaml"
cp "$root/deploy/compose.server.yaml" "$source_copy/deploy/compose.server.yaml"
cp "$root/deploy/compose.session-server.yaml.example" \
"$source_copy/deploy/compose.session-server.yaml.example"
cp "$root/deploy/compose.git-ssh.yaml" "$source_copy/deploy/compose.git-ssh.yaml"
cp "$root/deploy/pi/models.json" "$source_copy/deploy/pi/models.json"
cp "$root/deploy/pi/settings.json" "$source_copy/deploy/pi/settings.json"
cp "$root/deploy/workspaces/server-sessions.yaml.example" \
"$source_copy/deploy/workspaces/server-sessions.yaml.example"
write_private "$operator_dir/pi-auth.json" '{"zai":{"type":"api_key","key":"fixture-server-pi-key"}}'
write_private "$operator_dir/thothii.secrets" 'THT_MODEL_API_KEY=fixture-server-model-key'
write_private "$operator_dir/git-ssh-key" 'fixture-server-ssh-key'
write_private "$operator_dir/git-known-hosts" 'fixture-server-known-hosts'
write_private "$operator_dir/dwh-password" 'fixture-server-dwh-password'
write_private "$operator_dir/session-runtime-password" 'fixture-server-session-runtime-password'
write_private "$operator_dir/session-migrator-password" 'fixture-server-session-migrator-password'
write_private "$operator_dir/session-ca.pem" 'fixture-server-session-ca'
printf '%s\n' \
'THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=postgres_direct' \
'THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password' \
>"$operator_dir/workspace-bindings.env"
env_file="$operator_dir/server.env"
printf '%s\n' \
'THOTH_SERVER_BIND=127.0.0.1' \
'THOTH_HTTP_PORT=8080' \
'THT_WORKSPACE_GIT_REMOTE=ssh://git@git.example.invalid/platform/thoth-workspaces.git' \
'THT_WORKSPACE_GIT_BRANCH=main' \
"PI_AUTH_FILE=$operator_dir/pi-auth.json" \
"THT_SECRETS_FILE=$operator_dir/thothii.secrets" \
"THT_WORKSPACE_BINDINGS_ENV_FILE=$operator_dir/workspace-bindings.env" \
"THT_WORKSPACE_GIT_SSH_KEY_FILE=$operator_dir/git-ssh-key" \
"THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=$operator_dir/git-known-hosts" \
"THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=$operator_dir/dwh-password" \
"THT_DATA_ROOT=$operator_dir/data" \
"THT_PI_STATE_ROOT=$operator_dir/pi-state" \
"THT_WORKSPACE_REGISTRY_ROOT=$operator_dir/workspace-registry" \
"THT_SERVER_WORKSPACE_CONFIG=$source_copy/deploy/workspaces/server-sessions.yaml.example" \
'THT_LLM_URL=https://llm.example.invalid' \
'THT_SESSION_DB_HOST=sessions.example.invalid' \
'THT_SESSION_DB_NAME=thoth_sessions' \
'THT_SESSION_RUNTIME_USER=thoth_sessions_app' \
'THT_SESSION_MIGRATOR_USER=thoth_sessions_migrate' \
"THT_SESSION_RUNTIME_PASSWORD_SOURCE=$operator_dir/session-runtime-password" \
"THT_SESSION_MIGRATOR_PASSWORD_SOURCE=$operator_dir/session-migrator-password" \
"THT_SESSION_CA_SOURCE=$operator_dir/session-ca.pem" \
>"$env_file"
connector_override="$operator_dir/connector-secrets.server.yaml"
"$root/scripts/generate-connector-secrets-override.sh" \
--bindings-env "$operator_dir/workspace-bindings.env" \
--operator-env "$env_file" \
--output "$connector_override" >/dev/null
copied_example="$fixture/thothii-installation.yaml"
local contents
contents="$(<"$example")"
contents="${contents//\/absolute\/path\/to\/ThothII/$source_copy}"
contents="${contents//\/absolute\/path\/to\/thothii-server-operator/$operator_dir}"
printf '%s\n' "$contents" >"$copied_example"
local profile project_directory descriptor_env value
local -a overrides files
profile="$(sed -n 's/^profile: \([^[:space:]]*\)$/\1/p' "$copied_example")"
project_directory="$(sed -n 's/^projectDirectory: "\(.*\)"$/\1/p' "$copied_example")"
descriptor_env="$(sed -n 's/^envFile: "\(.*\)"$/\1/p' "$copied_example")"
while IFS= read -r value; do overrides+=("$value"); done < <(sed -n 's/^ - "\(.*\)"$/\1/p' "$copied_example")
[[ "$profile" == server && "$project_directory" == "$source_copy" && "$descriptor_env" == "$env_file" ]] || {
echo "server installation example does not resolve its required fields" >&2
return 1
}
[[ "${#overrides[@]}" -eq 3 && "${overrides[0]}" == "$source_copy/deploy/compose.session-server.yaml.example" \
&& "${overrides[2]}" == "$connector_override" ]] || {
echo "server installation example does not select the expected optional overrides" >&2
return 1
}
files=(-f "$project_directory/compose.yaml" -f "$project_directory/deploy/compose.$profile.yaml")
for value in "${overrides[@]}"; do files+=(-f "$value"); done
local rendered="$fixture/server-installation.json"
"$root/scripts/compose-with-preflight.sh" --env-file "$descriptor_env" \
"${files[@]}" config --format json >"$rendered"
node - "$rendered" <<'NODE'
const fs = require("fs");
const config = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
if (Object.keys(config.services).sort().join(",") !== "core,frontend") {
throw new Error("server installation example must render exactly core,frontend");
}
const core = config.services.core;
const frontend = config.services.frontend;
if (core.environment?.AUTH_MODE !== "upstream" || core.environment?.THOTH_PUBLIC_EXPOSURE !== "true") {
throw new Error("server installation example must fail closed behind upstream authentication");
}
if ((core.ports || []).length !== 0) throw new Error("server installation example published core");
const ports = frontend.ports || [];
if (ports.length !== 1 || ports[0].host_ip !== "127.0.0.1" || Number(ports[0].target) !== 8080) {
throw new Error("server installation example must publish only loopback frontend");
}
const rendered = JSON.stringify(config);
if (/omics_portal|chirone|localllm_default|datamart-builder/i.test(rendered)) {
throw new Error("server installation example contains application coupling");
}
for (const secret of [
"fixture-server-pi-key", "fixture-server-model-key", "fixture-server-ssh-key",
"fixture-server-known-hosts", "fixture-server-dwh-password",
"fixture-server-session-runtime-password", "fixture-server-session-migrator-password",
"fixture-server-session-ca",
]) {
if (rendered.includes(secret)) throw new Error("server installation rendering exposed a fixture secret");
}
NODE
echo "server installation example rendered from path with spaces passed"
}
write_private() {
local path="$1" value="$2"
printf '%s\n' "$value" >"$path"
@@ -772,7 +1083,11 @@ case "$mode" in
verify_local_guide
verify_windows_line_endings_guide
verify_pi_management_guide
verify_server_guide
verify_reverse_proxy_nginx_guide
verify_reverse_proxy_caddy_guide
verify_local_installation_example
verify_server_installation_example
verify_manual local
verify_manual server
verify_compose_fixtures
@@ -786,6 +1101,11 @@ case "$mode" in
verify_windows_line_endings_guide
verify_pi_management_guide
verify_local_installation_example
else
verify_server_guide
verify_reverse_proxy_nginx_guide
verify_reverse_proxy_caddy_guide
verify_server_installation_example
fi
verify_manual "$profile"
verify_compose_fixtures