docs: add autonomous server installation guide

2026-08-05 10:22:02 +02:00
parent df21046472
commit a707fb442c
7 changed files with 1056 additions and 53 deletions
@@ -488,6 +488,171 @@ NODE
  echo "Pi management guide contract passed"
 }

+verify_server_guide() {
+  local guide="$root/docs/install/server.md"
+  [[ -f "$guide" ]] || {
+    echo "missing server installation guide sections: docs/install/server.md" >&2
+    return 1
+  }
+  require_headings "$guide" "server installation guide" \
+    "Deployment contract" \
+    "Service account and directories" \
+    "Firewall and network boundaries" \
+    "Address co-resident external services" \
+    "Prepare operator files and secrets" \
+    "Build locally or select pinned images" \
+    "Install thothctl" \
+    "Start and verify readiness" \
+    "Configure TLS and upstream authentication" \
+    "Operate Pi, drain, and roll back" \
+    "Back up and restore" \
+    "Diagnostics" \
+    "Data-preserving uninstall"
+  require_text "$guide" "server installation guide" \
+    "frontend" \
+    "core" \
+    "UID/GID 10001" \
+    "/srv/thothii" \
+    "example operator root" \
+    "/run/secrets" \
+    "Git-backed workspace registry is the source of truth" \
+    "host.docker.internal" \
+    "host-gateway" \
+    "container 127.0.0.1" \
+    "collection" \
+    "embedding" \
+    "bash scripts/build-local.sh" \
+    "@sha256:" \
+    "bash scripts/build-thothctl.sh" \
+    "thothctl --installation" \
+    "curl --fail http://127.0.0.1:8080/health" \
+    "https://thoth.example.com" \
+    "pi update" \
+    "--drain" \
+    "pi rollback --yes" \
+    "pi maintenance recover --yes" \
+    "docker compose down --volumes" \
+    "reverse-proxy-nginx.md" \
+    "reverse-proxy-caddy.md"
+  node - "$guide" <<'NODE'
+const fs = require("fs");
+const source = fs.readFileSync(process.argv[2], "utf8");
+if (/omics_portal|chirone|localllm_default|datamart-builder|compose\.production|compose\.psd-local/i.test(source)) {
+  throw new Error("server installation guide introduces forbidden application coupling");
+}
+if (/\/var\/run\/docker\.sock|docker\.sock/i.test(source)) {
+  throw new Error("server installation guide introduces a Docker socket dependency");
+}
+for (const line of source.split(/\n/)) {
+  const match = line.match(/^\s*([A-Z][A-Z0-9_]*(?:PASSWORD|TOKEN|API_KEY|SECRET)[A-Z0-9_]*)\s*=\s*(\S.*)$/);
+  if (!match) continue;
+  const [, name, rawValue] = match;
+  const value = rawValue.trim();
+  if (!/(?:_FILE|_SOURCE)$/.test(name) && value && !/^\$\{?[A-Z_][A-Z0-9_]*\}?$/.test(value)) {
+    throw new Error("server installation guide embeds a secret value");
+  }
+}
+let inCodeFence = false;
+for (const line of source.split(/\n/)) {
+  if (line.trimStart().startsWith("```")) {
+    inCodeFence = !inCodeFence;
+    continue;
+  }
+  if (!line.includes("docker compose down --volumes")) continue;
+  const normalized = line.toLowerCase().replaceAll("*", "");
+  if (inCodeFence || !/(do not|never)/.test(normalized) || /^\s*(docker|&?\s*docker)/.test(normalized)) {
+    throw new Error("server docker compose down --volumes must appear only in an explicit prose prohibition");
+  }
+}
+if (/```(?:sh|bash)\n[\s\S]*?\bdocker compose\s+(?:up|stop|down|restart|pull|build)\b[\s\S]*?```/i.test(source)) {
+  throw new Error("server lifecycle must use thothctl, not raw Docker Compose");
+}
+NODE
+  echo "server installation guide contract passed"
+}
+
+verify_reverse_proxy_nginx_guide() {
+  local guide="$root/docs/install/reverse-proxy-nginx.md"
+  [[ -f "$guide" ]] || {
+    echo "missing Nginx reverse-proxy guide: docs/install/reverse-proxy-nginx.md" >&2
+    return 1
+  }
+  require_headings "$guide" "Nginx reverse-proxy guide" \
+    "Trust boundary" \
+    "Example configuration" \
+    "Validate and reload" \
+    "Test authentication and SSE"
+  require_text "$guide" "Nginx reverse-proxy guide" \
+    "Forwarding identity headers alone does not authenticate a user" \
+    "authentication gateway" \
+    "2xx" \
+    "TLS" \
+    "frontend"
+  node - "$guide" <<'NODE'
+const fs = require("fs");
+const source = fs.readFileSync(process.argv[2], "utf8");
+const block = [...source.matchAll(/```nginx\n([\s\S]*?)```/g)].map((match) => match[1]).join("\n");
+const tokens = [
+  "listen 443 ssl;", "ssl_certificate ", "ssl_certificate_key ",
+  "location = /_authenticate {", "internal;", "proxy_pass http://auth-gateway:4180/verify;",
+  "auth_request /_authenticate;", "auth_request_set $thoth_principal_subject",
+  "$upstream_http_x_thoth_principal_subject", "proxy_pass http://127.0.0.1:8080;",
+  "proxy_http_version 1.1;", "proxy_buffering off;", "proxy_cache off;",
+  "proxy_read_timeout 3600s;", "proxy_set_header X-Thoth-Principal-Subject \"\";",
+  "proxy_set_header X-Thoth-Trusted-Principal-Subject $thoth_principal_subject;",
+];
+if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(block) || !block.includes("http://127.0.0.1:8080")) {
+  throw new Error("Nginx proxy must forward only to frontend on 127.0.0.1:8080");
+}
+for (const token of tokens) {
+  if (!block.includes(token)) throw new Error(`Nginx proxy lacks structural token: ${token}`);
+}
+if (/proxy_set_header\s+X-Thoth-Trusted-[^;]+\$http_/i.test(block)) {
+  throw new Error("Nginx proxy trusts a client-supplied identity header");
+}
+NODE
+  echo "Nginx reverse-proxy guide contract passed"
+}
+
+verify_reverse_proxy_caddy_guide() {
+  local guide="$root/docs/install/reverse-proxy-caddy.md"
+  [[ -f "$guide" ]] || {
+    echo "missing Caddy reverse-proxy guide: docs/install/reverse-proxy-caddy.md" >&2
+    return 1
+  }
+  require_headings "$guide" "Caddy reverse-proxy guide" \
+    "Trust boundary" \
+    "Example configuration" \
+    "Validate and reload" \
+    "Test authentication and SSE"
+  require_text "$guide" "Caddy reverse-proxy guide" \
+    "Forwarding identity headers alone does not authenticate a user" \
+    "authentication gateway" \
+    "2xx" \
+    "automatic HTTPS" \
+    "frontend"
+  node - "$guide" <<'NODE'
+const fs = require("fs");
+const source = fs.readFileSync(process.argv[2], "utf8");
+const block = [...source.matchAll(/```caddyfile\n([\s\S]*?)```/g)].map((match) => match[1]).join("\n");
+const tokens = [
+  "thoth.example.com {", "route {",
+  "request_header -X-Thoth-Principal-Subject",
+  "request_header -X-Thoth-Trusted-Principal-Subject",
+  "forward_auth auth-gateway:4180 {", "uri /verify", "copy_headers {",
+  "X-Thoth-Principal-Subject>X-Thoth-Trusted-Principal-Subject",
+  "reverse_proxy 127.0.0.1:8080 {", "flush_interval -1",
+];
+if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(block) || !block.includes("127.0.0.1:8080")) {
+  throw new Error("Caddy proxy must forward only to frontend on 127.0.0.1:8080");
+}
+for (const token of tokens) {
+  if (!block.includes(token)) throw new Error(`Caddy proxy lacks structural token: ${token}`);
+}
+NODE
+  echo "Caddy reverse-proxy guide contract passed"
+}
+
 verify_manual() {
  local profile="$1" manual
  manual="$root/docs/install/$profile-workspace-registry.md"
@@ -520,11 +685,27 @@ verify_manual() {
      return 1
    }
  done
-  for expected in \
-    'export THT_SOURCE_ROOT=/absolute/path/to/ThothII' \
-    '--env-file "$THT_OPERATOR_ENV"' \
-    "-f \"\$THT_SOURCE_ROOT/compose.yaml\" -f \"\$THT_SOURCE_ROOT/deploy/compose.$profile.yaml\"" \
-    '"$THT_SOURCE_ROOT/scripts/generate-connector-secrets-override.sh"'; do
+  local -a expected_steps
+  if [[ "$profile" == local ]]; then
+    expected_steps=(
+      'export THT_SOURCE_ROOT=/absolute/path/to/ThothII'
+      '--env-file "$THT_OPERATOR_ENV"'
+      "-f \"\$THT_SOURCE_ROOT/compose.yaml\" -f \"\$THT_SOURCE_ROOT/deploy/compose.$profile.yaml\""
+      '"$THT_SOURCE_ROOT/scripts/generate-connector-secrets-override.sh"'
+    )
+  else
+    expected_steps=(
+      'THTCTL=/srv/thothii/operator/thothctl'
+      'INSTALLATION=/srv/thothii/operator/thothii-installation.yaml'
+      '"$THTCTL" --installation "$INSTALLATION" start'
+      '"$THTCTL" --installation "$INSTALLATION" doctor'
+      'docs/install/examples/thothii-installation.server.yaml'
+      'compose.yaml'
+      'deploy/compose.server.yaml'
+      'server.md'
+    )
+  fi
+  for expected in "${expected_steps[@]}"; do
    grep -Fq -- "$expected" "$manual" || {
      echo "$profile manual lacks canonical operator step: $expected" >&2
      return 1
@@ -633,6 +814,136 @@ NODE
  echo "local installation example rendered from path with spaces passed"
 }

+verify_server_installation_example() {
+  local example="$root/docs/install/examples/thothii-installation.server.yaml"
+  [[ -f "$example" ]] || {
+    echo "missing server installation example: docs/install/examples/thothii-installation.server.yaml" >&2
+    return 1
+  }
+
+  local fixture source_copy operator_dir copied_example connector_override env_file
+  fixture="$(mktemp -d "${TMPDIR%/}/thoth server install.XXXXXX")"
+  trap 'rm -rf "$fixture"' RETURN
+  [[ "$fixture" == *" "* ]] || {
+    echo "server installation fixture path does not contain spaces" >&2
+    return 1
+  }
+  source_copy="$fixture/ThothII server source"
+  operator_dir="$fixture/server operator files"
+  mkdir -p "$source_copy/deploy/pi" "$source_copy/deploy/workspaces" \
+    "$operator_dir/data" "$operator_dir/pi-state" "$operator_dir/workspace-registry"
+  cp "$root/compose.yaml" "$source_copy/compose.yaml"
+  cp "$root/deploy/compose.server.yaml" "$source_copy/deploy/compose.server.yaml"
+  cp "$root/deploy/compose.session-server.yaml.example" \
+    "$source_copy/deploy/compose.session-server.yaml.example"
+  cp "$root/deploy/compose.git-ssh.yaml" "$source_copy/deploy/compose.git-ssh.yaml"
+  cp "$root/deploy/pi/models.json" "$source_copy/deploy/pi/models.json"
+  cp "$root/deploy/pi/settings.json" "$source_copy/deploy/pi/settings.json"
+  cp "$root/deploy/workspaces/server-sessions.yaml.example" \
+    "$source_copy/deploy/workspaces/server-sessions.yaml.example"
+
+  write_private "$operator_dir/pi-auth.json" '{"zai":{"type":"api_key","key":"fixture-server-pi-key"}}'
+  write_private "$operator_dir/thothii.secrets" 'THT_MODEL_API_KEY=fixture-server-model-key'
+  write_private "$operator_dir/git-ssh-key" 'fixture-server-ssh-key'
+  write_private "$operator_dir/git-known-hosts" 'fixture-server-known-hosts'
+  write_private "$operator_dir/dwh-password" 'fixture-server-dwh-password'
+  write_private "$operator_dir/session-runtime-password" 'fixture-server-session-runtime-password'
+  write_private "$operator_dir/session-migrator-password" 'fixture-server-session-migrator-password'
+  write_private "$operator_dir/session-ca.pem" 'fixture-server-session-ca'
+  printf '%s\n' \
+    'THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=postgres_direct' \
+    'THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password' \
+    >"$operator_dir/workspace-bindings.env"
+  env_file="$operator_dir/server.env"
+  printf '%s\n' \
+    'THOTH_SERVER_BIND=127.0.0.1' \
+    'THOTH_HTTP_PORT=8080' \
+    'THT_WORKSPACE_GIT_REMOTE=ssh://git@git.example.invalid/platform/thoth-workspaces.git' \
+    'THT_WORKSPACE_GIT_BRANCH=main' \
+    "PI_AUTH_FILE=$operator_dir/pi-auth.json" \
+    "THT_SECRETS_FILE=$operator_dir/thothii.secrets" \
+    "THT_WORKSPACE_BINDINGS_ENV_FILE=$operator_dir/workspace-bindings.env" \
+    "THT_WORKSPACE_GIT_SSH_KEY_FILE=$operator_dir/git-ssh-key" \
+    "THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=$operator_dir/git-known-hosts" \
+    "THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=$operator_dir/dwh-password" \
+    "THT_DATA_ROOT=$operator_dir/data" \
+    "THT_PI_STATE_ROOT=$operator_dir/pi-state" \
+    "THT_WORKSPACE_REGISTRY_ROOT=$operator_dir/workspace-registry" \
+    "THT_SERVER_WORKSPACE_CONFIG=$source_copy/deploy/workspaces/server-sessions.yaml.example" \
+    'THT_LLM_URL=https://llm.example.invalid' \
+    'THT_SESSION_DB_HOST=sessions.example.invalid' \
+    'THT_SESSION_DB_NAME=thoth_sessions' \
+    'THT_SESSION_RUNTIME_USER=thoth_sessions_app' \
+    'THT_SESSION_MIGRATOR_USER=thoth_sessions_migrate' \
+    "THT_SESSION_RUNTIME_PASSWORD_SOURCE=$operator_dir/session-runtime-password" \
+    "THT_SESSION_MIGRATOR_PASSWORD_SOURCE=$operator_dir/session-migrator-password" \
+    "THT_SESSION_CA_SOURCE=$operator_dir/session-ca.pem" \
+    >"$env_file"
+  connector_override="$operator_dir/connector-secrets.server.yaml"
+  "$root/scripts/generate-connector-secrets-override.sh" \
+    --bindings-env "$operator_dir/workspace-bindings.env" \
+    --operator-env "$env_file" \
+    --output "$connector_override" >/dev/null
+
+  copied_example="$fixture/thothii-installation.yaml"
+  local contents
+  contents="$(<"$example")"
+  contents="${contents//\/absolute\/path\/to\/ThothII/$source_copy}"
+  contents="${contents//\/absolute\/path\/to\/thothii-server-operator/$operator_dir}"
+  printf '%s\n' "$contents" >"$copied_example"
+
+  local profile project_directory descriptor_env value
+  local -a overrides files
+  profile="$(sed -n 's/^profile: \([^[:space:]]*\)$/\1/p' "$copied_example")"
+  project_directory="$(sed -n 's/^projectDirectory: "\(.*\)"$/\1/p' "$copied_example")"
+  descriptor_env="$(sed -n 's/^envFile: "\(.*\)"$/\1/p' "$copied_example")"
+  while IFS= read -r value; do overrides+=("$value"); done < <(sed -n 's/^  - "\(.*\)"$/\1/p' "$copied_example")
+  [[ "$profile" == server && "$project_directory" == "$source_copy" && "$descriptor_env" == "$env_file" ]] || {
+    echo "server installation example does not resolve its required fields" >&2
+    return 1
+  }
+  [[ "${#overrides[@]}" -eq 3 && "${overrides[0]}" == "$source_copy/deploy/compose.session-server.yaml.example" \
+    && "${overrides[2]}" == "$connector_override" ]] || {
+    echo "server installation example does not select the expected optional overrides" >&2
+    return 1
+  }
+  files=(-f "$project_directory/compose.yaml" -f "$project_directory/deploy/compose.$profile.yaml")
+  for value in "${overrides[@]}"; do files+=(-f "$value"); done
+  local rendered="$fixture/server-installation.json"
+  "$root/scripts/compose-with-preflight.sh" --env-file "$descriptor_env" \
+    "${files[@]}" config --format json >"$rendered"
+  node - "$rendered" <<'NODE'
+const fs = require("fs");
+const config = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
+if (Object.keys(config.services).sort().join(",") !== "core,frontend") {
+  throw new Error("server installation example must render exactly core,frontend");
+}
+const core = config.services.core;
+const frontend = config.services.frontend;
+if (core.environment?.AUTH_MODE !== "upstream" || core.environment?.THOTH_PUBLIC_EXPOSURE !== "true") {
+  throw new Error("server installation example must fail closed behind upstream authentication");
+}
+if ((core.ports || []).length !== 0) throw new Error("server installation example published core");
+const ports = frontend.ports || [];
+if (ports.length !== 1 || ports[0].host_ip !== "127.0.0.1" || Number(ports[0].target) !== 8080) {
+  throw new Error("server installation example must publish only loopback frontend");
+}
+const rendered = JSON.stringify(config);
+if (/omics_portal|chirone|localllm_default|datamart-builder/i.test(rendered)) {
+  throw new Error("server installation example contains application coupling");
+}
+for (const secret of [
+  "fixture-server-pi-key", "fixture-server-model-key", "fixture-server-ssh-key",
+  "fixture-server-known-hosts", "fixture-server-dwh-password",
+  "fixture-server-session-runtime-password", "fixture-server-session-migrator-password",
+  "fixture-server-session-ca",
+]) {
+  if (rendered.includes(secret)) throw new Error("server installation rendering exposed a fixture secret");
+}
+NODE
+  echo "server installation example rendered from path with spaces passed"
+}
+
 write_private() {
  local path="$1" value="$2"
  printf '%s\n' "$value" >"$path"
@@ -772,7 +1083,11 @@ case "$mode" in
    verify_local_guide
    verify_windows_line_endings_guide
    verify_pi_management_guide
+    verify_server_guide
+    verify_reverse_proxy_nginx_guide
+    verify_reverse_proxy_caddy_guide
    verify_local_installation_example
+    verify_server_installation_example
    verify_manual local
    verify_manual server
    verify_compose_fixtures
@@ -786,6 +1101,11 @@ case "$mode" in
      verify_windows_line_endings_guide
      verify_pi_management_guide
      verify_local_installation_example
+    else
+      verify_server_guide
+      verify_reverse_proxy_nginx_guide
+      verify_reverse_proxy_caddy_guide
+      verify_server_installation_example
    fi
    verify_manual "$profile"
    verify_compose_fixtures