From a707fb442c8c99f5c5db27c8bede9a7fc156addb Mon Sep 17 00:00:00 2001 From: mptyl Date: Wed, 5 Aug 2026 10:22:02 +0200 Subject: [PATCH] docs: add autonomous server installation guide --- .../examples/thothii-installation.server.yaml | 9 + docs/install/reverse-proxy-caddy.md | 97 +++++ docs/install/reverse-proxy-nginx.md | 132 +++++++ docs/install/server-workspace-registry.md | 91 ++--- docs/install/server.md | 368 ++++++++++++++++++ scripts/test-verify-workspace-install-docs.sh | 82 +++- scripts/verify-workspace-install-docs.sh | 330 +++++++++++++++- 7 files changed, 1056 insertions(+), 53 deletions(-) create mode 100644 docs/install/examples/thothii-installation.server.yaml create mode 100644 docs/install/reverse-proxy-caddy.md create mode 100644 docs/install/reverse-proxy-nginx.md create mode 100644 docs/install/server.md diff --git a/docs/install/examples/thothii-installation.server.yaml b/docs/install/examples/thothii-installation.server.yaml new file mode 100644 index 00000000..0b6e86c9 --- /dev/null +++ b/docs/install/examples/thothii-installation.server.yaml @@ -0,0 +1,9 @@ +# Copy this file to a protected operator path named exactly thothii-installation.yaml. +# Replace every absolute placeholder. Select exactly one Git transport override. +profile: server +projectDirectory: "/absolute/path/to/ThothII" +envFile: "/absolute/path/to/thothii-server-operator/server.env" +overrides: + - "/absolute/path/to/ThothII/deploy/compose.session-server.yaml.example" + - "/absolute/path/to/ThothII/deploy/compose.git-ssh.yaml" + - "/absolute/path/to/thothii-server-operator/connector-secrets.server.yaml" diff --git a/docs/install/reverse-proxy-caddy.md b/docs/install/reverse-proxy-caddy.md new file mode 100644 index 00000000..78690e53 --- /dev/null +++ b/docs/install/reverse-proxy-caddy.md @@ -0,0 +1,97 @@ +# Put ThothII behind Caddy + +This example assumes Caddy runs on the Linux host, ThothII `frontend` listens only on +`127.0.0.1:8080`, public DNS points to the host, and a separate authentication gateway validates +the user's real login/session. Replace the domain and auth-gateway address. + +## Trust boundary + +Caddy is the only public listener. It provides automatic HTTPS, performs `forward_auth`, and +proxies only to `frontend`; frontend then sends same-origin `/api` requests to private `core`. +Never send the public reverse proxy to core port 8787. + +The authentication gateway must return 2xx only after validating a real credential or session. +The ordered `route` below deletes every public/private Thoth identity request header before auth. +Only on auth success does `copy_headers` rename normalized response claims into the private +`X-Thoth-Trusted-*` headers consumed by frontend. +Forwarding identity headers alone does not authenticate a user. + +Do not replace the authentication gateway with static `header_up` values, a network allowlist, or +browser-provided identity. The admin claim must come from reviewed identity-provider authorization. + +## Example configuration + +Save a reviewed site block in the Caddyfile. Caddy obtains and renews TLS certificates for the real +DNS name; use the organization's approved ACME issuer or certificate policy. + +```caddyfile +thoth.example.com { + route { + # Remove untrusted browser claims before the auth subrequest. + request_header -X-Authenticated-User + request_header -X-Thoth-Principal-Issuer + request_header -X-Thoth-Principal-Subject + request_header -X-Thoth-Principal-Display-Name + request_header -X-Thoth-Is-Admin + request_header -X-Thoth-Trusted-Principal-Issuer + request_header -X-Thoth-Trusted-Principal-Subject + request_header -X-Thoth-Trusted-Principal-Display-Name + request_header -X-Thoth-Trusted-Is-Admin + + forward_auth auth-gateway:4180 { + uri /verify + copy_headers { + X-Thoth-Principal-Issuer>X-Thoth-Trusted-Principal-Issuer + X-Thoth-Principal-Subject>X-Thoth-Trusted-Principal-Subject + X-Thoth-Principal-Display-Name>X-Thoth-Trusted-Principal-Display-Name + X-Thoth-Is-Admin>X-Thoth-Trusted-Is-Admin + } + } + + reverse_proxy 127.0.0.1:8080 { + # Disable response batching so SSE reaches the browser immediately. + flush_interval -1 + header_up Host {host} + header_up X-Forwarded-Proto https + } + } + + log { + output file /var/log/caddy/thoth-access.log + format json + } +} +``` + +Configure log processing to remove cookies, authorization data, query strings, and identity +headers. Keep Caddy's private keys and state outside the ThothII source/operator directories. + +## Validate and reload + +Keep the public firewall rule closed while validating. Confirm ThothII responds only on loopback, +format a review copy if desired, validate the active file, then reload through the service manager: + +```sh +curl --fail http://127.0.0.1:8080/health +caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile +sudo systemctl reload caddy +``` + +Do not remove `forward_auth` if validation fails. Correct the Caddy version, adapter syntax, +authentication upstream, DNS, or certificate policy instead. + +## Test authentication and SSE + +Open the firewall only after all of these pass: + +1. An unauthenticated HTTPS request is redirected to login or returns 401/403. +2. Supplying forged `X-Thoth-Principal-*`, `X-Thoth-Is-Admin`, or `X-Thoth-Trusted-*` request + headers does not grant access. +3. A real authenticated non-admin can use the application but cannot open Pi Management. +4. A real authenticated admin can use Pi Management. +5. A browser session receives live model updates without batching; a protected + `curl --no-buffer` request using a real short-lived login cookie is also acceptable. Delete the + cookie jar immediately afterward. + +Re-run these checks after changing the identity provider, authentication gateway, Caddy, or +ThothII release. diff --git a/docs/install/reverse-proxy-nginx.md b/docs/install/reverse-proxy-nginx.md new file mode 100644 index 00000000..849a6fd9 --- /dev/null +++ b/docs/install/reverse-proxy-nginx.md @@ -0,0 +1,132 @@ +# Put ThothII behind Nginx + +This example assumes Nginx runs on the Linux host, ThothII `frontend` listens only on +`127.0.0.1:8080`, and a separate authentication gateway validates the user's real login/session. +Replace the documentation domain, certificate paths, and auth-gateway address. + +## Trust boundary + +Nginx is the only public listener. It terminates TLS, performs an `auth_request`, and proxies only +to `frontend`; frontend then uses its private same-origin `/api` route to reach `core`. Never proxy +the public listener directly to core port 8787. + +The authentication gateway must return 2xx only after validating a real credential or session. It +returns normalized `X-Thoth-Principal-*` and `X-Thoth-Is-Admin` response headers. Nginx clears all +client-supplied public and private-hop identity headers and copies only those successful auth +response values into `X-Thoth-Trusted-*` on the private hop to frontend. +Forwarding identity headers alone does not authenticate a user. + +Do not substitute a static header, network allowlist, or client-provided header for the +authentication gateway. The admin value must come from reviewed identity-provider authorization, +not from a username supplied by the browser. + +## Example configuration + +Install an Nginx build that includes `ngx_http_auth_request_module`. Save a reviewed version of +this server block under the host's Nginx configuration directory: + +```nginx +server { + listen 80; + server_name thoth.example.com; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl; + server_name thoth.example.com; + + ssl_certificate /etc/nginx/tls/thoth/fullchain.pem; + ssl_certificate_key /etc/nginx/tls/thoth/privkey.pem; + ssl_protocols TLSv1.2 TLSv1.3; + + location = /_authenticate { + internal; + proxy_pass http://auth-gateway:4180/verify; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-URI $request_uri; + proxy_set_header X-Original-Method $request_method; + + # The auth service derives identity from the real login/session, never these headers. + proxy_set_header X-Authenticated-User ""; + proxy_set_header X-Thoth-Principal-Issuer ""; + proxy_set_header X-Thoth-Principal-Subject ""; + proxy_set_header X-Thoth-Principal-Display-Name ""; + proxy_set_header X-Thoth-Is-Admin ""; + proxy_set_header X-Thoth-Trusted-Principal-Issuer ""; + proxy_set_header X-Thoth-Trusted-Principal-Subject ""; + proxy_set_header X-Thoth-Trusted-Principal-Display-Name ""; + proxy_set_header X-Thoth-Trusted-Is-Admin ""; + } + + location / { + auth_request /_authenticate; + auth_request_set $thoth_principal_issuer + $upstream_http_x_thoth_principal_issuer; + auth_request_set $thoth_principal_subject + $upstream_http_x_thoth_principal_subject; + auth_request_set $thoth_principal_display_name + $upstream_http_x_thoth_principal_display_name; + auth_request_set $thoth_is_admin + $upstream_http_x_thoth_is_admin; + + # Discard browser claims. Carry only successful auth-subrequest values to frontend. + proxy_set_header X-Authenticated-User ""; + proxy_set_header X-Thoth-Principal-Issuer ""; + proxy_set_header X-Thoth-Principal-Subject ""; + proxy_set_header X-Thoth-Principal-Display-Name ""; + proxy_set_header X-Thoth-Is-Admin ""; + proxy_set_header X-Thoth-Trusted-Principal-Issuer $thoth_principal_issuer; + proxy_set_header X-Thoth-Trusted-Principal-Subject $thoth_principal_subject; + proxy_set_header X-Thoth-Trusted-Principal-Display-Name $thoth_principal_display_name; + proxy_set_header X-Thoth-Trusted-Is-Admin $thoth_is_admin; + + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header Connection ""; + proxy_pass http://127.0.0.1:8080; + proxy_http_version 1.1; + + # SSE must reach the browser without response buffering or cache delay. + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 3600s; + add_header X-Accel-Buffering no always; + } +} +``` + +Keep the certificate private key outside the ThothII tree. Do not log cookies, authorization +headers, auth response bodies, or trusted identity headers. + +## Validate and reload + +First keep the public firewall rule closed. Confirm ThothII responds only on loopback, validate the +full Nginx configuration, then reload without stopping existing connections: + +```sh +curl --fail http://127.0.0.1:8080/health +sudo nginx -t +sudo systemctl reload nginx +``` + +If `nginx -t` reports that `auth_request` is unknown, install the distribution package/module that +provides it. Do not remove authentication to make the syntax check pass. + +## Test authentication and SSE + +Open the firewall only after all of these pass: + +1. An unauthenticated HTTPS request is redirected to login or returns 401/403. +2. Supplying forged `X-Thoth-Principal-*`, `X-Thoth-Is-Admin`, or `X-Thoth-Trusted-*` request + headers does not grant access. +3. A real authenticated non-admin can use the application but cannot open Pi Management. +4. A real authenticated admin can use Pi Management. +5. A browser session receives live model updates for longer than the default proxy timeout without + batching; a protected `curl --no-buffer` test using a real short-lived login cookie is also + acceptable. Delete the cookie jar immediately afterward. + +Re-run these checks after changing the identity provider, auth gateway, Nginx, or ThothII release. diff --git a/docs/install/server-workspace-registry.md b/docs/install/server-workspace-registry.md index bb3a7c5c..4b794ced 100644 --- a/docs/install/server-workspace-registry.md +++ b/docs/install/server-workspace-registry.md @@ -1,8 +1,9 @@ # Server workspace-registry installation -This is the production operator guide. The application image is read-only, secrets are mounted -read-only, and sessions use immutable Git-validated snapshots. Expose the application only behind -an authenticated same-origin reverse proxy; never publish the core port directly. +This is the production workspace-registry companion to [the autonomous Linux server guide](server.md). +The application image is read-only, secrets are mounted read-only, and sessions use immutable +Git-validated snapshots. Expose the application only behind an authenticated same-origin reverse +proxy; never publish the core port directly. ## Service account, storage, and firewall @@ -142,52 +143,50 @@ The Git registry itself may still use SSH normally. Use the repository's canonical `compose.yaml` plus `deploy/compose.server.yaml`; they always start the mandatory `frontend` and `core` services. Do not copy or maintain a standalone -application Compose file. Review `deploy/workspaces/server-sessions.yaml.example`, materialize it -as a protected host file, and set its absolute `THT_SERVER_WORKSPACE_CONFIG` path. Copy the -bindings env example into the operator directory, then set absolute `PI_AUTH_FILE`, -`THT_SECRETS_FILE`, `THT_WORKSPACE_BINDINGS_ENV_FILE`, and connector `*_SOURCE` paths. -The same operator env must set +application Compose file. Copy `docs/install/examples/thothii-installation.server.yaml` to the +protected operator directory and preserve its required session-server overlay, exactly one Git +transport override, and generated connector-secret override. + +Review `deploy/workspaces/server-sessions.yaml.example`, materialize it as a protected host file, +and set its absolute `THT_SERVER_WORKSPACE_CONFIG` path. Copy the bindings env example into the +operator directory, then set absolute `PI_AUTH_FILE`, `THT_SECRETS_FILE`, +`THT_WORKSPACE_BINDINGS_ENV_FILE`, and connector `*_SOURCE` paths. The same operator env must set `THT_SESSION_DB_HOST`, `THT_SESSION_DB_NAME`, `THT_SESSION_RUNTIME_USER`, `THT_SESSION_RUNTIME_PASSWORD_SOURCE`, and `THT_SESSION_CA_SOURCE`; `deploy/compose.session-server.yaml.example` wires `postgres`, `verify-full`, and separate -runtime/CA Docker secret mount paths. This is the public server profile, -not a filesystem-session fallback. A Compose `.env` file is not a shell environment, so do not -import it into the maintenance shell. Explicitly export the non-secret source and bindings paths -before running the commands below. +runtime/CA secret targets under `/run/secrets`. This public server profile never falls back to +filesystem sessions. The path-only environment file is not shell code; do not source it. -Configure the portal proxy so the frontend and `/api` share one origin. It authenticates first and -clears client identity headers, carries auth-request claims over the private hop as -`X-Thoth-Trusted-*`, and lets the frontend proxy inject only the normalized -`X-Thoth-Principal-Issuer`, `X-Thoth-Principal-Subject`, `X-Thoth-Principal-Display-Name`, and -`X-Thoth-Is-Admin` claims expected by `AUTH_MODE=upstream`; it is the only public listener. Use -`deploy/nginx-authenticated-proxy.conf.example` as the forwarding contract. -From a trusted maintenance shell: +Generate the connector override, then use the installation-aware operator CLI. Building +`thothctl` requires only Docker and no Go knowledge. From a trusted maintenance shell: ```sh -export THT_SOURCE_ROOT=/absolute/path/to/ThothII -export THT_OPERATOR_ENV=/srv/thothii/operator/server.env -export THT_WORKSPACE_BINDINGS_ENV_FILE=/srv/thothii/operator/workspace-bindings.env -export THT_CONNECTOR_OVERRIDE=/srv/thothii/operator/connector-secrets.local.yaml -"$THT_SOURCE_ROOT/scripts/generate-connector-secrets-override.sh" --bindings-env "$THT_WORKSPACE_BINDINGS_ENV_FILE" --operator-env "$THT_OPERATOR_ENV" --output "$THT_CONNECTOR_OVERRIDE" -"$THT_SOURCE_ROOT/scripts/compose-with-preflight.sh" --env-file "$THT_OPERATOR_ENV" \ - -f "$THT_SOURCE_ROOT/compose.yaml" -f "$THT_SOURCE_ROOT/deploy/compose.server.yaml" \ - -f "$THT_SOURCE_ROOT/deploy/compose.session-server.yaml.example" \ - -f "$THT_SOURCE_ROOT/deploy/compose.git-ssh.yaml" -f "$THT_CONNECTOR_OVERRIDE" up --build -d -"$THT_SOURCE_ROOT/scripts/compose-with-preflight.sh" --env-file "$THT_OPERATOR_ENV" \ - -f "$THT_SOURCE_ROOT/compose.yaml" -f "$THT_SOURCE_ROOT/deploy/compose.server.yaml" \ - -f "$THT_SOURCE_ROOT/deploy/compose.session-server.yaml.example" \ - -f "$THT_SOURCE_ROOT/deploy/compose.git-ssh.yaml" -f "$THT_CONNECTOR_OVERRIDE" \ - exec -T core curl --fail --silent http://127.0.0.1:8787/health -"$THT_SOURCE_ROOT/scripts/compose-with-preflight.sh" --env-file "$THT_OPERATOR_ENV" \ - -f "$THT_SOURCE_ROOT/compose.yaml" -f "$THT_SOURCE_ROOT/deploy/compose.server.yaml" \ - -f "$THT_SOURCE_ROOT/deploy/compose.session-server.yaml.example" \ - -f "$THT_SOURCE_ROOT/deploy/compose.git-ssh.yaml" -f "$THT_CONNECTOR_OVERRIDE" \ - exec -T core curl --fail --silent http://127.0.0.1:8787/workspace-registry/status +THT_SOURCE_ROOT=/srv/thothii/source/ThothII +THT_OPERATOR_ENV=/srv/thothii/operator/server.env +THT_WORKSPACE_BINDINGS_ENV_FILE=/srv/thothii/operator/workspace-bindings.env +THT_CONNECTOR_OVERRIDE=/srv/thothii/operator/connector-secrets.server.yaml +THTCTL=/srv/thothii/operator/thothctl +INSTALLATION=/srv/thothii/operator/thothii-installation.yaml +"$THT_SOURCE_ROOT/scripts/generate-connector-secrets-override.sh" \ + --bindings-env "$THT_WORKSPACE_BINDINGS_ENV_FILE" \ + --operator-env "$THT_OPERATOR_ENV" --output "$THT_CONNECTOR_OVERRIDE" +"$THTCTL" --installation "$INSTALLATION" update --check-only +"$THTCTL" --installation "$INSTALLATION" start +"$THTCTL" --installation "$INSTALLATION" status +"$THTCTL" --installation "$INSTALLATION" doctor +"$THTCTL" --installation "$INSTALLATION" pi doctor +"$THTCTL" --installation "$INSTALLATION" pi test ``` -`/health` is liveness. Registry status verifies branch/head/degraded state and the active validated -snapshot; authenticated `/workspaces` verifies application access. A server with no active snapshot -is not ready for workspace sessions even if liveness succeeds. +Configure [Nginx](reverse-proxy-nginx.md) or [Caddy](reverse-proxy-caddy.md) so frontend and `/api` +share one TLS origin. The proxy authenticates first, clears client identity headers, and carries +only successful authentication claims over the private `X-Thoth-Trusted-*` hop. Forwarding claims +without authenticating the request is not an identity boundary. + +`/health` is liveness. The authenticated Workspace Management page's registry status verifies +branch/head/degraded state and the active validated snapshot; its workspace listing verifies +application access. A server with no active snapshot is not ready for workspace sessions even if +liveness succeeds. ## Pull, publish, upgrade, backup, and recovery @@ -196,10 +195,12 @@ browser-local. Publish takes a canonical diff, validates before commit, and push lock. On `workspace_conflict`, pull, resolve the reviewed field-level draft, validate/test, and publish; never edit `repo/` inside a running volume. -For upgrades, record active status/head, drain active Pi work, stop `core`, and take a -filesystem-consistent backup of `/srv/thothii/workspace-registry` plus `/srv/thothii/data`. Exclude -`/srv/thothii/secrets`. Render Compose, deploy the compatible image, verify health/status, then -resume proxy traffic. +For upgrades, record active status/head, finish active work, use the documented `thothctl pi update +--drain` transaction when Pi/core changes, and take a stopped, filesystem-consistent backup of +`/srv/thothii/workspace-registry` plus `/srv/thothii/data` and Pi state. Exclude +`/srv/thothii/secrets` from the ordinary archive. Validate the descriptor with `thothctl update +--check-only`, deploy the compatible image through `thothctl`, verify health/status, then resume +proxy traffic. For legacy descriptor migration, use a temporary review clone and the legacy transformer with absolute paths. Its schema-v1 output is `migration_required`; explicitly supply vector database/schema, collection diff --git a/docs/install/server.md b/docs/install/server.md new file mode 100644 index 00000000..7049945f --- /dev/null +++ b/docs/install/server.md @@ -0,0 +1,368 @@ +# Install ThothII on a Linux server + +This guide is for an installer with basic Linux administration and very basic Docker knowledge. +It deploys the same Compose distribution used on a local PC: the mandatory application is exactly +`frontend` plus `core`, and pinned Pi is inside `core`. The server does not need host Pi, Node.js, +Python, Go, a browser shell, or a Docker socket inside either container. + +Examples use `/srv/thothii` as an **example operator root**, `thoth.example.com` as a replaceable +DNS name, and systemd-based command names. Adapt them to local policy. Complete the server session +storage overlay and migration procedure before exposing a production installation. + +## Deployment contract + +- The generic Linux host and Docker Compose v2 are the deployment platform. No other + application's Compose project, network, path, or runtime is required. +- `frontend` is the only published service and defaults to `127.0.0.1:8080`; `core` has no host + port. A host Nginx or Caddy listener terminates TLS and sends all application traffic to + `frontend`, never directly to `core`. +- DWH, vector database, embedding service, and LLM are external configurable endpoints. This + remains true when they happen to run on the same physical server. +- Application, Git, connector, and session credentials are protected host files mounted read-only + under `/run/secrets`. Pi's protected auth JSON uses its dedicated read-only Pi mount. No secret + value belongs in Git, images, browser storage, environment values, rendered Compose, or logs. +- The Git-backed workspace registry is the source of truth. Installation-local bindings identify + endpoints and secret-file paths; they do not replace the reviewed Git workspace descriptors. +- `thothctl` is the operator CLI for start, stop, status, health, logs, Pi lifecycle, drain, and + rollback. Raw Compose lifecycle commands bypass installation state and are unsupported. + +Read [server workspace-registry installation](server-workspace-registry.md), +[Pi management](pi-management.md), and the session-server comments in +`deploy/compose.session-server.yaml.example` before the first public start. + +## Service account and directories + +The container runtime identity is fixed at UID/GID 10001. Reserve the same host ID for a dedicated +non-login `thothii` account so bind-mounted ownership is obvious. Stop if either ID is already used +by another account; choose a reviewed host mapping instead of changing the image identity. + +```sh +getent passwd 10001 +getent group 10001 +sudo useradd --system --uid 10001 --user-group --home-dir /srv/thothii \ + --create-home --shell /usr/sbin/nologin thothii +``` + +The human operator who runs `thothctl` needs Docker access. On many installations membership in +the `docker` group is effectively host-root access; grant it only according to site policy. The +non-login `thothii` account owns application data and secrets but does not itself need Docker +access. + +Create explicit directories. `source` contains the clone; `operator` contains untracked path-only +configuration; the three writable trees are bind-mounted into `core`; `secrets` contains regular +files only. Backups are separate from live data. + +```sh +sudo install -d -o 10001 -g 10001 -m 0750 /srv/thothii/source +sudo install -d -o 10001 -g 10001 -m 0700 /srv/thothii/operator +sudo install -d -o 10001 -g 10001 -m 0700 /srv/thothii/secrets +sudo install -d -o 10001 -g 10001 -m 0750 /srv/thothii/data +sudo install -d -o 10001 -g 10001 -m 0750 /srv/thothii/pi-state +sudo install -d -o 10001 -g 10001 -m 0750 /srv/thothii/workspace-registry +sudo install -d -o root -g root -m 0700 /srv/thothii-backups +``` + +Do not make `/srv/thothii` a shared application directory. The source checkout may be read by the +operator, while secret contents and writable data remain limited to reviewed administrators and +UID 10001. + +## Firewall and network boundaries + +Set `THOTH_SERVER_BIND=127.0.0.1`. Permit inbound TCP 80/443 only to the TLS proxy; port 80 should +redirect to HTTPS. Do not open 8080 externally, and do not add a core port. If a separate proxy +host is used, replace loopback with a private, firewalled address and allow only that proxy source. + +Allow outbound DNS and HTTPS to the source/Git registries, plus only the configured ports for the +Git remote, DWH, vector database, embedding service, LLM, session PostgreSQL, and any approved +bastion. Docker's private `thothii` network carries only `frontend`↔`core` traffic. Do not attach +the mandatory stack to another application's network. + +After start, confirm the host listens as intended: + +```sh +sudo ss -lntp +``` + +Expected public listeners are the proxy on 80/443 and the frontend on loopback 8080. There must be +no host listener for core port 8787. + +## Address co-resident external services + +Endpoint values are resolved inside `core`. Therefore container 127.0.0.1 means the container +itself, not the Linux host. Prefer real DNS names with TLS, authentication, and firewall policy, +even for services on this physical server. + +When DNS is unavailable for a host-published service, create an untracked override such as +`/srv/thothii/operator/host-gateway.yaml` and add it to the installation descriptor: + +```yaml +services: + core: + extra_hosts: + - "host.docker.internal:host-gateway" +``` + +Use `host.docker.internal` in the endpoint binding. The `host-gateway` mapping supplies routing; +it does not bundle or trust the target service. Keep the target port bound/firewalled for Docker +host access only. A stable internal DNS record is the preferred alternative. + +Configure each boundary independently: + +- DWH: read-only runtime identity, database/schema, verified TLS, and direct or REST endpoint. +- Vector database: endpoint plus exact database/schema, collection, distance metric, and writer + policy declared by the reviewed workspace. +- Embedding service: endpoint and the collection/embedding pairing—model and dimensions must match + the existing collection. Co-residence does not permit silently changing that pairing. +- LLM: authenticated endpoint selected through deployment and Pi configuration. + +Never add those services to ThothII's mandatory Compose files. Follow +[the diagnostic protocol](../workspace-diagnostic-protocol.md) before enabling a workspace. + +## Prepare operator files and secrets + +Clone with LF line endings, then verify before every build: + +```sh +sudo -u thothii git -c core.autocrlf=false clone \ + https://github.example.invalid/your-org/ThothII.git /srv/thothii/source/ThothII +cd /srv/thothii/source/ThothII +sudo -u thothii git config --local core.autocrlf false +bash scripts/verify-line-endings.sh +``` + +Copy the path-only server environment and installation descriptor: + +```sh +sudo -u thothii cp deploy/env/server.env.example /srv/thothii/operator/server.env +sudo -u thothii cp docs/install/examples/thothii-installation.server.yaml \ + /srv/thothii/operator/thothii-installation.yaml +sudo chmod 0600 /srv/thothii/operator/server.env \ + /srv/thothii/operator/thothii-installation.yaml +``` + +Replace every placeholder with an absolute path. Use exactly one Git transport override. For +HTTPS, replace `deploy/compose.git-ssh.yaml` with `deploy/compose.git-https.yaml`. Keep the required +session-server overlay and generated connector-secret override. Optional host-gateway or pinned +image overrides go after them. + +Create each credential as an independent regular file in `/srv/thothii/secrets`, owned by +UID/GID 10001 and mode `0600`. The operator environment records only absolute `*_FILE` or +`*_SOURCE` paths. Compose mounts application and connector targets read-only under `/run/secrets`; +the frontend receives none. Do not print file contents while testing permissions. + +```sh +sudo find /srv/thothii/secrets -type f -exec chown 10001:10001 {} + +sudo find /srv/thothii/secrets -type f -exec chmod 0600 {} + +sudo find /srv/thothii/secrets -type f ! -user thothii -print +sudo find /srv/thothii/secrets -type f ! -perm 0600 -print +``` + +Add `THT_WORKSPACE_BINDINGS_ENV_FILE=/srv/thothii/operator/workspace-bindings.env` and the matching +connector `*_SOURCE` paths to `server.env`. Generate +`/srv/thothii/operator/connector-secrets.server.yaml` as described in +[server workspace-registry installation](server-workspace-registry.md). Secret values must never +be pasted into `server.env`, the installation YAML, a URL, or a shell argument. + +## Build locally or select pinned images + +Choose one image source. For a source build, the repository's reproducible launcher builds the +same `core` and `frontend` images used by the local profile. It requires only Git, Docker, and +Compose; copy the reviewed path-only server environment to the launcher's untracked input first: + +```sh +cd /srv/thothii/source/ThothII +sudo -u thothii cp /srv/thothii/operator/server.env deploy/env/local.env +bash scripts/build-local.sh +``` + +The printed local-profile start command is not the server start command; use `thothctl` below. + +Alternatively, create a reviewed untracked override with release images pinned by immutable +digest. Mutable tags are not a production pin: + +```yaml +services: + core: + build: !reset null + image: registry.example.com/thothii/core@sha256:<64-lowercase-hex-digits> + frontend: + build: !reset null + image: registry.example.com/thothii/frontend@sha256:<64-lowercase-hex-digits> +``` + +Add that absolute file last in `overrides`. Both images must come from one compatible release; the +core image must retain the declared Pi version labels checked by `thothctl pi doctor`. Pull access +belongs in the host Docker credential store, not in Compose or the installation descriptor. + +## Install thothctl + +Build the operator binaries with Docker. No Go installation or Go knowledge is required: + +```sh +cd /srv/thothii/source/ThothII +bash scripts/build-thothctl.sh +sudo install -o 10001 -g 10001 -m 0755 dist/thothctl/thothctl-linux-amd64 \ + /srv/thothii/operator/thothctl +``` + +Use `thothctl-linux-arm64` on an ARM64 server. Set these variables in the maintenance shell; do +not source `server.env` as shell code: + +```sh +THTCTL=/srv/thothii/operator/thothctl +INSTALLATION=/srv/thothii/operator/thothii-installation.yaml +"$THTCTL" --installation "$INSTALLATION" --help +"$THTCTL" --installation "$INSTALLATION" update --check-only +``` + +Every operator command includes the descriptor explicitly. This preserves the installation's +profile, overrides, project identity, and durable current-image selector. The general form is +`thothctl --installation /absolute/path/thothii-installation.yaml `. + +## Start and verify readiness + +Keep the TLS proxy stopped or firewalled during bootstrap: + +```sh +"$THTCTL" --installation "$INSTALLATION" start +"$THTCTL" --installation "$INSTALLATION" status +"$THTCTL" --installation "$INSTALLATION" doctor +curl --fail http://127.0.0.1:8080/health +"$THTCTL" --installation "$INSTALLATION" pi doctor +"$THTCTL" --installation "$INSTALLATION" pi test +``` + +`/health` proves process liveness. Readiness additionally requires both healthy services, a valid +Pi provider/model smoke, a successful Git registry pull with an active validated snapshot, valid +workspace diagnostics, and ready session PostgreSQL. Use the authenticated Workspace Management +page to pull and diagnose the reviewed workspace. A liveness response alone is not release +approval. + +After configuring the proxy, open in a browser. Verify an unauthenticated +request is denied or redirected by the real identity provider, an authorized user can load the +same-origin UI and `/api`, an unauthorized user is denied, and an administrator alone can open Pi +Management. Keep port 8080 inaccessible from other hosts. + +## Configure TLS and upstream authentication + +Choose [Nginx](reverse-proxy-nginx.md) or [Caddy](reverse-proxy-caddy.md). Both examples terminate +TLS and proxy only to loopback `frontend`. They preserve SSE and clear client-supplied identity +headers before authentication. + +The authentication gateway must validate a real login/session and return normalized issuer, +subject, display-name, and admin claims only after success. Merely forwarding those headers does +not authenticate anyone. Do not enable `AUTH_MODE=upstream` on a listener reachable around the +trusted proxy, and never expose `core`. + +## Operate Pi, drain, and roll back + +Configure only closed provider/model/reasoning choices. Credentials remain protected files: + +```sh +"$THTCTL" --installation "$INSTALLATION" pi status +"$THTCTL" --installation "$INSTALLATION" pi configure +"$THTCTL" --installation "$INSTALLATION" pi doctor +"$THTCTL" --installation "$INSTALLATION" pi logs +``` + +Before an update, announce maintenance and ask users to finish active work. `--drain` closes new +admission and waits until no active sessions remain; it does not discard sessions. Build-source +and registry-source examples are: + +```sh +"$THTCTL" --installation "$INSTALLATION" pi update \ + --version 0.81.0 --source build --yes --drain +"$THTCTL" --installation "$INSTALLATION" pi update \ + --version 0.81.0 --source pull \ + --image registry.example.com/thothii/core@sha256:<64-lowercase-hex-digits> \ + --yes --drain +``` + +The transaction recreates only `core`, preserves volumes, verifies health/configuration/Pi, and +automatically attempts rollback after a post-mutation failure. For interrupted or ambiguous state: + +```sh +"$THTCTL" --installation "$INSTALLATION" pi maintenance status +"$THTCTL" --installation "$INSTALLATION" pi rollback --yes +"$THTCTL" --installation "$INSTALLATION" pi maintenance recover --yes +``` + +Leave maintenance active if rollback cannot be verified. Preserve `.thothctl//` +recovery state, repair the reported host/configuration issue, and rerun rollback or maintenance +recovery. Never delete or edit `current-image.yaml` or `update-state.json` to force progress. + +## Back up and restore + +Back up before source, workspace, session-schema, or Pi changes. Drain work, stop the installation, +record `git rev-parse HEAD`, image digests, and `thothctl status`, then archive the three bind trees +with numeric ownership. Do not include live secrets in this ordinary archive. + +```sh +"$THTCTL" --installation "$INSTALLATION" stop +BACKUP=/srv/thothii-backups/2026-08-05 +sudo install -d -o root -g root -m 0700 "$BACKUP" +sudo tar --numeric-owner --xattrs --acls -C /srv/thothii -czf "$BACKUP/runtime-data.tgz" \ + data pi-state workspace-registry +sudo sha256sum "$BACKUP/runtime-data.tgz" >"$BACKUP/SHA256SUMS" +``` + +Back up the installation descriptor, path-only environment, generated overrides, source revision, +and secret files to separate encrypted access-controlled storage. Database-backed production +sessions require their own PostgreSQL-native consistent backup; the local bind tree is not a +substitute. Test both restore paths periodically. + +Restore only while stopped. Verify the checksum, extract first into a new empty root, inspect +ownership and expected registry layout, then retain the old trees by renaming them before placing +the restored set. This keeps the previous state recoverable: + +```sh +RESTORE=/srv/thothii-restore-2026-08-05 +sudo install -d -o root -g root -m 0700 "$RESTORE" +sudo sha256sum --check /srv/thothii-backups/2026-08-05/SHA256SUMS +sudo tar --numeric-owner --xattrs --acls -C "$RESTORE" \ + -xzf /srv/thothii-backups/2026-08-05/runtime-data.tgz +sudo test -d "$RESTORE/workspace-registry/repo" +sudo test -d "$RESTORE/workspace-registry/snapshots" +``` + +During the reviewed restore window, move each old tree to a timestamped sibling, move the matching +restored tree into `/srv/thothii`, restore the PostgreSQL session backup from the same recovery +point, and keep the proxy closed. Run `update --check-only`, `start`, `doctor`, `pi test`, registry +status, workspace diagnostics, and a known historical session before reopening traffic. Never +merge an archive into a non-empty tree. + +## Diagnostics + +Begin with bounded, sanitized installation-aware commands: + +```sh +"$THTCTL" --installation "$INSTALLATION" status +"$THTCTL" --installation "$INSTALLATION" doctor +"$THTCTL" --installation "$INSTALLATION" logs +"$THTCTL" --installation "$INSTALLATION" pi status +"$THTCTL" --installation "$INSTALLATION" pi doctor +"$THTCTL" --installation "$INSTALLATION" pi test +"$THTCTL" --installation "$INSTALLATION" pi logs +"$THTCTL" --installation "$INSTALLATION" pi maintenance status +``` + +Use the authenticated Workspace Management status and diagnostic actions for Git revision, +degraded snapshot, bindings, DWH, vector, and embedding checks. Review proxy logs separately, but +configure both proxy and log shipping to exclude cookies, authorization data, identity payloads, +query strings, and secret values. Do not render Compose or print an environment as a diagnostic. + +Typical boundaries are: `doctor` for Docker/Compose/LF/volume/service health; `pi doctor` for image +and provider/model integrity; registry status for Git/snapshot health; workspace diagnostics for +external service identity; and the proxy/identity provider for login failures. + +## Data-preserving uninstall + +Drain and stop through `thothctl`, take and verify one final backup, disable the TLS proxy route, +and remove only this installation's stopped `frontend` and `core` containers and optional images +by their exact Compose project labels. Keep `/srv/thothii/data`, `pi-state`, +`workspace-registry`, `operator`, protected secrets, database backups, and the installation +descriptor if reinstallation is possible. Do not prune global Docker data. + +Do **not** run `docker compose down --volumes`; it deletes persistent application data. Reusing the +same protected descriptor path preserves the `thothctl` installation identity and allows a later +compatible source checkout to reconnect the retained state. diff --git a/scripts/test-verify-workspace-install-docs.sh b/scripts/test-verify-workspace-install-docs.sh index 29ff5692..984abf46 100755 --- a/scripts/test-verify-workspace-install-docs.sh +++ b/scripts/test-verify-workspace-install-docs.sh @@ -15,7 +15,11 @@ for fixture in \ "source update fail-closed semantics" \ "Windows line-ending recovery guide contract" \ "Pi management guide contract" \ + "server installation guide contract" \ + "Nginx reverse-proxy guide contract" \ + "Caddy reverse-proxy guide contract" \ "local installation example rendered from path with spaces" \ + "server installation example rendered from path with spaces" \ "local manual canonical base+override references" \ "server manual canonical base+override references" \ "canonical local base+override fixture" \ @@ -29,9 +33,7 @@ for fixture in \ } done -for manual in \ - "$root/docs/install/local-workspace-registry.md" \ - "$root/docs/install/server-workspace-registry.md"; do +for manual in "$root/docs/install/local-workspace-registry.md"; do grep -Fq 'export THT_SOURCE_ROOT=/absolute/path/to/ThothII' "$manual" || { echo "installation manual does not publish a self-contained THT_SOURCE_ROOT export: $manual" >&2 exit 1 @@ -46,6 +48,17 @@ for manual in \ fi done +grep -Fq 'THTCTL=/srv/thothii/operator/thothctl' \ + "$root/docs/install/server-workspace-registry.md" || { + echo "server installation manual does not use the installation-aware operator CLI" >&2 + exit 1 +} +grep -Fq 'INSTALLATION=/srv/thothii/operator/thothii-installation.yaml' \ + "$root/docs/install/server-workspace-registry.md" || { + echo "server installation manual does not identify the server installation descriptor" >&2 + exit 1 +} + if rg -n 'local-compose\.workspace-registry|server-compose\.workspace-registry|connector-secrets\.workspace-registry|docker compose' \ "$root/docs/install/local-workspace-registry.md" \ "$root/docs/install/server-workspace-registry.md"; then @@ -92,6 +105,33 @@ switch (mutation) { case "raw-pi": changed += "\n```sh\ndocker compose exec core pi --version\n```\n"; break; + case "server-secret-env": + changed += "\n```dotenv\nTHT_MODEL_API_KEY=unsafe-secret-value\n```\n"; + break; + case "server-docker-socket": + changed += "\nMount /var/run/docker.sock into core for management.\n"; + break; + case "server-coupling": + changed += "\nAttach core to the omics_portal application network.\n"; + break; + case "nginx-no-auth": + changed = original.replace(" auth_request /_authenticate;", " # authentication omitted"); + break; + case "nginx-core-upstream": + changed = original.replaceAll("http://127.0.0.1:8080", "http://127.0.0.1:8787"); + break; + case "nginx-no-sse": + changed = original.replace(" proxy_buffering off;", " proxy_buffering on;"); + break; + case "caddy-no-auth": + changed = original.replace("forward_auth auth-gateway:4180 {", "# forward authentication omitted"); + break; + case "caddy-client-identity": + changed = original.replace("X-Thoth-Principal-Subject>X-Thoth-Trusted-Principal-Subject", "X-Thoth-Principal-Subject"); + break; + case "caddy-core-upstream": + changed = original.replaceAll("127.0.0.1:8080", "127.0.0.1:8787"); + break; case "dirty-source": changed = original.replaceAll("git status --porcelain --untracked-files=all", "git status --short"); break; @@ -163,6 +203,42 @@ expect_guide_rejected \ "raw non-installation-aware Pi access" verify_pi_management_guide \ "$root/docs/install/pi-management.md" docs/install/pi-management.md raw-pi \ "raw non-installation-aware Compose Pi access is forbidden" +expect_guide_rejected \ + "server secret in environment" verify_server_guide \ + "$root/docs/install/server.md" docs/install/server.md server-secret-env \ + "server installation guide embeds a secret value" +expect_guide_rejected \ + "server Docker socket mount" verify_server_guide \ + "$root/docs/install/server.md" docs/install/server.md server-docker-socket \ + "server installation guide introduces a Docker socket dependency" +expect_guide_rejected \ + "server application coupling" verify_server_guide \ + "$root/docs/install/server.md" docs/install/server.md server-coupling \ + "server installation guide introduces forbidden application coupling" +expect_guide_rejected \ + "Nginx identity without authentication" verify_reverse_proxy_nginx_guide \ + "$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-auth \ + "Nginx proxy lacks structural token: auth_request /_authenticate;" +expect_guide_rejected \ + "Nginx direct core exposure" verify_reverse_proxy_nginx_guide \ + "$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-core-upstream \ + "Nginx proxy must forward only to frontend on 127.0.0.1:8080" +expect_guide_rejected \ + "Nginx buffered SSE" verify_reverse_proxy_nginx_guide \ + "$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-sse \ + "Nginx proxy lacks structural token: proxy_buffering off;" +expect_guide_rejected \ + "Caddy identity without authentication" verify_reverse_proxy_caddy_guide \ + "$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-auth \ + "Caddy proxy lacks structural token: forward_auth auth-gateway:4180 {" +expect_guide_rejected \ + "Caddy untrusted identity forwarding" verify_reverse_proxy_caddy_guide \ + "$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-client-identity \ + "Caddy proxy lacks structural token: X-Thoth-Principal-Subject>X-Thoth-Trusted-Principal-Subject" +expect_guide_rejected \ + "Caddy direct core exposure" verify_reverse_proxy_caddy_guide \ + "$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-core-upstream \ + "Caddy proxy must forward only to frontend on 127.0.0.1:8080" expect_guide_rejected \ "dirty or untracked source tree" verify_local_guide \ diff --git a/scripts/verify-workspace-install-docs.sh b/scripts/verify-workspace-install-docs.sh index 7bb9ed68..92fe5921 100755 --- a/scripts/verify-workspace-install-docs.sh +++ b/scripts/verify-workspace-install-docs.sh @@ -488,6 +488,171 @@ NODE echo "Pi management guide contract passed" } +verify_server_guide() { + local guide="$root/docs/install/server.md" + [[ -f "$guide" ]] || { + echo "missing server installation guide sections: docs/install/server.md" >&2 + return 1 + } + require_headings "$guide" "server installation guide" \ + "Deployment contract" \ + "Service account and directories" \ + "Firewall and network boundaries" \ + "Address co-resident external services" \ + "Prepare operator files and secrets" \ + "Build locally or select pinned images" \ + "Install thothctl" \ + "Start and verify readiness" \ + "Configure TLS and upstream authentication" \ + "Operate Pi, drain, and roll back" \ + "Back up and restore" \ + "Diagnostics" \ + "Data-preserving uninstall" + require_text "$guide" "server installation guide" \ + "frontend" \ + "core" \ + "UID/GID 10001" \ + "/srv/thothii" \ + "example operator root" \ + "/run/secrets" \ + "Git-backed workspace registry is the source of truth" \ + "host.docker.internal" \ + "host-gateway" \ + "container 127.0.0.1" \ + "collection" \ + "embedding" \ + "bash scripts/build-local.sh" \ + "@sha256:" \ + "bash scripts/build-thothctl.sh" \ + "thothctl --installation" \ + "curl --fail http://127.0.0.1:8080/health" \ + "https://thoth.example.com" \ + "pi update" \ + "--drain" \ + "pi rollback --yes" \ + "pi maintenance recover --yes" \ + "docker compose down --volumes" \ + "reverse-proxy-nginx.md" \ + "reverse-proxy-caddy.md" + node - "$guide" <<'NODE' +const fs = require("fs"); +const source = fs.readFileSync(process.argv[2], "utf8"); +if (/omics_portal|chirone|localllm_default|datamart-builder|compose\.production|compose\.psd-local/i.test(source)) { + throw new Error("server installation guide introduces forbidden application coupling"); +} +if (/\/var\/run\/docker\.sock|docker\.sock/i.test(source)) { + throw new Error("server installation guide introduces a Docker socket dependency"); +} +for (const line of source.split(/\n/)) { + const match = line.match(/^\s*([A-Z][A-Z0-9_]*(?:PASSWORD|TOKEN|API_KEY|SECRET)[A-Z0-9_]*)\s*=\s*(\S.*)$/); + if (!match) continue; + const [, name, rawValue] = match; + const value = rawValue.trim(); + if (!/(?:_FILE|_SOURCE)$/.test(name) && value && !/^\$\{?[A-Z_][A-Z0-9_]*\}?$/.test(value)) { + throw new Error("server installation guide embeds a secret value"); + } +} +let inCodeFence = false; +for (const line of source.split(/\n/)) { + if (line.trimStart().startsWith("```")) { + inCodeFence = !inCodeFence; + continue; + } + if (!line.includes("docker compose down --volumes")) continue; + const normalized = line.toLowerCase().replaceAll("*", ""); + if (inCodeFence || !/(do not|never)/.test(normalized) || /^\s*(docker|&?\s*docker)/.test(normalized)) { + throw new Error("server docker compose down --volumes must appear only in an explicit prose prohibition"); + } +} +if (/```(?:sh|bash)\n[\s\S]*?\bdocker compose\s+(?:up|stop|down|restart|pull|build)\b[\s\S]*?```/i.test(source)) { + throw new Error("server lifecycle must use thothctl, not raw Docker Compose"); +} +NODE + echo "server installation guide contract passed" +} + +verify_reverse_proxy_nginx_guide() { + local guide="$root/docs/install/reverse-proxy-nginx.md" + [[ -f "$guide" ]] || { + echo "missing Nginx reverse-proxy guide: docs/install/reverse-proxy-nginx.md" >&2 + return 1 + } + require_headings "$guide" "Nginx reverse-proxy guide" \ + "Trust boundary" \ + "Example configuration" \ + "Validate and reload" \ + "Test authentication and SSE" + require_text "$guide" "Nginx reverse-proxy guide" \ + "Forwarding identity headers alone does not authenticate a user" \ + "authentication gateway" \ + "2xx" \ + "TLS" \ + "frontend" + node - "$guide" <<'NODE' +const fs = require("fs"); +const source = fs.readFileSync(process.argv[2], "utf8"); +const block = [...source.matchAll(/```nginx\n([\s\S]*?)```/g)].map((match) => match[1]).join("\n"); +const tokens = [ + "listen 443 ssl;", "ssl_certificate ", "ssl_certificate_key ", + "location = /_authenticate {", "internal;", "proxy_pass http://auth-gateway:4180/verify;", + "auth_request /_authenticate;", "auth_request_set $thoth_principal_subject", + "$upstream_http_x_thoth_principal_subject", "proxy_pass http://127.0.0.1:8080;", + "proxy_http_version 1.1;", "proxy_buffering off;", "proxy_cache off;", + "proxy_read_timeout 3600s;", "proxy_set_header X-Thoth-Principal-Subject \"\";", + "proxy_set_header X-Thoth-Trusted-Principal-Subject $thoth_principal_subject;", +]; +if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(block) || !block.includes("http://127.0.0.1:8080")) { + throw new Error("Nginx proxy must forward only to frontend on 127.0.0.1:8080"); +} +for (const token of tokens) { + if (!block.includes(token)) throw new Error(`Nginx proxy lacks structural token: ${token}`); +} +if (/proxy_set_header\s+X-Thoth-Trusted-[^;]+\$http_/i.test(block)) { + throw new Error("Nginx proxy trusts a client-supplied identity header"); +} +NODE + echo "Nginx reverse-proxy guide contract passed" +} + +verify_reverse_proxy_caddy_guide() { + local guide="$root/docs/install/reverse-proxy-caddy.md" + [[ -f "$guide" ]] || { + echo "missing Caddy reverse-proxy guide: docs/install/reverse-proxy-caddy.md" >&2 + return 1 + } + require_headings "$guide" "Caddy reverse-proxy guide" \ + "Trust boundary" \ + "Example configuration" \ + "Validate and reload" \ + "Test authentication and SSE" + require_text "$guide" "Caddy reverse-proxy guide" \ + "Forwarding identity headers alone does not authenticate a user" \ + "authentication gateway" \ + "2xx" \ + "automatic HTTPS" \ + "frontend" + node - "$guide" <<'NODE' +const fs = require("fs"); +const source = fs.readFileSync(process.argv[2], "utf8"); +const block = [...source.matchAll(/```caddyfile\n([\s\S]*?)```/g)].map((match) => match[1]).join("\n"); +const tokens = [ + "thoth.example.com {", "route {", + "request_header -X-Thoth-Principal-Subject", + "request_header -X-Thoth-Trusted-Principal-Subject", + "forward_auth auth-gateway:4180 {", "uri /verify", "copy_headers {", + "X-Thoth-Principal-Subject>X-Thoth-Trusted-Principal-Subject", + "reverse_proxy 127.0.0.1:8080 {", "flush_interval -1", +]; +if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(block) || !block.includes("127.0.0.1:8080")) { + throw new Error("Caddy proxy must forward only to frontend on 127.0.0.1:8080"); +} +for (const token of tokens) { + if (!block.includes(token)) throw new Error(`Caddy proxy lacks structural token: ${token}`); +} +NODE + echo "Caddy reverse-proxy guide contract passed" +} + verify_manual() { local profile="$1" manual manual="$root/docs/install/$profile-workspace-registry.md" @@ -520,11 +685,27 @@ verify_manual() { return 1 } done - for expected in \ - 'export THT_SOURCE_ROOT=/absolute/path/to/ThothII' \ - '--env-file "$THT_OPERATOR_ENV"' \ - "-f \"\$THT_SOURCE_ROOT/compose.yaml\" -f \"\$THT_SOURCE_ROOT/deploy/compose.$profile.yaml\"" \ - '"$THT_SOURCE_ROOT/scripts/generate-connector-secrets-override.sh"'; do + local -a expected_steps + if [[ "$profile" == local ]]; then + expected_steps=( + 'export THT_SOURCE_ROOT=/absolute/path/to/ThothII' + '--env-file "$THT_OPERATOR_ENV"' + "-f \"\$THT_SOURCE_ROOT/compose.yaml\" -f \"\$THT_SOURCE_ROOT/deploy/compose.$profile.yaml\"" + '"$THT_SOURCE_ROOT/scripts/generate-connector-secrets-override.sh"' + ) + else + expected_steps=( + 'THTCTL=/srv/thothii/operator/thothctl' + 'INSTALLATION=/srv/thothii/operator/thothii-installation.yaml' + '"$THTCTL" --installation "$INSTALLATION" start' + '"$THTCTL" --installation "$INSTALLATION" doctor' + 'docs/install/examples/thothii-installation.server.yaml' + 'compose.yaml' + 'deploy/compose.server.yaml' + 'server.md' + ) + fi + for expected in "${expected_steps[@]}"; do grep -Fq -- "$expected" "$manual" || { echo "$profile manual lacks canonical operator step: $expected" >&2 return 1 @@ -633,6 +814,136 @@ NODE echo "local installation example rendered from path with spaces passed" } +verify_server_installation_example() { + local example="$root/docs/install/examples/thothii-installation.server.yaml" + [[ -f "$example" ]] || { + echo "missing server installation example: docs/install/examples/thothii-installation.server.yaml" >&2 + return 1 + } + + local fixture source_copy operator_dir copied_example connector_override env_file + fixture="$(mktemp -d "${TMPDIR%/}/thoth server install.XXXXXX")" + trap 'rm -rf "$fixture"' RETURN + [[ "$fixture" == *" "* ]] || { + echo "server installation fixture path does not contain spaces" >&2 + return 1 + } + source_copy="$fixture/ThothII server source" + operator_dir="$fixture/server operator files" + mkdir -p "$source_copy/deploy/pi" "$source_copy/deploy/workspaces" \ + "$operator_dir/data" "$operator_dir/pi-state" "$operator_dir/workspace-registry" + cp "$root/compose.yaml" "$source_copy/compose.yaml" + cp "$root/deploy/compose.server.yaml" "$source_copy/deploy/compose.server.yaml" + cp "$root/deploy/compose.session-server.yaml.example" \ + "$source_copy/deploy/compose.session-server.yaml.example" + cp "$root/deploy/compose.git-ssh.yaml" "$source_copy/deploy/compose.git-ssh.yaml" + cp "$root/deploy/pi/models.json" "$source_copy/deploy/pi/models.json" + cp "$root/deploy/pi/settings.json" "$source_copy/deploy/pi/settings.json" + cp "$root/deploy/workspaces/server-sessions.yaml.example" \ + "$source_copy/deploy/workspaces/server-sessions.yaml.example" + + write_private "$operator_dir/pi-auth.json" '{"zai":{"type":"api_key","key":"fixture-server-pi-key"}}' + write_private "$operator_dir/thothii.secrets" 'THT_MODEL_API_KEY=fixture-server-model-key' + write_private "$operator_dir/git-ssh-key" 'fixture-server-ssh-key' + write_private "$operator_dir/git-known-hosts" 'fixture-server-known-hosts' + write_private "$operator_dir/dwh-password" 'fixture-server-dwh-password' + write_private "$operator_dir/session-runtime-password" 'fixture-server-session-runtime-password' + write_private "$operator_dir/session-migrator-password" 'fixture-server-session-migrator-password' + write_private "$operator_dir/session-ca.pem" 'fixture-server-session-ca' + printf '%s\n' \ + 'THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=postgres_direct' \ + 'THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password' \ + >"$operator_dir/workspace-bindings.env" + env_file="$operator_dir/server.env" + printf '%s\n' \ + 'THOTH_SERVER_BIND=127.0.0.1' \ + 'THOTH_HTTP_PORT=8080' \ + 'THT_WORKSPACE_GIT_REMOTE=ssh://git@git.example.invalid/platform/thoth-workspaces.git' \ + 'THT_WORKSPACE_GIT_BRANCH=main' \ + "PI_AUTH_FILE=$operator_dir/pi-auth.json" \ + "THT_SECRETS_FILE=$operator_dir/thothii.secrets" \ + "THT_WORKSPACE_BINDINGS_ENV_FILE=$operator_dir/workspace-bindings.env" \ + "THT_WORKSPACE_GIT_SSH_KEY_FILE=$operator_dir/git-ssh-key" \ + "THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=$operator_dir/git-known-hosts" \ + "THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=$operator_dir/dwh-password" \ + "THT_DATA_ROOT=$operator_dir/data" \ + "THT_PI_STATE_ROOT=$operator_dir/pi-state" \ + "THT_WORKSPACE_REGISTRY_ROOT=$operator_dir/workspace-registry" \ + "THT_SERVER_WORKSPACE_CONFIG=$source_copy/deploy/workspaces/server-sessions.yaml.example" \ + 'THT_LLM_URL=https://llm.example.invalid' \ + 'THT_SESSION_DB_HOST=sessions.example.invalid' \ + 'THT_SESSION_DB_NAME=thoth_sessions' \ + 'THT_SESSION_RUNTIME_USER=thoth_sessions_app' \ + 'THT_SESSION_MIGRATOR_USER=thoth_sessions_migrate' \ + "THT_SESSION_RUNTIME_PASSWORD_SOURCE=$operator_dir/session-runtime-password" \ + "THT_SESSION_MIGRATOR_PASSWORD_SOURCE=$operator_dir/session-migrator-password" \ + "THT_SESSION_CA_SOURCE=$operator_dir/session-ca.pem" \ + >"$env_file" + connector_override="$operator_dir/connector-secrets.server.yaml" + "$root/scripts/generate-connector-secrets-override.sh" \ + --bindings-env "$operator_dir/workspace-bindings.env" \ + --operator-env "$env_file" \ + --output "$connector_override" >/dev/null + + copied_example="$fixture/thothii-installation.yaml" + local contents + contents="$(<"$example")" + contents="${contents//\/absolute\/path\/to\/ThothII/$source_copy}" + contents="${contents//\/absolute\/path\/to\/thothii-server-operator/$operator_dir}" + printf '%s\n' "$contents" >"$copied_example" + + local profile project_directory descriptor_env value + local -a overrides files + profile="$(sed -n 's/^profile: \([^[:space:]]*\)$/\1/p' "$copied_example")" + project_directory="$(sed -n 's/^projectDirectory: "\(.*\)"$/\1/p' "$copied_example")" + descriptor_env="$(sed -n 's/^envFile: "\(.*\)"$/\1/p' "$copied_example")" + while IFS= read -r value; do overrides+=("$value"); done < <(sed -n 's/^ - "\(.*\)"$/\1/p' "$copied_example") + [[ "$profile" == server && "$project_directory" == "$source_copy" && "$descriptor_env" == "$env_file" ]] || { + echo "server installation example does not resolve its required fields" >&2 + return 1 + } + [[ "${#overrides[@]}" -eq 3 && "${overrides[0]}" == "$source_copy/deploy/compose.session-server.yaml.example" \ + && "${overrides[2]}" == "$connector_override" ]] || { + echo "server installation example does not select the expected optional overrides" >&2 + return 1 + } + files=(-f "$project_directory/compose.yaml" -f "$project_directory/deploy/compose.$profile.yaml") + for value in "${overrides[@]}"; do files+=(-f "$value"); done + local rendered="$fixture/server-installation.json" + "$root/scripts/compose-with-preflight.sh" --env-file "$descriptor_env" \ + "${files[@]}" config --format json >"$rendered" + node - "$rendered" <<'NODE' +const fs = require("fs"); +const config = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); +if (Object.keys(config.services).sort().join(",") !== "core,frontend") { + throw new Error("server installation example must render exactly core,frontend"); +} +const core = config.services.core; +const frontend = config.services.frontend; +if (core.environment?.AUTH_MODE !== "upstream" || core.environment?.THOTH_PUBLIC_EXPOSURE !== "true") { + throw new Error("server installation example must fail closed behind upstream authentication"); +} +if ((core.ports || []).length !== 0) throw new Error("server installation example published core"); +const ports = frontend.ports || []; +if (ports.length !== 1 || ports[0].host_ip !== "127.0.0.1" || Number(ports[0].target) !== 8080) { + throw new Error("server installation example must publish only loopback frontend"); +} +const rendered = JSON.stringify(config); +if (/omics_portal|chirone|localllm_default|datamart-builder/i.test(rendered)) { + throw new Error("server installation example contains application coupling"); +} +for (const secret of [ + "fixture-server-pi-key", "fixture-server-model-key", "fixture-server-ssh-key", + "fixture-server-known-hosts", "fixture-server-dwh-password", + "fixture-server-session-runtime-password", "fixture-server-session-migrator-password", + "fixture-server-session-ca", +]) { + if (rendered.includes(secret)) throw new Error("server installation rendering exposed a fixture secret"); +} +NODE + echo "server installation example rendered from path with spaces passed" +} + write_private() { local path="$1" value="$2" printf '%s\n' "$value" >"$path" @@ -772,7 +1083,11 @@ case "$mode" in verify_local_guide verify_windows_line_endings_guide verify_pi_management_guide + verify_server_guide + verify_reverse_proxy_nginx_guide + verify_reverse_proxy_caddy_guide verify_local_installation_example + verify_server_installation_example verify_manual local verify_manual server verify_compose_fixtures @@ -786,6 +1101,11 @@ case "$mode" in verify_windows_line_endings_guide verify_pi_management_guide verify_local_installation_example + else + verify_server_guide + verify_reverse_proxy_nginx_guide + verify_reverse_proxy_caddy_guide + verify_server_installation_example fi verify_manual "$profile" verify_compose_fixtures