mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix: make server operations executable

Browse Source
This commit is contained in:
marcopan
2026-08-05 11:06:34 +02:00
parent a707fb442c
commit 96fe5bfa79
11 changed files with 1299 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/env/server.env.example
Vendored
+1
View File
@@ -9,6 +9,7 @@ THT_SECRETS_FILE=/absolute/path/to/thothii.secrets
THT_DATA_ROOT=/srv/thothii/data THT_DATA_ROOT=/srv/thothii/data
THT_PI_STATE_ROOT=/srv/thothii/pi-state THT_PI_STATE_ROOT=/srv/thothii/pi-state
THT_WORKSPACE_REGISTRY_ROOT=/srv/thothii/workspace-registry THT_WORKSPACE_REGISTRY_ROOT=/srv/thothii/workspace-registry
THT_BACKUP_ROOT=/srv/thothii-backups
THT_SERVER_WORKSPACE_CONFIG=/absolute/path/to/server-sessions.yaml THT_SERVER_WORKSPACE_CONFIG=/absolute/path/to/server-sessions.yaml
THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git
THT_WORKSPACE_GIT_BRANCH=main THT_WORKSPACE_GIT_BRANCH=main
docs/install/server-workspace-registry.md
+6 -4
View File
@@ -14,8 +14,8 @@ first startup. Keep storage separated:
```text ```text
/srv/thothii/data/ # settings, session data, Pi state as applicable /srv/thothii/data/ # settings, session data, Pi state as applicable
/srv/thothii/workspace-registry/ # repo/, snapshots/, state/, locks/ /srv/thothii/workspace-registry/ # repo/, snapshots/, state/, locks/
/srv/thothii/secrets/ # Git and connector secret files, mode 0700 /srv/thothii/secrets/ # Git and connector secret files, setgid mode 2750
/srv/thothii/operator/ # untracked Compose/.env, mode 0700 /srv/thothii/operator/ # untracked Compose/.env, setgid mode 2750
``` ```
Permit outbound TCP only to approved Git/Gitea, DWH, vector, embedding, and bastion endpoints. Permit outbound TCP only to approved Git/Gitea, DWH, vector, embedding, and bastion endpoints.
@@ -41,8 +41,10 @@ authoring environment for migration.
## Git credentials, CA, SSH key, and known-hosts mounts ## Git credentials, CA, SSH key, and known-hosts mounts
Use the secret manager or a protected host-only procedure to create independent regular files under Use the secret manager or a protected host-only procedure to create independent regular files under
`/srv/thothii/secrets`. Set individual mode `0600`, directory mode `0700`, and ownership readable `/srv/thothii/secrets`. As established in the server guide, use owner UID 10001, group
by the service account. These path-only variables are mounted read-only by Compose: `thothii-ops`, file mode `0640`, and setgid directory mode `2750`. This lets the UID 10001
container and the reviewed Docker operator running `thothctl` read the files without making them
public. These path-only variables are mounted read-only by Compose:
```dotenv ```dotenv
THT_WORKSPACE_GIT_CREDENTIALS_FILE=/srv/thothii/secrets/git-credentials THT_WORKSPACE_GIT_CREDENTIALS_FILE=/srv/thothii/secrets/git-credentials
docs/install/server.md
+123 -25
View File
@@ -43,19 +43,29 @@ sudo useradd --system --uid 10001 --user-group --home-dir /srv/thothii \
--create-home --shell /usr/sbin/nologin thothii --create-home --shell /usr/sbin/nologin thothii
``` ```
The human operator who runs `thothctl` needs Docker access. On many installations membership in Use a dedicated `thothii-ops` group for the small set of human operators. A human who runs
the `docker` group is effectively host-root access; grant it only according to site policy. The `thothctl` must be in both `thothii-ops` (to traverse operator paths and read declared secret files
non-login `thothii` account owns application data and secrets but does not itself need Docker for output redaction) and the host `docker` group (to invoke Docker). Docker-group membership is
access. effectively host-root access; grant both memberships only to reviewed administrators. The
non-login `thothii` account owns files and writable data but does not need Docker access.
```sh
sudo groupadd --system thothii-ops
sudo usermod --append --groups thothii-ops,docker "$USER"
```
Log out and back in before continuing; `id` must show both groups. Do not run `thothctl` through
`sudo -u thothii`: that account deliberately lacks Docker access. Do not grant the human direct
write access to runtime bind trees.
Create explicit directories. `source` contains the clone; `operator` contains untracked path-only Create explicit directories. `source` contains the clone; `operator` contains untracked path-only
configuration; the three writable trees are bind-mounted into `core`; `secrets` contains regular configuration; the three writable trees are bind-mounted into `core`; `secrets` contains regular
files only. Backups are separate from live data. files only. Backups are separate from live data.
```sh ```sh
sudo install -d -o 10001 -g 10001 -m 0750 /srv/thothii/source sudo install -d -o 10001 -g thothii-ops -m 2750 /srv/thothii/source
sudo install -d -o 10001 -g 10001 -m 0700 /srv/thothii/operator sudo install -d -o 10001 -g thothii-ops -m 2750 /srv/thothii/operator
sudo install -d -o 10001 -g 10001 -m 0700 /srv/thothii/secrets sudo install -d -o 10001 -g thothii-ops -m 2750 /srv/thothii/secrets
sudo install -d -o 10001 -g 10001 -m 0750 /srv/thothii/data sudo install -d -o 10001 -g 10001 -m 0750 /srv/thothii/data
sudo install -d -o 10001 -g 10001 -m 0750 /srv/thothii/pi-state sudo install -d -o 10001 -g 10001 -m 0750 /srv/thothii/pi-state
sudo install -d -o 10001 -g 10001 -m 0750 /srv/thothii/workspace-registry sudo install -d -o 10001 -g 10001 -m 0750 /srv/thothii/workspace-registry
@@ -103,8 +113,50 @@ services:
``` ```
Use `host.docker.internal` in the endpoint binding. The `host-gateway` mapping supplies routing; Use `host.docker.internal` in the endpoint binding. The `host-gateway` mapping supplies routing;
it does not bundle or trust the target service. Keep the target port bound/firewalled for Docker it does not bundle or trust the target service. A host service listening only on host
host access only. A stable internal DNS record is the preferred alternative. `127.0.0.1` is **not reachable** through this mapping. Bind that service to the ThothII Docker
bridge gateway address or to a dedicated private host interface—never to `0.0.0.0` merely to make
the check pass. A stable internal DNS record routed through an authenticated private listener is
the preferred alternative.
After the first bounded start attempt, copy the exact core container name from `thothctl status`
into `CORE_NAME`, then derive—not guess—the network ID, Linux bridge interface, gateway, and
subnet. Compose networks normally use `br-<first-12-network-id>`; an explicit
`com.docker.network.bridge.name` option takes precedence:
```sh
CORE_NAME=replace-with-exact-core-container-name
NETWORK_ID=$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.NetworkID}}{{end}}' "$CORE_NAME")
NETWORK_NAME=$(docker network inspect --format '{{.Name}}' "$NETWORK_ID")
BRIDGE=$(docker network inspect --format '{{index .Options "com.docker.network.bridge.name"}}' "$NETWORK_ID")
test -n "$BRIDGE" || BRIDGE="br-${NETWORK_ID%${NETWORK_ID#????????????}}"
GATEWAY=$(docker network inspect --format '{{(index .IPAM.Config 0).Gateway}}' "$NETWORK_ID")
SUBNET=$(docker network inspect --format '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK_ID")
printf 'network=%s bridge=%s gateway=%s subnet=%s\n' "$NETWORK_NAME" "$BRIDGE" "$GATEWAY" "$SUBNET"
ip address show dev "$BRIDGE"
```
Bind the co-resident service to `$GATEWAY`. In the host firewall `INPUT` chain, allow its exact
TCP port only when source is `$SUBNET`, input interface is `$BRIDGE`, and destination is
`$GATEWAY`; reject other sources to that listener and persist the rules using the distribution's
firewall manager. Docker's `DOCKER-USER` chain governs forwarded/published traffic and does not
replace this host-input rule. Ask the firewall administrator to implement the equivalent policy
with nftables when iptables is not the site's source of truth.
For an iptables-managed host, replace the port before applying these reviewed rules; the second
rule prevents any other interface/source from reaching that gateway listener:
```sh
EXTERNAL_PORT=replace-with-exact-service-port
sudo iptables -I INPUT 1 -i "$BRIDGE" -s "$SUBNET" -d "$GATEWAY" -p tcp --dport "$EXTERNAL_PORT" -j ACCEPT
sudo iptables -I INPUT 2 -d "$GATEWAY" -p tcp --dport "$EXTERNAL_PORT" -j REJECT
```
Confirm reachability with `thothctl pi test` for the configured LLM/Pi path and with the
authenticated Workspace Diagnostics action for DWH, vector collection/embedding pairing, and
embedding endpoints. A timeout paired with `ss -lntp`, `ip address show dev "$BRIDGE"`, and the
firewall counters distinguishes a loopback bind from a subnet/interface rule failure. Do not add
a shell to the browser or mount the Docker socket into core for this diagnostic.
Configure each boundary independently: Configure each boundary independently:
@@ -136,7 +188,9 @@ Copy the path-only server environment and installation descriptor:
sudo -u thothii cp deploy/env/server.env.example /srv/thothii/operator/server.env sudo -u thothii cp deploy/env/server.env.example /srv/thothii/operator/server.env
sudo -u thothii cp docs/install/examples/thothii-installation.server.yaml \ sudo -u thothii cp docs/install/examples/thothii-installation.server.yaml \
/srv/thothii/operator/thothii-installation.yaml /srv/thothii/operator/thothii-installation.yaml
sudo chmod 0600 /srv/thothii/operator/server.env \ sudo chown 10001:thothii-ops /srv/thothii/operator/server.env \
/srv/thothii/operator/thothii-installation.yaml
sudo chmod 0640 /srv/thothii/operator/server.env \
/srv/thothii/operator/thothii-installation.yaml /srv/thothii/operator/thothii-installation.yaml
``` ```
@@ -146,15 +200,16 @@ session-server overlay and generated connector-secret override. Optional host-ga
image overrides go after them. image overrides go after them.
Create each credential as an independent regular file in `/srv/thothii/secrets`, owned by Create each credential as an independent regular file in `/srv/thothii/secrets`, owned by
UID/GID 10001 and mode `0600`. The operator environment records only absolute `*_FILE` or UID 10001, group `thothii-ops`, and mode `0640`. Owner access lets the UID 10001 container read a
file mounted under `/run/secrets`; group access lets the reviewed human run `thothctl`. The
operator environment records only absolute `*_FILE` or
`*_SOURCE` paths. Compose mounts application and connector targets read-only under `/run/secrets`; `*_SOURCE` paths. Compose mounts application and connector targets read-only under `/run/secrets`;
the frontend receives none. Do not print file contents while testing permissions. the frontend receives none. Do not print file contents while testing permissions.
```sh ```sh
sudo find /srv/thothii/secrets -type f -exec chown 10001:10001 {} + sudo find /srv/thothii/secrets -type f -exec chown 10001:thothii-ops {} +
sudo find /srv/thothii/secrets -type f -exec chmod 0600 {} + sudo find /srv/thothii/secrets -type f -exec chmod 0640 {} +
sudo find /srv/thothii/secrets -type f ! -user thothii -print sudo find /srv/thothii/secrets -type f \( ! -user thothii -o ! -group thothii-ops -o ! -perm 0640 \) -print
sudo find /srv/thothii/secrets -type f ! -perm 0600 -print
``` ```
Add `THT_WORKSPACE_BINDINGS_ENV_FILE=/srv/thothii/operator/workspace-bindings.env` and the matching Add `THT_WORKSPACE_BINDINGS_ENV_FILE=/srv/thothii/operator/workspace-bindings.env` and the matching
@@ -185,13 +240,18 @@ services:
core: core:
build: !reset null build: !reset null
image: registry.example.com/thothii/core@sha256:<64-lowercase-hex-digits> image: registry.example.com/thothii/core@sha256:<64-lowercase-hex-digits>
session-migrate:
build: !reset null
image: registry.example.com/thothii/core@sha256:<64-lowercase-hex-digits>
frontend: frontend:
build: !reset null build: !reset null
image: registry.example.com/thothii/frontend@sha256:<64-lowercase-hex-digits> image: registry.example.com/thothii/frontend@sha256:<64-lowercase-hex-digits>
``` ```
Add that absolute file last in `overrides`. Both images must come from one compatible release; the Add that absolute file last in `overrides`. `core` and `session-migrate` must use the exact same
core image must retain the declared Pi version labels checked by `thothctl pi doctor`. Pull access core digest; neither may retain a local build or `:local` image. Frontend uses its own exact digest.
Both images must come from one compatible release; the core image must retain the declared Pi
version labels checked by `thothctl pi doctor`. Pull access
belongs in the host Docker credential store, not in Compose or the installation descriptor. belongs in the host Docker credential store, not in Compose or the installation descriptor.
## Install thothctl ## Install thothctl
@@ -201,7 +261,7 @@ Build the operator binaries with Docker. No Go installation or Go knowledge is r
```sh ```sh
cd /srv/thothii/source/ThothII cd /srv/thothii/source/ThothII
bash scripts/build-thothctl.sh bash scripts/build-thothctl.sh
sudo install -o 10001 -g 10001 -m 0755 dist/thothctl/thothctl-linux-amd64 \ sudo install -o root -g thothii-ops -m 0750 dist/thothctl/thothctl-linux-amd64 \
/srv/thothii/operator/thothctl /srv/thothii/operator/thothctl
``` ```
@@ -211,7 +271,7 @@ not source `server.env` as shell code:
```sh ```sh
THTCTL=/srv/thothii/operator/thothctl THTCTL=/srv/thothii/operator/thothctl
INSTALLATION=/srv/thothii/operator/thothii-installation.yaml INSTALLATION=/srv/thothii/operator/thothii-installation.yaml
"$THTCTL" --installation "$INSTALLATION" --help "$THTCTL" --help
"$THTCTL" --installation "$INSTALLATION" update --check-only "$THTCTL" --installation "$INSTALLATION" update --check-only
``` ```
@@ -221,7 +281,24 @@ profile, overrides, project identity, and durable current-image selector. The ge
## Start and verify readiness ## Start and verify readiness
Keep the TLS proxy stopped or firewalled during bootstrap: Keep the TLS proxy stopped or firewalled during bootstrap. First stop the app, run the
installation-aware session migration, and inspect its pristine JSON. The command activates only
the `session-migrate` profile/service with `--no-deps --no-TTY`; it derives the migrator image from the
selected core image after all installation overrides, so this procedure is identical for source
and pinned modes. It exits nonzero unless both arrays are empty:
```sh
"$THTCTL" --installation "$INSTALLATION" stop
"$THTCTL" --installation "$INSTALLATION" sessions migrate --yes
```
Successful output has this shape (the `applied` list may contain versions on first use):
```json
{"applied":[],"drifted":[],"pending":[]}
```
Only after seeing `"pending":[]` and `"drifted":[]`, start and verify:
```sh ```sh
"$THTCTL" --installation "$INSTALLATION" start "$THTCTL" --installation "$INSTALLATION" start
@@ -303,7 +380,7 @@ BACKUP=/srv/thothii-backups/2026-08-05
sudo install -d -o root -g root -m 0700 "$BACKUP" sudo install -d -o root -g root -m 0700 "$BACKUP"
sudo tar --numeric-owner --xattrs --acls -C /srv/thothii -czf "$BACKUP/runtime-data.tgz" \ sudo tar --numeric-owner --xattrs --acls -C /srv/thothii -czf "$BACKUP/runtime-data.tgz" \
data pi-state workspace-registry data pi-state workspace-registry
sudo sha256sum "$BACKUP/runtime-data.tgz" >"$BACKUP/SHA256SUMS" sudo sh -ceu 'cd "$1"; sha256sum runtime-data.tgz > SHA256SUMS; sha256sum --check SHA256SUMS' sh "$BACKUP"
``` ```
Back up the installation descriptor, path-only environment, generated overrides, source revision, Back up the installation descriptor, path-only environment, generated overrides, source revision,
@@ -318,7 +395,7 @@ the restored set. This keeps the previous state recoverable:
```sh ```sh
RESTORE=/srv/thothii-restore-2026-08-05 RESTORE=/srv/thothii-restore-2026-08-05
sudo install -d -o root -g root -m 0700 "$RESTORE" sudo install -d -o root -g root -m 0700 "$RESTORE"
sudo sha256sum --check /srv/thothii-backups/2026-08-05/SHA256SUMS sudo sh -ceu 'cd "$1"; sha256sum --check SHA256SUMS' sh /srv/thothii-backups/2026-08-05
sudo tar --numeric-owner --xattrs --acls -C "$RESTORE" \ sudo tar --numeric-owner --xattrs --acls -C "$RESTORE" \
-xzf /srv/thothii-backups/2026-08-05/runtime-data.tgz -xzf /srv/thothii-backups/2026-08-05/runtime-data.tgz
sudo test -d "$RESTORE/workspace-registry/repo" sudo test -d "$RESTORE/workspace-registry/repo"
@@ -357,9 +434,30 @@ external service identity; and the proxy/identity provider for login failures.
## Data-preserving uninstall ## Data-preserving uninstall
Drain and stop through `thothctl`, take and verify one final backup, disable the TLS proxy route, Drain and stop through `thothctl`, take and verify one final backup, and disable the TLS proxy
and remove only this installation's stopped `frontend` and `core` containers and optional images route. Set `THT_BACKUP_ROOT=/srv/thothii-backups` in `server.env`; the removal command verifies the
by their exact Compose project labels. Keep `/srv/thothii/data`, `pi-state`, filesystem identity of that backup root, all three bind trees, and every declared secret before
and after removing anything.
First run without confirmation. It displays the exact installation project, service, container
name, container ID, and stopped state, then exits without mutation. Check every target:
```sh
"$THTCTL" --installation "$INSTALLATION" stop
"$THTCTL" --installation "$INSTALLATION" remove
```
If and only if both targets are the expected stopped `frontend` and `core` containers, confirm:
```sh
"$THTCTL" --installation "$INSTALLATION" remove --yes exact-core-id exact-frontend-id
```
Replace both example IDs with the values from the immediately preceding dry-run. The command
refuses confirmation if the current target set differs. The confirmed operation passes only those
previously displayed immutable container IDs to Docker,
uses no force or volume option, rejects running/replaced containers, and proves the preservation
paths still identify the same filesystem objects. Keep `/srv/thothii/data`, `pi-state`,
`workspace-registry`, `operator`, protected secrets, database backups, and the installation `workspace-registry`, `operator`, protected secrets, database backups, and the installation
descriptor if reinstallation is possible. Do not prune global Docker data. descriptor if reinstallation is possible. Do not prune global Docker data.
scripts/test-verify-workspace-install-docs.sh
+113 -1
View File
@@ -20,6 +20,8 @@ for fixture in \
"Caddy reverse-proxy guide contract" \ "Caddy reverse-proxy guide contract" \
"local installation example rendered from path with spaces" \ "local installation example rendered from path with spaces" \
"server installation example rendered from path with spaces" \ "server installation example rendered from path with spaces" \
"server pinned migration image fixture" \
"server backup checksum root-only fixture" \
"local manual canonical base+override references" \ "local manual canonical base+override references" \
"server manual canonical base+override references" \ "server manual canonical base+override references" \
"canonical local base+override fixture" \ "canonical local base+override fixture" \
@@ -33,6 +35,32 @@ for fixture in \
} }
done done
server_guide="$root/docs/install/server.md"
for required in \
'thothii-ops' \
'THT_BACKUP_ROOT=/srv/thothii-backups' \
'sessions migrate --yes' \
'"pending":[]' \
'"drifted":[]' \
'remove --yes' \
'sha256sum --check SHA256SUMS' \
'DOCKER-USER' \
'iptables -I INPUT' \
'com.docker.network.bridge.name'; do
grep -Fq -- "$required" "$server_guide" || {
echo "server operations guide lacks executable contract: $required" >&2
exit 1
}
done
grep -Fq '"$THTCTL" --help' "$server_guide" || {
echo "server guide lacks plain thothctl --help" >&2
exit 1
}
if grep -Fq '"$THTCTL" --installation "$INSTALLATION" --help' "$server_guide"; then
echo "server guide still uses installation-scoped --help" >&2
exit 1
fi
for manual in "$root/docs/install/local-workspace-registry.md"; do for manual in "$root/docs/install/local-workspace-registry.md"; do
grep -Fq 'export THT_SOURCE_ROOT=/absolute/path/to/ThothII' "$manual" || { grep -Fq 'export THT_SOURCE_ROOT=/absolute/path/to/ThothII' "$manual" || {
echo "installation manual does not publish a self-contained THT_SOURCE_ROOT export: $manual" >&2 echo "installation manual does not publish a self-contained THT_SOURCE_ROOT export: $manual" >&2
@@ -114,6 +142,18 @@ switch (mutation) {
case "server-coupling": case "server-coupling":
changed += "\nAttach core to the omics_portal application network.\n"; changed += "\nAttach core to the omics_portal application network.\n";
break; break;
case "server-host-loopback":
changed += "\nFor host-gateway, keep the external service listening on 127.0.0.1.\n";
break;
case "server-raw-remove":
changed += "\n```sh\ndocker rm thothii-core thothii-frontend\n```\n";
break;
case "server-pinned-migrator-mismatch":
changed += "\n```yaml\nservices:\n core:\n image: registry.invalid/core@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n session-migrate:\n image: thothii-core:local\n```\n";
break;
case "server-pinned-frontend-missing":
changed = original.replace(' frontend:\n build: !reset null\n image: registry.example.com/thothii/frontend@sha256:<64-lowercase-hex-digits>\n', '');
break;
case "nginx-no-auth": case "nginx-no-auth":
changed = original.replace(" auth_request /_authenticate;", " # authentication omitted"); changed = original.replace(" auth_request /_authenticate;", " # authentication omitted");
break; break;
@@ -123,6 +163,18 @@ switch (mutation) {
case "nginx-no-sse": case "nginx-no-sse":
changed = original.replace(" proxy_buffering off;", " proxy_buffering on;"); changed = original.replace(" proxy_buffering off;", " proxy_buffering on;");
break; break;
case "nginx-no-issuer-clear":
changed = original.replaceAll('proxy_set_header X-Thoth-Principal-Issuer "";', 'proxy_set_header X-Thoth-Principal-Issuer $http_x_thoth_principal_issuer;');
break;
case "nginx-no-subject-capture":
changed = original.replace("auth_request_set $thoth_principal_subject", "# missing auth capture $thoth_principal_subject");
break;
case "nginx-no-display-map":
changed = original.replace("proxy_set_header X-Thoth-Trusted-Principal-Display-Name $thoth_principal_display_name;", "proxy_set_header X-Thoth-Trusted-Principal-Display-Name \"\";");
break;
case "nginx-no-admin-map":
changed = original.replace("proxy_set_header X-Thoth-Trusted-Is-Admin $thoth_is_admin;", "proxy_set_header X-Thoth-Trusted-Is-Admin \"\";");
break;
case "caddy-no-auth": case "caddy-no-auth":
changed = original.replace("forward_auth auth-gateway:4180 {", "# forward authentication omitted"); changed = original.replace("forward_auth auth-gateway:4180 {", "# forward authentication omitted");
break; break;
@@ -132,6 +184,18 @@ switch (mutation) {
case "caddy-core-upstream": case "caddy-core-upstream":
changed = original.replaceAll("127.0.0.1:8080", "127.0.0.1:8787"); changed = original.replaceAll("127.0.0.1:8080", "127.0.0.1:8787");
break; break;
case "caddy-no-issuer-public-clear":
changed = original.replace("request_header -X-Thoth-Principal-Issuer", "request_header X-Thoth-Principal-Issuer {header.X-Thoth-Principal-Issuer}");
break;
case "caddy-no-subject-trusted-clear":
changed = original.replace("request_header -X-Thoth-Trusted-Principal-Subject", "request_header X-Thoth-Trusted-Principal-Subject {header.X-Thoth-Trusted-Principal-Subject}");
break;
case "caddy-no-display-map":
changed = original.replace("X-Thoth-Principal-Display-Name>X-Thoth-Trusted-Principal-Display-Name", "X-Thoth-Principal-Display-Name");
break;
case "caddy-no-admin-map":
changed = original.replace("X-Thoth-Is-Admin>X-Thoth-Trusted-Is-Admin", "X-Thoth-Is-Admin");
break;
case "dirty-source": case "dirty-source":
changed = original.replaceAll("git status --porcelain --untracked-files=all", "git status --short"); changed = original.replaceAll("git status --porcelain --untracked-files=all", "git status --short");
break; break;
@@ -215,6 +279,22 @@ expect_guide_rejected \
"server application coupling" verify_server_guide \ "server application coupling" verify_server_guide \
"$root/docs/install/server.md" docs/install/server.md server-coupling \ "$root/docs/install/server.md" docs/install/server.md server-coupling \
"server installation guide introduces forbidden application coupling" "server installation guide introduces forbidden application coupling"
expect_guide_rejected \
"server host-gateway loopback listener" verify_server_guide \
"$root/docs/install/server.md" docs/install/server.md server-host-loopback \
"server host-gateway guidance assumes a host loopback listener"
expect_guide_rejected \
"server raw container removal" verify_server_guide \
"$root/docs/install/server.md" docs/install/server.md server-raw-remove \
"server uninstall bypasses installation-aware removal"
expect_guide_rejected \
"server pinned migrator differs from core" verify_server_guide \
"$root/docs/install/server.md" docs/install/server.md server-pinned-migrator-mismatch \
"server pinned migration image must equal the pinned core image"
expect_guide_rejected \
"server pinned frontend is missing" verify_server_guide \
"$root/docs/install/server.md" docs/install/server.md server-pinned-frontend-missing \
"server pinned image override must pin core, session-migrate, and frontend without builds"
expect_guide_rejected \ expect_guide_rejected \
"Nginx identity without authentication" verify_reverse_proxy_nginx_guide \ "Nginx identity without authentication" verify_reverse_proxy_nginx_guide \
"$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-auth \ "$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-auth \
@@ -227,6 +307,22 @@ expect_guide_rejected \
"Nginx buffered SSE" verify_reverse_proxy_nginx_guide \ "Nginx buffered SSE" verify_reverse_proxy_nginx_guide \
"$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-sse \ "$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-sse \
"Nginx proxy lacks structural token: proxy_buffering off;" "Nginx proxy lacks structural token: proxy_buffering off;"
expect_guide_rejected \
"Nginx issuer inbound claim not cleared" verify_reverse_proxy_nginx_guide \
"$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-issuer-clear \
"Nginx proxy does not clear inbound issuer identity"
expect_guide_rejected \
"Nginx subject auth response not captured" verify_reverse_proxy_nginx_guide \
"$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-subject-capture \
"Nginx proxy does not capture authenticated subject identity"
expect_guide_rejected \
"Nginx display identity not mapped to private hop" verify_reverse_proxy_nginx_guide \
"$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-display-map \
"Nginx proxy does not map authenticated display identity"
expect_guide_rejected \
"Nginx admin identity not mapped to private hop" verify_reverse_proxy_nginx_guide \
"$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-admin-map \
"Nginx proxy does not map authenticated admin identity"
expect_guide_rejected \ expect_guide_rejected \
"Caddy identity without authentication" verify_reverse_proxy_caddy_guide \ "Caddy identity without authentication" verify_reverse_proxy_caddy_guide \
"$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-auth \ "$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-auth \
@@ -234,11 +330,27 @@ expect_guide_rejected \
expect_guide_rejected \ expect_guide_rejected \
"Caddy untrusted identity forwarding" verify_reverse_proxy_caddy_guide \ "Caddy untrusted identity forwarding" verify_reverse_proxy_caddy_guide \
"$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-client-identity \ "$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-client-identity \
"Caddy proxy lacks structural token: X-Thoth-Principal-Subject>X-Thoth-Trusted-Principal-Subject" "Caddy proxy does not map authenticated subject identity"
expect_guide_rejected \ expect_guide_rejected \
"Caddy direct core exposure" verify_reverse_proxy_caddy_guide \ "Caddy direct core exposure" verify_reverse_proxy_caddy_guide \
"$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-core-upstream \ "$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-core-upstream \
"Caddy proxy must forward only to frontend on 127.0.0.1:8080" "Caddy proxy must forward only to frontend on 127.0.0.1:8080"
expect_guide_rejected \
"Caddy issuer inbound claim not cleared" verify_reverse_proxy_caddy_guide \
"$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-issuer-public-clear \
"Caddy proxy does not clear inbound issuer identity"
expect_guide_rejected \
"Caddy subject private-hop claim not cleared" verify_reverse_proxy_caddy_guide \
"$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-subject-trusted-clear \
"Caddy proxy does not clear inbound trusted subject identity"
expect_guide_rejected \
"Caddy display identity not mapped to private hop" verify_reverse_proxy_caddy_guide \
"$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-display-map \
"Caddy proxy does not map authenticated display identity"
expect_guide_rejected \
"Caddy admin identity not mapped to private hop" verify_reverse_proxy_caddy_guide \
"$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-admin-map \
"Caddy proxy does not map authenticated admin identity"
expect_guide_rejected \ expect_guide_rejected \
"dirty or untracked source tree" verify_local_guide \ "dirty or untracked source tree" verify_local_guide \
scripts/verify-workspace-install-docs.sh
+140 -9
View File
@@ -512,12 +512,16 @@ verify_server_guide() {
"frontend" \ "frontend" \
"core" \ "core" \
"UID/GID 10001" \ "UID/GID 10001" \
"thothii-ops" \
"/srv/thothii" \ "/srv/thothii" \
"example operator root" \ "example operator root" \
"/run/secrets" \ "/run/secrets" \
"Git-backed workspace registry is the source of truth" \ "Git-backed workspace registry is the source of truth" \
"host.docker.internal" \ "host.docker.internal" \
"host-gateway" \ "host-gateway" \
"com.docker.network.bridge.name" \
"DOCKER-USER" \
"iptables -I INPUT" \
"container 127.0.0.1" \ "container 127.0.0.1" \
"collection" \ "collection" \
"embedding" \ "embedding" \
@@ -525,6 +529,12 @@ verify_server_guide() {
"@sha256:" \ "@sha256:" \
"bash scripts/build-thothctl.sh" \ "bash scripts/build-thothctl.sh" \
"thothctl --installation" \ "thothctl --installation" \
"sessions migrate --yes" \
'"pending":[]' \
'"drifted":[]' \
"remove --yes" \
"THT_BACKUP_ROOT=/srv/thothii-backups" \
"sha256sum --check SHA256SUMS" \
"curl --fail http://127.0.0.1:8080/health" \ "curl --fail http://127.0.0.1:8080/health" \
"https://thoth.example.com" \ "https://thoth.example.com" \
"pi update" \ "pi update" \
@@ -564,6 +574,36 @@ for (const line of source.split(/\n/)) {
throw new Error("server docker compose down --volumes must appear only in an explicit prose prohibition"); throw new Error("server docker compose down --volumes must appear only in an explicit prose prohibition");
} }
} }
if (/```(?:sh|bash)\n[\s\S]*?\bdocker\s+rm\b[\s\S]*?```/i.test(source)) {
throw new Error("server uninstall bypasses installation-aware removal");
}
if (/host-gateway[^\n]{0,120}(?:listen|listening|bound)[^\n]{0,80}127\.0\.0\.1|(?:listen|listening|bound)[^\n]{0,80}127\.0\.0\.1[^\n]{0,120}host-gateway/i.test(source)) {
throw new Error("server host-gateway guidance assumes a host loopback listener");
}
const pinnedStart = source.indexOf("## Build locally or select pinned images");
const pinnedEnd = source.indexOf("\n## ", pinnedStart + 3);
const pinnedSection = source.slice(pinnedStart, pinnedEnd < 0 ? source.length : pinnedEnd);
const pinnedBlock = [...pinnedSection.matchAll(/```yaml\n([\s\S]*?)```/g)].map((match) => match[1])
.find((block) => block.includes("session-migrate:")) || "";
function pinnedService(name) {
const match = pinnedBlock.match(new RegExp(`^ ${name}:\\n((?: [^\\n]*\\n)+)`, "m"));
return match ? match[1] : "";
}
const pinnedCore = pinnedService("core");
const pinnedMigrator = pinnedService("session-migrate");
const pinnedFrontend = pinnedService("frontend");
const coreImage = pinnedCore.match(/image:\s*(\S+)/)?.[1];
const migratorImage = pinnedMigrator.match(/image:\s*(\S+)/)?.[1];
const frontendImage = pinnedFrontend.match(/image:\s*(\S+)/)?.[1];
if (![pinnedCore, pinnedMigrator, pinnedFrontend].every((block) => block.includes("build: !reset null")) ||
!coreImage || coreImage !== migratorImage || !/@sha256:<64-lowercase-hex-digits>$/.test(coreImage) ||
!frontendImage || !/@sha256:<64-lowercase-hex-digits>$/.test(frontendImage)) {
throw new Error("server pinned image override must pin core, session-migrate, and frontend without builds");
}
if (/session-migrate:[\s\S]{0,180}image:\s*thothii-core:local/.test(source) &&
/core:[\s\S]{0,180}image:\s*registry\.[^\n]+@sha256:[a-f0-9]{64}/.test(source)) {
throw new Error("server pinned migration image must equal the pinned core image");
}
if (/```(?:sh|bash)\n[\s\S]*?\bdocker compose\s+(?:up|stop|down|restart|pull|build)\b[\s\S]*?```/i.test(source)) { if (/```(?:sh|bash)\n[\s\S]*?\bdocker compose\s+(?:up|stop|down|restart|pull|build)\b[\s\S]*?```/i.test(source)) {
throw new Error("server lifecycle must use thothctl, not raw Docker Compose"); throw new Error("server lifecycle must use thothctl, not raw Docker Compose");
} }
@@ -595,11 +635,9 @@ const block = [...source.matchAll(/```nginx\n([\s\S]*?)```/g)].map((match) => ma
const tokens = [ const tokens = [
"listen 443 ssl;", "ssl_certificate ", "ssl_certificate_key ", "listen 443 ssl;", "ssl_certificate ", "ssl_certificate_key ",
"location = /_authenticate {", "internal;", "proxy_pass http://auth-gateway:4180/verify;", "location = /_authenticate {", "internal;", "proxy_pass http://auth-gateway:4180/verify;",
"auth_request /_authenticate;", "auth_request_set $thoth_principal_subject", "auth_request /_authenticate;", "proxy_pass http://127.0.0.1:8080;",
"$upstream_http_x_thoth_principal_subject", "proxy_pass http://127.0.0.1:8080;",
"proxy_http_version 1.1;", "proxy_buffering off;", "proxy_cache off;", "proxy_http_version 1.1;", "proxy_buffering off;", "proxy_cache off;",
"proxy_read_timeout 3600s;", "proxy_set_header X-Thoth-Principal-Subject \"\";", "proxy_read_timeout 3600s;",
"proxy_set_header X-Thoth-Trusted-Principal-Subject $thoth_principal_subject;",
]; ];
if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(block) || !block.includes("http://127.0.0.1:8080")) { if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(block) || !block.includes("http://127.0.0.1:8080")) {
throw new Error("Nginx proxy must forward only to frontend on 127.0.0.1:8080"); throw new Error("Nginx proxy must forward only to frontend on 127.0.0.1:8080");
@@ -610,6 +648,28 @@ for (const token of tokens) {
if (/proxy_set_header\s+X-Thoth-Trusted-[^;]+\$http_/i.test(block)) { if (/proxy_set_header\s+X-Thoth-Trusted-[^;]+\$http_/i.test(block)) {
throw new Error("Nginx proxy trusts a client-supplied identity header"); throw new Error("Nginx proxy trusts a client-supplied identity header");
} }
const identities = [
["issuer", "Principal-Issuer", "thoth_principal_issuer", "x_thoth_principal_issuer"],
["subject", "Principal-Subject", "thoth_principal_subject", "x_thoth_principal_subject"],
["display", "Principal-Display-Name", "thoth_principal_display_name", "x_thoth_principal_display_name"],
["admin", "Is-Admin", "thoth_is_admin", "x_thoth_is_admin"],
];
function escaped(value) { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); }
for (const [label, publicName, variable, upstream] of identities) {
const trustedName = publicName === "Is-Admin" ? "Is-Admin" : publicName;
const publicClears = block.match(new RegExp(`proxy_set_header\\s+X-Thoth-${escaped(publicName)}\\s+"";`, "g")) || [];
if (publicClears.length < 2) throw new Error(`Nginx proxy does not clear inbound ${label} identity`);
const trustedHeader = `X-Thoth-Trusted-${trustedName}`;
if (!new RegExp(`proxy_set_header\\s+${escaped(trustedHeader)}\\s+"";`).test(block)) {
throw new Error(`Nginx proxy does not clear inbound trusted ${label} identity`);
}
if (!new RegExp(`auth_request_set\\s+\\$${variable}\\s+\\$upstream_http_${upstream};`, "m").test(block.replace(/\s+/g, " "))) {
throw new Error(`Nginx proxy does not capture authenticated ${label} identity`);
}
if (!new RegExp(`proxy_set_header\\s+${escaped(trustedHeader)}\\s+\\$${variable};`).test(block)) {
throw new Error(`Nginx proxy does not map authenticated ${label} identity`);
}
}
NODE NODE
echo "Nginx reverse-proxy guide contract passed" echo "Nginx reverse-proxy guide contract passed"
} }
@@ -637,10 +697,7 @@ const source = fs.readFileSync(process.argv[2], "utf8");
const block = [...source.matchAll(/```caddyfile\n([\s\S]*?)```/g)].map((match) => match[1]).join("\n"); const block = [...source.matchAll(/```caddyfile\n([\s\S]*?)```/g)].map((match) => match[1]).join("\n");
const tokens = [ const tokens = [
"thoth.example.com {", "route {", "thoth.example.com {", "route {",
"request_header -X-Thoth-Principal-Subject",
"request_header -X-Thoth-Trusted-Principal-Subject",
"forward_auth auth-gateway:4180 {", "uri /verify", "copy_headers {", "forward_auth auth-gateway:4180 {", "uri /verify", "copy_headers {",
"X-Thoth-Principal-Subject>X-Thoth-Trusted-Principal-Subject",
"reverse_proxy 127.0.0.1:8080 {", "flush_interval -1", "reverse_proxy 127.0.0.1:8080 {", "flush_interval -1",
]; ];
if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(block) || !block.includes("127.0.0.1:8080")) { if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(block) || !block.includes("127.0.0.1:8080")) {
@@ -649,6 +706,22 @@ if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(block) || !block.includes("127.0.0.1:
for (const token of tokens) { for (const token of tokens) {
if (!block.includes(token)) throw new Error(`Caddy proxy lacks structural token: ${token}`); if (!block.includes(token)) throw new Error(`Caddy proxy lacks structural token: ${token}`);
} }
for (const [label, publicName, trustedName] of [
["issuer", "X-Thoth-Principal-Issuer", "X-Thoth-Trusted-Principal-Issuer"],
["subject", "X-Thoth-Principal-Subject", "X-Thoth-Trusted-Principal-Subject"],
["display", "X-Thoth-Principal-Display-Name", "X-Thoth-Trusted-Principal-Display-Name"],
["admin", "X-Thoth-Is-Admin", "X-Thoth-Trusted-Is-Admin"],
]) {
if (!block.includes(`request_header -${publicName}`)) {
throw new Error(`Caddy proxy does not clear inbound ${label} identity`);
}
if (!block.includes(`request_header -${trustedName}`)) {
throw new Error(`Caddy proxy does not clear inbound trusted ${label} identity`);
}
if (!block.includes(`${publicName}>${trustedName}`)) {
throw new Error(`Caddy proxy does not map authenticated ${label} identity`);
}
}
NODE NODE
echo "Caddy reverse-proxy guide contract passed" echo "Caddy reverse-proxy guide contract passed"
} }
@@ -821,7 +894,7 @@ verify_server_installation_example() {
return 1 return 1
} }
local fixture source_copy operator_dir copied_example connector_override env_file local fixture source_copy operator_dir copied_example connector_override env_file backup_root
fixture="$(mktemp -d "${TMPDIR%/}/thoth server install.XXXXXX")" fixture="$(mktemp -d "${TMPDIR%/}/thoth server install.XXXXXX")"
trap 'rm -rf "$fixture"' RETURN trap 'rm -rf "$fixture"' RETURN
[[ "$fixture" == *" "* ]] || { [[ "$fixture" == *" "* ]] || {
@@ -830,8 +903,9 @@ verify_server_installation_example() {
} }
source_copy="$fixture/ThothII server source" source_copy="$fixture/ThothII server source"
operator_dir="$fixture/server operator files" operator_dir="$fixture/server operator files"
backup_root="$fixture/server backups"
mkdir -p "$source_copy/deploy/pi" "$source_copy/deploy/workspaces" \ mkdir -p "$source_copy/deploy/pi" "$source_copy/deploy/workspaces" \
"$operator_dir/data" "$operator_dir/pi-state" "$operator_dir/workspace-registry" "$operator_dir/data" "$operator_dir/pi-state" "$operator_dir/workspace-registry" "$backup_root"
cp "$root/compose.yaml" "$source_copy/compose.yaml" cp "$root/compose.yaml" "$source_copy/compose.yaml"
cp "$root/deploy/compose.server.yaml" "$source_copy/deploy/compose.server.yaml" cp "$root/deploy/compose.server.yaml" "$source_copy/deploy/compose.server.yaml"
cp "$root/deploy/compose.session-server.yaml.example" \ cp "$root/deploy/compose.session-server.yaml.example" \
@@ -869,6 +943,7 @@ verify_server_installation_example() {
"THT_DATA_ROOT=$operator_dir/data" \ "THT_DATA_ROOT=$operator_dir/data" \
"THT_PI_STATE_ROOT=$operator_dir/pi-state" \ "THT_PI_STATE_ROOT=$operator_dir/pi-state" \
"THT_WORKSPACE_REGISTRY_ROOT=$operator_dir/workspace-registry" \ "THT_WORKSPACE_REGISTRY_ROOT=$operator_dir/workspace-registry" \
"THT_BACKUP_ROOT=$backup_root" \
"THT_SERVER_WORKSPACE_CONFIG=$source_copy/deploy/workspaces/server-sessions.yaml.example" \ "THT_SERVER_WORKSPACE_CONFIG=$source_copy/deploy/workspaces/server-sessions.yaml.example" \
'THT_LLM_URL=https://llm.example.invalid' \ 'THT_LLM_URL=https://llm.example.invalid' \
'THT_SESSION_DB_HOST=sessions.example.invalid' \ 'THT_SESSION_DB_HOST=sessions.example.invalid' \
@@ -942,6 +1017,62 @@ for (const secret of [
} }
NODE NODE
echo "server installation example rendered from path with spaces passed" echo "server installation example rendered from path with spaces passed"
local migration_rendered="$fixture/server-migration.json"
"$root/scripts/compose-with-preflight.sh" --env-file "$descriptor_env" \
"${files[@]}" --profile session-migrate config --format json >"$migration_rendered"
node - "$migration_rendered" <<'NODE'
const fs = require("fs");
const config = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
const services = config.services || {};
if (!services.core || !services["session-migrate"]) throw new Error("server migration profile is missing core or session-migrate");
if (services.core.image !== services["session-migrate"].image) throw new Error("source migration image differs from core");
if (services["session-migrate"].build) throw new Error("source migration service unexpectedly declares a build");
NODE
local pinned_template="$fixture/pinned-template.yaml" pinned_override="$operator_dir/pinned-images.yaml"
awk '
/^## Build locally or select pinned images$/ { section=1; next }
section && /^```yaml$/ { code=1; next }
code && /^```$/ { exit }
code { print }
' "$root/docs/install/server.md" >"$pinned_template"
sed \
-e "s#registry.example.com/thothii/core@sha256:<64-lowercase-hex-digits>#registry.example.com/thothii/core@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa#g" \
-e "s#registry.example.com/thothii/frontend@sha256:<64-lowercase-hex-digits>#registry.example.com/thothii/frontend@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb#g" \
"$pinned_template" >"$pinned_override"
chmod 0600 "$pinned_override"
local pinned_rendered="$fixture/server-pinned-migration.json"
"$root/scripts/compose-with-preflight.sh" --env-file "$descriptor_env" \
"${files[@]}" -f "$pinned_override" --profile session-migrate config --format json >"$pinned_rendered"
node - "$pinned_rendered" <<'NODE'
const fs = require("fs");
const config = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
const core = config.services?.core;
const frontend = config.services?.frontend;
const migrator = config.services?.["session-migrate"];
if (!core || !frontend || !migrator) throw new Error("pinned migration profile lacks core, frontend, or session-migrate");
if (core.image !== migrator.image || !/@sha256:[a-f0-9]{64}$/.test(core.image)) {
throw new Error("pinned migration image does not equal the exact core digest");
}
if (!/@sha256:[a-f0-9]{64}$/.test(frontend.image)) throw new Error("frontend is not pinned by exact digest");
for (const [name, service] of Object.entries({core, frontend, migrator})) {
if (service.build) throw new Error(name + " retained a local build in pinned mode");
if (/:local$/.test(service.image || "")) throw new Error(name + " retained a local image in pinned mode");
}
NODE
echo "server pinned migration image fixture passed"
local checksum_root="$fixture/root-only-checksum"
mkdir -m 0700 "$checksum_root"
printf 'fixture backup bytes\n' >"$checksum_root/runtime-data.tgz"
/bin/sh -ceu 'cd "$1"; sha256sum runtime-data.tgz > SHA256SUMS; sha256sum --check SHA256SUMS' sh "$checksum_root" >/dev/null
printf 'corruption\n' >>"$checksum_root/runtime-data.tgz"
if (cd "$checksum_root" && sha256sum --check SHA256SUMS) >/dev/null 2>&1; then
echo "corrupted server backup checksum fixture was accepted" >&2
return 1
fi
echo "server backup checksum root-only fixture passed"
} }
write_private() { write_private() {
tools/thothctl/cmd/thothctl/main.go
+60
View File
@@ -18,6 +18,7 @@ import (
"github.com/aritmolab/thothii/tools/thothctl/internal/config" "github.com/aritmolab/thothii/tools/thothctl/internal/config"
"github.com/aritmolab/thothii/tools/thothctl/internal/output" "github.com/aritmolab/thothii/tools/thothctl/internal/output"
"github.com/aritmolab/thothii/tools/thothctl/internal/pi" "github.com/aritmolab/thothii/tools/thothctl/internal/pi"
"github.com/aritmolab/thothii/tools/thothctl/internal/serverops"
) )
const usage = `Usage: thothctl --installation <absolute-path>/thothii-installation.yaml <command> const usage = `Usage: thothctl --installation <absolute-path>/thothii-installation.yaml <command>
@@ -29,6 +30,10 @@ Commands:
start Start the installation in the background. start Start the installation in the background.
stop Stop the installation. stop Stop the installation.
update --check-only Validate the current installation without changing containers. update --check-only Validate the current installation without changing containers.
sessions migrate --yes
Run only the server session migrator and verify pending=[] and drifted=[].
remove Display exact stopped app container IDs without mutation.
remove --yes ID... Remove only the stopped IDs copied from the preceding display.
pi status Show the Pi version embedded in core. pi status Show the Pi version embedded in core.
pi doctor Check Pi preconditions without changing the installation. pi doctor Check Pi preconditions without changing the installation.
pi test Run the temporary Pi/core smoke checks. pi test Run the temporary Pi/core smoke checks.
@@ -113,12 +118,67 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
return doctor(ctx, installation, runner, secretValues, stdout, stderr) return doctor(ctx, installation, runner, secretValues, stdout, stderr)
case "pi": case "pi":
return piCommand(ctx, installation, runner, commandArgs, secretValues, stdout, stderr) return piCommand(ctx, installation, runner, commandArgs, secretValues, stdout, stderr)
case "sessions":
if len(commandArgs) != 2 || commandArgs[0] != "migrate" || commandArgs[1] != "--yes" {
return commandUsageError(stderr, "sessions migrate requires --yes")
}
status, operationErr := serverops.MigrateSessions(ctx, installation, runner, true)
if operationErr != nil {
return serverOperationFailure(stderr, operationErr, secretValues)
}
if encodeErr := json.NewEncoder(stdout).Encode(status); encodeErr != nil {
fmt.Fprintln(stderr, "thothctl: migration status could not be written")
return 1
}
return 0
case "remove":
var confirmedIDs []string
if len(commandArgs) > 0 {
if commandArgs[0] != "--yes" || len(commandArgs) < 2 {
return commandUsageError(stderr, "remove requires either no arguments or --yes followed by every displayed container ID")
}
confirmedIDs = commandArgs[1:]
}
removal, operationErr := serverops.Remove(ctx, installation, runner, confirmedIDs)
writeRemovalTargets(stdout, installation.ProjectName(), removal.Targets)
if errors.Is(operationErr, serverops.ErrConfirmationRequired) {
fmt.Fprint(stderr, "thothctl: inspect the exact targets above, then re-run with remove --yes")
for _, target := range removal.Targets {
fmt.Fprintf(stderr, " %s", target.ID)
}
fmt.Fprintln(stderr)
return 2
}
if operationErr != nil {
return serverOperationFailure(stderr, operationErr, secretValues)
}
fmt.Fprintf(stdout, "Removed %d stopped app containers; verified %d preserved paths.\n", len(removal.Targets), removal.Preserved)
return 0
default: default:
return commandUsageError(stderr, fmt.Sprintf("unknown command %q", command)) return commandUsageError(stderr, fmt.Sprintf("unknown command %q", command))
} }
return writeResult(result, err, secretValues, stdout, stderr) return writeResult(result, err, secretValues, stdout, stderr)
} }
func writeRemovalTargets(outputWriter io.Writer, project string, targets []serverops.Container) {
fmt.Fprintf(outputWriter, "Removal targets for installation project %s:\n", project)
if len(targets) == 0 {
fmt.Fprintln(outputWriter, " (none)")
return
}
for _, target := range targets {
fmt.Fprintf(outputWriter, " service=%s name=%s id=%s state=%s\n", target.Service, target.Name, target.ID, target.State)
}
}
func serverOperationFailure(stderr io.Writer, err error, secretValues []string) int {
fmt.Fprintf(stderr, "thothctl: %s\n", output.Sanitize(err.Error(), secretValues))
if errors.Is(err, serverops.ErrConfirmationRequired) || errors.Is(err, serverops.ErrUnsafeState) {
return 2
}
return 1
}
// installationRunner transforms only Compose invocations into the installation's validated, // installationRunner transforms only Compose invocations into the installation's validated,
// profile-specific argument list. Direct Docker image commands remain host-side and use arguments. // profile-specific argument list. Direct Docker image commands remain host-side and use arguments.
type installationRunner struct { type installationRunner struct {
tools/thothctl/cmd/thothctl/main_test.go
+58
View File
@@ -79,6 +79,8 @@ func TestUsageDocumentsClosedConfigureUpdateSourcesAndMaintenanceRecovery(t *tes
"--source pull --image IMAGE@sha256:DIGEST", "--source pull --image IMAGE@sha256:DIGEST",
"pi maintenance status", "pi maintenance status",
"pi maintenance recover --yes", "pi maintenance recover --yes",
"sessions migrate --yes",
"remove --yes ID...",
} { } {
if !strings.Contains(usage, required) { if !strings.Contains(usage, required) {
t.Errorf("usage missing %q", required) t.Errorf("usage missing %q", required)
@@ -86,6 +88,48 @@ func TestUsageDocumentsClosedConfigureUpdateSourcesAndMaintenanceRecovery(t *tes
} }
} }
func TestRunSessionsMigrateRequiresExplicitConfirmationBeforeDocker(t *testing.T) {
fixture := newCLIFixture(t, "")
fixture.setProfile(t, "server")
fixture.setEnvironment(t)
var stdout, stderr bytes.Buffer
code := run(context.Background(), []string{
"--installation", fixture.installationPath, "sessions", "migrate",
}, &stdout, &stderr)
if code != 2 || !strings.Contains(stderr.String(), "sessions migrate requires --yes") {
t.Fatalf("exit = %d, stderr = %q", code, stderr.String())
}
assertDockerNotInvoked(t, fixture)
}
func TestRunRemoveDisplaysExactInstallationTargetsBeforeConfirmation(t *testing.T) {
fixture := newCLIFixture(t, "")
fixture.setProfile(t, "server")
fixture.setEnvironment(t)
t.Setenv("THOTHCTL_FAKE_STOPPED_PS", `[{"ID":"core-id","Name":"exact-core","Service":"core","State":"exited"},{"ID":"front-id","Name":"exact-frontend","Service":"frontend","State":"exited"}]`)
var stdout, stderr bytes.Buffer
code := run(context.Background(), []string{
"--installation", fixture.installationPath, "remove",
}, &stdout, &stderr)
if code != 2 || !strings.Contains(stderr.String(), "re-run with remove --yes core-id front-id") {
t.Fatalf("exit = %d, stderr = %q", code, stderr.String())
}
for _, value := range []string{"exact-core", "core-id", "exact-frontend", "front-id", "exited"} {
if !strings.Contains(stdout.String(), value) {
t.Errorf("target display %q missing %q", stdout.String(), value)
}
}
calls := fixture.invocations(t)
if len(calls) != 1 {
t.Fatalf("Docker calls = %#v", calls)
}
assertInvocationContains(t, calls, "ps", "--all", "--format", "json", "core", "frontend")
}
type wizardRunner struct{ calls []string } type wizardRunner struct{ calls []string }
func (r *wizardRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) { func (r *wizardRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
@@ -615,6 +659,7 @@ func newCLIFixture(t *testing.T, envTemplate string) cliFixture {
printf '%s\n' "$@" >> "$THOTHCTL_FAKE_ARGS" printf '%s\n' "$@" >> "$THOTHCTL_FAKE_ARGS"
printf '%s\n' -- >> "$THOTHCTL_FAKE_ARGS" printf '%s\n' -- >> "$THOTHCTL_FAKE_ARGS"
case " $* " in case " $* " in
*" ps --all --format json core frontend "*) printf '%s\n' "${THOTHCTL_FAKE_STOPPED_PS:-[]}" ;;
*" config --format json "*) printf '%s\n' '{"volumes":{"settings":{}},"services":{"core":{"image":"thothii-core:local","environment":{"THT_LLM_URL":"https://llm.example.invalid"}}}}' ;; *" config --format json "*) printf '%s\n' '{"volumes":{"settings":{}},"services":{"core":{"image":"thothii-core:local","environment":{"THT_LLM_URL":"https://llm.example.invalid"}}}}' ;;
*" ps --format json "*) printf '%s\n' '[{"Service":"core","State":"running","Health":"healthy"},{"Service":"frontend","State":"running","Health":"healthy"}]' ;; *" ps --format json "*) printf '%s\n' '[{"Service":"core","State":"running","Health":"healthy"},{"Service":"frontend","State":"running","Health":"healthy"}]' ;;
*"io.thothii.pi.version"*) printf '%s\n' '0.80.3' ;; *"io.thothii.pi.version"*) printf '%s\n' '0.80.3' ;;
@@ -661,6 +706,19 @@ func (f cliFixture) setEnvContents(t *testing.T, env string) {
t.Setenv("THOTHCTL_FAKE_LOG", "") t.Setenv("THOTHCTL_FAKE_LOG", "")
t.Setenv("THOTHCTL_FAKE_FAILURE", "") t.Setenv("THOTHCTL_FAKE_FAILURE", "")
t.Setenv("THOTHCTL_FAKE_FAIL_ON", "") t.Setenv("THOTHCTL_FAKE_FAIL_ON", "")
t.Setenv("THOTHCTL_FAKE_STOPPED_PS", "[]")
}
func (f cliFixture) setProfile(t *testing.T, profile string) {
t.Helper()
composePath := filepath.Join(f.projectDirectory, "deploy", "compose."+profile+".yaml")
if err := os.WriteFile(composePath, []byte("services: {}\n"), 0o600); err != nil {
t.Fatal(err)
}
contents := "profile: " + profile + "\nprojectDirectory: " + f.projectDirectory + "\nenvFile: " + f.envFile + "\n"
if err := os.WriteFile(f.installationPath, []byte(contents), 0o600); err != nil {
t.Fatal(err)
}
} }
func (f cliFixture) invocations(t *testing.T) [][]string { func (f cliFixture) invocations(t *testing.T) [][]string {
tools/thothctl/internal/config/installation.go
+81 -4
View File
@@ -148,8 +148,25 @@ func (i Installation) ProjectName() string {
// ComposeArgs builds Docker Compose arguments without shell quoting or interpolation. // ComposeArgs builds Docker Compose arguments without shell quoting or interpolation.
func (i Installation) ComposeArgs(command ...string) []string { func (i Installation) ComposeArgs(command ...string) []string {
return i.composeArgs(i.ComposeFiles(), command...)
}
// ComposeArgsWithFinalOverride appends one validated, generated override after every durable
// installation selector and before the Compose command.
func (i Installation) ComposeArgsWithFinalOverride(override string, command ...string) ([]string, error) {
if filepath.Clean(override) != override || !filepath.IsAbs(override) {
return nil, errors.New("final Compose override must be an absolute canonical path")
}
if err := requireRegularFile(override, "final Compose override"); err != nil {
return nil, err
}
files := append(i.ComposeFiles(), override)
return i.composeArgs(files, command...), nil
}
func (i Installation) composeArgs(files []string, command ...string) []string {
args := []string{"compose", "--project-name", i.ProjectName(), "--project-directory", i.ProjectDirectory, "--env-file", i.EnvFile} args := []string{"compose", "--project-name", i.ProjectName(), "--project-directory", i.ProjectDirectory, "--env-file", i.EnvFile}
for _, composeFile := range i.ComposeFiles() { for _, composeFile := range files {
args = append(args, "-f", composeFile) args = append(args, "-f", composeFile)
} }
return append(args, command...) return append(args, command...)
@@ -193,15 +210,75 @@ func (i Installation) SecretFiles() ([]string, error) {
// EnvironmentValue returns one declared installation value without exposing dotenv parsing to // EnvironmentValue returns one declared installation value without exposing dotenv parsing to
// callers. It is used only for operator-visible file locations, never for secret content. // callers. It is used only for operator-visible file locations, never for secret content.
func (i Installation) EnvironmentValue(name string) (string, error) { func (i Installation) EnvironmentValue(name string) (string, error) {
values, err := i.environmentValues()
if err != nil {
return "", err
}
return values[name], nil
}
func (i Installation) environmentValues() (map[string]string, error) {
contents, err := safeio.ReadCanonicalRegular(i.EnvFile, maxEnvironmentFileBytes) contents, err := safeio.ReadCanonicalRegular(i.EnvFile, maxEnvironmentFileBytes)
if err != nil { if err != nil {
return "", errors.New("installation environment could not be read") return nil, errors.New("installation environment could not be read")
} }
values, err := parseComposeDotenv(contents) values, err := parseComposeDotenv(contents)
if err != nil { if err != nil {
return "", errors.New("installation environment could not be read") return nil, errors.New("installation environment could not be read")
} }
return values[name], nil return values, nil
}
// PreservationPaths returns the server bind roots, backup root, and declared secret files whose
// filesystem identities must survive a data-preserving removal.
func (i Installation) PreservationPaths() ([]string, error) {
if i.Profile != "server" {
return nil, errors.New("data-preserving removal requires a server installation")
}
values, err := i.environmentValues()
if err != nil {
return nil, err
}
paths := make([]string, 0)
seen := make(map[string]struct{})
for _, name := range []string{
"THT_DATA_ROOT", "THT_PI_STATE_ROOT", "THT_WORKSPACE_REGISTRY_ROOT", "THT_BACKUP_ROOT",
} {
path := values[name]
if err := requireCanonicalDirectory(path); err != nil {
return nil, fmt.Errorf("%s must identify an existing canonical directory", name)
}
if _, exists := seen[path]; !exists {
paths = append(paths, path)
seen[path] = struct{}{}
}
}
secretFiles, err := i.SecretFiles()
if err != nil {
return nil, err
}
for _, path := range secretFiles {
if _, exists := seen[path]; !exists {
paths = append(paths, path)
seen[path] = struct{}{}
}
}
return paths, nil
}
func requireCanonicalDirectory(path string) error {
if err := safeio.ValidateCanonicalPath(path); err != nil {
return err
}
resolved, err := filepath.EvalSymlinks(path)
if err != nil || resolved != path {
return errors.New("directory path is unavailable or contains a symlink")
}
info, err := os.Stat(path)
if err != nil || !info.IsDir() {
return errors.New("directory path is unavailable")
}
return nil
} }
func parseComposeDotenv(contents []byte) (map[string]string, error) { func parseComposeDotenv(contents []byte) (map[string]string, error) {
tools/thothctl/internal/config/installation_test.go
+70
View File
@@ -78,6 +78,76 @@ func TestComposeArgsAutomaticallyIncludeTheInstallationCurrentImageOverride(t *t
} }
} }
func TestComposeArgsWithFinalOverridePreservesCurrentImagePrecedence(t *testing.T) {
installationPath, _, _, _ := writeInstallation(t, "server")
seed, err := Load(installationPath)
if err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(seed.ControlDirectory(), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(seed.CurrentImageOverridePath(), []byte("services: {}\n"), 0o600); err != nil {
t.Fatal(err)
}
final := filepath.Join(seed.ControlDirectory(), "migration.yaml")
if err := os.WriteFile(final, []byte("services: {}\n"), 0o600); err != nil {
t.Fatal(err)
}
installation, err := Load(installationPath)
if err != nil {
t.Fatal(err)
}
args, err := installation.ComposeArgsWithFinalOverride(final, "--profile", "session-migrate", "config")
if err != nil {
t.Fatal(err)
}
want := []string{"-f", installation.CurrentImageOverridePath(), "-f", final, "--profile", "session-migrate", "config"}
if !containsSequence(args, want) {
t.Fatalf("ComposeArgsWithFinalOverride() = %#v, want %#v", args, want)
}
}
func TestPreservationPathsReturnsCanonicalBindRootsBackupsAndSecretFiles(t *testing.T) {
installationPath, _, envFile, _ := writeInstallation(t, "server")
root := filepath.Dir(envFile)
var wanted []string
var lines []string
for _, item := range []struct{ key, name string }{
{"THT_DATA_ROOT", "data"},
{"THT_PI_STATE_ROOT", "pi-state"},
{"THT_WORKSPACE_REGISTRY_ROOT", "workspace-registry"},
{"THT_BACKUP_ROOT", "backups"},
} {
path := filepath.Join(root, item.name)
if err := os.Mkdir(path, 0o700); err != nil {
t.Fatal(err)
}
wanted = append(wanted, path)
lines = append(lines, item.key+"="+path)
}
secret := filepath.Join(root, "secret")
if err := os.WriteFile(secret, []byte("secret"), 0o600); err != nil {
t.Fatal(err)
}
wanted = append(wanted, secret)
lines = append(lines, "APP_TOKEN_FILE="+secret)
if err := os.WriteFile(envFile, []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil {
t.Fatal(err)
}
installation, err := Load(installationPath)
if err != nil {
t.Fatal(err)
}
got, err := installation.PreservationPaths()
if err != nil {
t.Fatal(err)
}
assertStringsEqual(t, got, wanted)
}
func TestInstallationControlPathsAreIsolatedForDescriptorsSharingOneCheckout(t *testing.T) { func TestInstallationControlPathsAreIsolatedForDescriptorsSharingOneCheckout(t *testing.T) {
projectDirectory := t.TempDir() projectDirectory := t.TempDir()
first := Installation{Path: filepath.Join(t.TempDir(), installationFileName), ProjectDirectory: projectDirectory} first := Installation{Path: filepath.Join(t.TempDir(), installationFileName), ProjectDirectory: projectDirectory}
tools/thothctl/internal/serverops/operations.go
+333
View File
@@ -0,0 +1,333 @@
// Package serverops implements bounded, installation-aware server maintenance operations.
package serverops
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/aritmolab/thothii/tools/thothctl/internal/compose"
"github.com/aritmolab/thothii/tools/thothctl/internal/config"
)
var (
ErrConfirmationRequired = errors.New("explicit confirmation is required")
ErrUnsafeState = errors.New("server operation refused in the current state")
)
type Runner interface {
Run(context.Context, []string, io.Reader) (compose.Result, error)
}
type MigrationStatus struct {
Applied []string `json:"applied"`
Drifted []string `json:"drifted"`
Pending []string `json:"pending"`
}
type Container struct {
ID string `json:"ID"`
Name string `json:"Name"`
Service string `json:"Service"`
State string `json:"State"`
}
type RemovalResult struct {
Targets []Container
Preserved int
}
// MigrateSessions runs only the one-shot migration service and proves the resulting schema state.
func MigrateSessions(ctx context.Context, installation config.Installation, runner Runner, confirmed bool) (MigrationStatus, error) {
if !confirmed {
return MigrationStatus{}, ErrConfirmationRequired
}
if installation.Profile != "server" {
return MigrationStatus{}, fmt.Errorf("%w: session migration requires a server installation", ErrUnsafeState)
}
containers, err := inspectContainers(ctx, installation, runner)
if err != nil {
return MigrationStatus{}, err
}
if err := requireStopped(containers); err != nil {
return MigrationStatus{}, err
}
rendered, err := runCompose(ctx, runner, installation.ComposeArgs("--profile", "session-migrate", "config", "--format", "json"))
if err != nil {
return MigrationStatus{}, err
}
coreImage, err := selectedCoreImage(rendered.Stdout)
if err != nil {
return MigrationStatus{}, err
}
override, cleanup, err := migrationOverride(installation, coreImage)
if err != nil {
return MigrationStatus{}, err
}
defer cleanup()
configArgs, err := installation.ComposeArgsWithFinalOverride(override, "--profile", "session-migrate", "config", "--format", "json")
if err != nil {
return MigrationStatus{}, err
}
finalConfig, err := runCompose(ctx, runner, configArgs)
if err != nil {
return MigrationStatus{}, err
}
if err := requireMigrationImage(finalConfig.Stdout, coreImage); err != nil {
return MigrationStatus{}, err
}
runArgs, err := installation.ComposeArgsWithFinalOverride(
override, "--profile", "session-migrate", "run", "--rm", "--no-deps", "--no-TTY", "session-migrate",
)
if err != nil {
return MigrationStatus{}, err
}
result, err := runCompose(ctx, runner, runArgs)
if err != nil {
return MigrationStatus{}, err
}
status, err := parseMigrationStatus(result.Stdout)
if err != nil {
return MigrationStatus{}, err
}
if len(status.Pending) != 0 || len(status.Drifted) != 0 {
return status, fmt.Errorf("%w: session migration did not finish cleanly", ErrUnsafeState)
}
return status, nil
}
// Remove deletes only the exact stopped core/frontend container IDs displayed by the command.
// A nil confirmation performs inspection only; a non-nil confirmation must equal every target ID.
func Remove(ctx context.Context, installation config.Installation, runner Runner, confirmedIDs []string) (RemovalResult, error) {
if installation.Profile != "server" {
return RemovalResult{}, fmt.Errorf("%w: removal requires a server installation", ErrUnsafeState)
}
targets, err := inspectContainers(ctx, installation, runner)
result := RemovalResult{Targets: targets}
if err != nil {
return result, err
}
if err := requireStopped(targets); err != nil {
return result, err
}
if confirmedIDs == nil {
return result, ErrConfirmationRequired
}
if !sameTargetIDs(targets, confirmedIDs) {
return result, fmt.Errorf("%w: confirmed container IDs differ from current targets", ErrUnsafeState)
}
paths, err := installation.PreservationPaths()
if err != nil {
return result, fmt.Errorf("%w: preservation paths could not be verified", ErrUnsafeState)
}
snapshots, err := snapshotPaths(paths)
if err != nil {
return result, err
}
if len(targets) > 0 {
args := []string{"rm"}
for _, target := range targets {
args = append(args, target.ID)
}
if _, err := runDocker(ctx, runner, args); err != nil {
return result, err
}
}
remaining, err := inspectContainers(ctx, installation, runner)
if err != nil {
return result, err
}
if len(remaining) != 0 {
return result, fmt.Errorf("%w: installation containers changed during removal", ErrUnsafeState)
}
if err := verifySnapshots(snapshots); err != nil {
return result, err
}
result.Preserved = len(snapshots)
return result, nil
}
func sameTargetIDs(targets []Container, confirmed []string) bool {
if len(targets) != len(confirmed) {
return false
}
wanted := make(map[string]struct{}, len(confirmed))
for _, id := range confirmed {
if strings.TrimSpace(id) == "" {
return false
}
if _, duplicate := wanted[id]; duplicate {
return false
}
wanted[id] = struct{}{}
}
for _, target := range targets {
if _, exists := wanted[target.ID]; !exists {
return false
}
}
return true
}
func inspectContainers(ctx context.Context, installation config.Installation, runner Runner) ([]Container, error) {
result, err := runCompose(ctx, runner, installation.ComposeArgs("ps", "--all", "--format", "json", "core", "frontend"))
if err != nil {
return nil, err
}
var containers []Container
if err := json.Unmarshal([]byte(result.Stdout), &containers); err != nil {
return nil, fmt.Errorf("%w: Compose returned invalid container status", ErrUnsafeState)
}
seen := make(map[string]struct{})
for _, container := range containers {
if (container.Service != "core" && container.Service != "frontend") || container.ID == "" || container.Name == "" {
return nil, fmt.Errorf("%w: Compose returned an unexpected removal target", ErrUnsafeState)
}
if _, exists := seen[container.ID]; exists {
return nil, fmt.Errorf("%w: Compose returned duplicate container IDs", ErrUnsafeState)
}
seen[container.ID] = struct{}{}
}
return containers, nil
}
func requireStopped(containers []Container) error {
for _, container := range containers {
if strings.ToLower(container.State) != "exited" {
return fmt.Errorf("%w: %s is not stopped", ErrUnsafeState, container.Service)
}
}
return nil
}
func selectedCoreImage(document string) (string, error) {
services, err := renderedServices(document)
if err != nil {
return "", err
}
core, exists := services["core"]
if !exists || strings.TrimSpace(core.Image) == "" {
return "", fmt.Errorf("%w: rendered core image is missing", ErrUnsafeState)
}
if _, exists := services["session-migrate"]; !exists {
return "", fmt.Errorf("%w: rendered migration service is missing", ErrUnsafeState)
}
return core.Image, nil
}
type renderedService struct {
Image string `json:"image"`
Build json.RawMessage `json:"build"`
}
func renderedServices(document string) (map[string]renderedService, error) {
var configDocument struct {
Services map[string]renderedService `json:"services"`
}
if err := json.Unmarshal([]byte(document), &configDocument); err != nil {
return nil, fmt.Errorf("%w: Compose returned invalid rendered configuration", ErrUnsafeState)
}
return configDocument.Services, nil
}
func requireMigrationImage(document, coreImage string) error {
services, err := renderedServices(document)
if err != nil {
return err
}
migrator, exists := services["session-migrate"]
if !exists || migrator.Image != coreImage {
return fmt.Errorf("%w: migration image differs from selected core image", ErrUnsafeState)
}
if len(migrator.Build) != 0 && strings.TrimSpace(string(migrator.Build)) != "null" {
return fmt.Errorf("%w: migration service unexpectedly declares a build", ErrUnsafeState)
}
return nil
}
func migrationOverride(installation config.Installation, image string) (string, func(), error) {
control := installation.ControlDirectory()
if err := os.MkdirAll(control, 0o700); err != nil {
return "", func() {}, errors.New("migration control directory could not be created")
}
info, err := os.Lstat(control)
if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
return "", func() {}, errors.New("migration control directory is unsafe")
}
directory, err := os.MkdirTemp(control, "session-migrate-")
if err != nil {
return "", func() {}, errors.New("migration override directory could not be created")
}
cleanup := func() {
_ = os.Remove(filepath.Join(directory, "override.yaml"))
_ = os.Remove(directory)
}
path := filepath.Join(directory, "override.yaml")
contents := "services:\n session-migrate:\n build: !reset null\n image: " + strconv.Quote(image) + "\n"
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
cleanup()
return "", func() {}, errors.New("migration override could not be written")
}
return path, cleanup, nil
}
func parseMigrationStatus(document string) (MigrationStatus, error) {
var status MigrationStatus
decoder := json.NewDecoder(strings.NewReader(document))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&status); err != nil || status.Applied == nil || status.Drifted == nil || status.Pending == nil {
return MigrationStatus{}, fmt.Errorf("%w: migration did not return verified JSON status", ErrUnsafeState)
}
var extra any
if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) {
return MigrationStatus{}, fmt.Errorf("%w: migration returned trailing output", ErrUnsafeState)
}
return status, nil
}
type pathSnapshot struct {
path string
info os.FileInfo
}
func snapshotPaths(paths []string) ([]pathSnapshot, error) {
snapshots := make([]pathSnapshot, 0, len(paths))
for _, path := range paths {
info, err := os.Stat(path)
if err != nil {
return nil, fmt.Errorf("%w: preservation target is unavailable", ErrUnsafeState)
}
snapshots = append(snapshots, pathSnapshot{path: path, info: info})
}
return snapshots, nil
}
func verifySnapshots(snapshots []pathSnapshot) error {
for _, snapshot := range snapshots {
info, err := os.Stat(snapshot.path)
if err != nil || !os.SameFile(snapshot.info, info) {
return fmt.Errorf("%w: a preserved path changed during removal", ErrUnsafeState)
}
}
return nil
}
func runCompose(ctx context.Context, runner Runner, args []string) (compose.Result, error) {
return runDocker(ctx, runner, args)
}
func runDocker(ctx context.Context, runner Runner, args []string) (compose.Result, error) {
result, err := runner.Run(ctx, args, nil)
if err != nil {
return result, errors.New("Docker operation failed")
}
return result, nil
}
tools/thothctl/internal/serverops/operations_test.go
+314
View File
@@ -0,0 +1,314 @@
package serverops
import (
"context"
"errors"
"io"
"os"
"path/filepath"
"reflect"
"strconv"
"strings"
"testing"
"github.com/aritmolab/thothii/tools/thothctl/internal/compose"
"github.com/aritmolab/thothii/tools/thothctl/internal/config"
)
type fakeRunner struct {
run func(args []string) (compose.Result, error)
all [][]string
}
func (r *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
r.all = append(r.all, append([]string(nil), args...))
return r.run(args)
}
func TestMigrateSessionsUsesOnlyTheMigrationProfileAndSelectedCoreImage(t *testing.T) {
for _, image := range []string{
"thothii-core:local",
"registry.example.invalid/thothii/core@sha256:" + strings.Repeat("a", 64),
} {
t.Run(image, func(t *testing.T) {
installation := testInstallation(t)
if err := os.MkdirAll(installation.ControlDirectory(), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(installation.CurrentImageOverridePath(), []byte("services:\n core:\n image: "+image+"\n"), 0o600); err != nil {
t.Fatal(err)
}
var temporaryOverride string
configCalls := 0
runner := &fakeRunner{run: func(args []string) (compose.Result, error) {
switch {
case contains(args, "ps", "--all", "--format", "json", "core", "frontend"):
return compose.Result{Stdout: `[{"ID":"core-id","Name":"core-name","Service":"core","State":"exited"},{"ID":"front-id","Name":"front-name","Service":"frontend","State":"exited"}]`}, nil
case contains(args, "--profile", "session-migrate", "config", "--format", "json"):
configCalls++
if configCalls == 1 {
return compose.Result{Stdout: `{"services":{"core":{"image":"` + image + `"},"session-migrate":{"image":"thothii-core:local"}}}`}, nil
}
temporaryOverride = lastComposeFile(args)
contents, err := os.ReadFile(temporaryOverride)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(contents), "image: "+strconv.Quote(image)) || !strings.Contains(string(contents), "build: !reset null") {
t.Fatalf("migration override = %q", contents)
}
if indexOf(args, installation.CurrentImageOverridePath()) >= indexOf(args, temporaryOverride) {
t.Fatalf("temporary override does not follow durable selector: %#v", args)
}
return compose.Result{Stdout: `{"services":{"core":{"image":"` + image + `"},"session-migrate":{"image":"` + image + `"}}}`}, nil
case contains(args, "--profile", "session-migrate", "run", "--rm", "--no-deps", "--no-TTY", "session-migrate"):
return compose.Result{Stdout: `{"applied":["0001"],"drifted":[],"pending":[]}` + "\n"}, nil
default:
t.Fatalf("unexpected Docker invocation: %#v", args)
return compose.Result{}, nil
}
}}
status, err := MigrateSessions(context.Background(), installation, runner, true)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(status.Pending, []string{}) || !reflect.DeepEqual(status.Drifted, []string{}) {
t.Fatalf("status = %#v", status)
}
if temporaryOverride == "" {
t.Fatal("migration override was not inspected")
}
if _, err := os.Stat(temporaryOverride); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("temporary override remains after migration: %v", err)
}
})
}
}
func TestMigrateSessionsFailsClosedBeforeMutation(t *testing.T) {
installation := testInstallation(t)
for name, spec := range map[string]struct {
confirmed bool
ps string
migrationJSON string
}{
"confirmation missing": {false, `[]`, `{"applied":[],"drifted":[],"pending":[]}`},
"service running": {true, `[{"ID":"core-id","Name":"core","Service":"core","State":"running"}]`, `{"applied":[],"drifted":[],"pending":[]}`},
"pending migration": {true, `[]`, `{"applied":[],"drifted":[],"pending":["0002"]}`},
"drifted migration": {true, `[]`, `{"applied":[],"drifted":["0001"],"pending":[]}`},
} {
t.Run(name, func(t *testing.T) {
runCalled := false
configCalls := 0
runner := &fakeRunner{run: func(args []string) (compose.Result, error) {
switch {
case contains(args, "ps", "--all"):
return compose.Result{Stdout: spec.ps}, nil
case contains(args, "config", "--format", "json"):
configCalls++
return compose.Result{Stdout: `{"services":{"core":{"image":"thothii-core:local"},"session-migrate":{"image":"thothii-core:local"}}}`}, nil
case contains(args, "run", "--rm", "--no-deps", "--no-TTY", "session-migrate"):
runCalled = true
return compose.Result{Stdout: spec.migrationJSON}, nil
default:
t.Fatalf("unexpected Docker invocation: %#v", args)
return compose.Result{}, nil
}
}}
_, err := MigrateSessions(context.Background(), installation, runner, spec.confirmed)
if err == nil {
t.Fatal("MigrateSessions() error = nil")
}
if !spec.confirmed && len(runner.all) != 0 {
t.Fatalf("Docker invoked without confirmation: %#v", runner.all)
}
if strings.Contains(name, "service running") && (runCalled || configCalls != 0) {
t.Fatalf("migration advanced while app was running: %#v", runner.all)
}
})
}
}
func TestRemovePreservesEveryDeclaredBindSecretAndBackup(t *testing.T) {
installation, preserved := removalInstallation(t)
psCalls := 0
runner := &fakeRunner{run: func(args []string) (compose.Result, error) {
switch {
case contains(args, "ps", "--all", "--format", "json", "core", "frontend"):
psCalls++
if psCalls == 1 {
return compose.Result{Stdout: `[{"ID":"core-id","Name":"project-core-1","Service":"core","State":"exited"},{"ID":"frontend-id","Name":"project-frontend-1","Service":"frontend","State":"exited"}]`}, nil
}
return compose.Result{Stdout: `[]`}, nil
case reflect.DeepEqual(args, []string{"rm", "core-id", "frontend-id"}):
return compose.Result{Stdout: "core-id\nfrontend-id\n"}, nil
default:
t.Fatalf("unexpected Docker invocation: %#v", args)
return compose.Result{}, nil
}
}}
result, err := Remove(context.Background(), installation, runner, []string{"core-id", "frontend-id"})
if err != nil {
t.Fatal(err)
}
if result.Preserved != len(preserved) {
t.Fatalf("preserved = %d, want %d", result.Preserved, len(preserved))
}
if got := result.Targets; len(got) != 2 || got[0].ID != "core-id" || got[1].ID != "frontend-id" {
t.Fatalf("targets = %#v", got)
}
for _, args := range runner.all {
joined := strings.Join(args, " ")
if strings.Contains(joined, " -v") || strings.Contains(joined, "volume") || strings.Contains(joined, "down") || strings.Contains(joined, "prune") {
t.Fatalf("destructive removal invocation: %q", joined)
}
}
for _, path := range preserved {
if _, err := os.Stat(path); err != nil {
t.Errorf("preserved path %q: %v", path, err)
}
}
}
func TestRemoveDisplaysTargetsButDoesNotMutateWithoutConfirmation(t *testing.T) {
installation, _ := removalInstallation(t)
runner := &fakeRunner{run: func(args []string) (compose.Result, error) {
if !contains(args, "ps", "--all") {
t.Fatalf("mutation without confirmation: %#v", args)
}
return compose.Result{Stdout: `[{"ID":"core-id","Name":"project-core-1","Service":"core","State":"exited"}]`}, nil
}}
result, err := Remove(context.Background(), installation, runner, nil)
if !errors.Is(err, ErrConfirmationRequired) {
t.Fatalf("Remove() error = %v, want confirmation", err)
}
if len(result.Targets) != 1 || result.Targets[0].ID != "core-id" {
t.Fatalf("targets = %#v", result.Targets)
}
if len(runner.all) != 1 {
t.Fatalf("Docker calls = %#v", runner.all)
}
}
func TestRemoveRejectsRunningOrReplacedContainers(t *testing.T) {
for name, spec := range map[string]struct{ first, second string }{
"running": {`[{"ID":"core-id","Name":"core","Service":"core","State":"running"}]`, `[]`},
"replaced": {`[{"ID":"core-id","Name":"core","Service":"core","State":"exited"}]`, `[{"ID":"new-id","Name":"core","Service":"core","State":"exited"}]`},
} {
t.Run(name, func(t *testing.T) {
installation, _ := removalInstallation(t)
psCalls := 0
runner := &fakeRunner{run: func(args []string) (compose.Result, error) {
if contains(args, "ps", "--all") {
psCalls++
if psCalls == 1 {
return compose.Result{Stdout: spec.first}, nil
}
return compose.Result{Stdout: spec.second}, nil
}
if reflect.DeepEqual(args, []string{"rm", "core-id"}) {
return compose.Result{}, nil
}
t.Fatalf("unexpected Docker invocation: %#v", args)
return compose.Result{}, nil
}}
_, err := Remove(context.Background(), installation, runner, []string{"core-id"})
if err == nil {
t.Fatal("Remove() error = nil")
}
if name == "running" && len(runner.all) != 1 {
t.Fatalf("running container was mutated: %#v", runner.all)
}
})
}
}
func TestRemoveRejectsConfirmationForDifferentContainerIDs(t *testing.T) {
installation, _ := removalInstallation(t)
runner := &fakeRunner{run: func(args []string) (compose.Result, error) {
if !contains(args, "ps", "--all") {
t.Fatalf("mismatched confirmation caused mutation: %#v", args)
}
return compose.Result{Stdout: `[{"ID":"replacement-id","Name":"core","Service":"core","State":"exited"}]`}, nil
}}
result, err := Remove(context.Background(), installation, runner, []string{"previously-displayed-id"})
if !errors.Is(err, ErrUnsafeState) || len(result.Targets) != 1 {
t.Fatalf("Remove() = %#v, %v", result, err)
}
if len(runner.all) != 1 {
t.Fatalf("Docker calls = %#v", runner.all)
}
}
func testInstallation(t *testing.T) config.Installation {
t.Helper()
root := t.TempDir()
project := filepath.Join(root, "project")
if err := os.Mkdir(project, 0o700); err != nil {
t.Fatal(err)
}
return config.Installation{
Path: filepath.Join(root, "thothii-installation.yaml"), Profile: "server",
ProjectDirectory: project, EnvFile: filepath.Join(root, "server.env"),
}
}
func removalInstallation(t *testing.T) (config.Installation, []string) {
t.Helper()
installation := testInstallation(t)
paths := make([]string, 0, 5)
values := map[string]string{}
for _, name := range []string{"data", "pi-state", "workspace-registry", "backups"} {
path := filepath.Join(filepath.Dir(installation.Path), name)
if err := os.Mkdir(path, 0o700); err != nil {
t.Fatal(err)
}
paths = append(paths, path)
values[name] = path
}
secret := filepath.Join(filepath.Dir(installation.Path), "secret")
if err := os.WriteFile(secret, []byte("never-log-this"), 0o600); err != nil {
t.Fatal(err)
}
paths = append(paths, secret)
env := "THT_DATA_ROOT=" + values["data"] + "\n" +
"THT_PI_STATE_ROOT=" + values["pi-state"] + "\n" +
"THT_WORKSPACE_REGISTRY_ROOT=" + values["workspace-registry"] + "\n" +
"THT_BACKUP_ROOT=" + values["backups"] + "\n" +
"APP_TOKEN_FILE=" + secret + "\n"
if err := os.WriteFile(installation.EnvFile, []byte(env), 0o600); err != nil {
t.Fatal(err)
}
return installation, paths
}
func contains(values []string, sequence ...string) bool {
for start := range values {
if start+len(sequence) <= len(values) && reflect.DeepEqual(values[start:start+len(sequence)], sequence) {
return true
}
}
return false
}
func indexOf(values []string, value string) int {
for index, candidate := range values {
if candidate == value {
return index
}
}
return -1
}
func lastComposeFile(args []string) string {
last := ""
for index := 0; index+1 < len(args); index++ {
if args[index] == "-f" {
last = args[index+1]
}
}
return last
}