fix: make server operations executable

2026-08-05 11:06:34 +02:00
parent a707fb442c
commit 96fe5bfa79
11 changed files with 1299 additions and 43 deletions
@@ -9,6 +9,7 @@ THT_SECRETS_FILE=/absolute/path/to/thothii.secrets
 THT_DATA_ROOT=/srv/thothii/data
 THT_PI_STATE_ROOT=/srv/thothii/pi-state
 THT_WORKSPACE_REGISTRY_ROOT=/srv/thothii/workspace-registry
+THT_BACKUP_ROOT=/srv/thothii-backups
 THT_SERVER_WORKSPACE_CONFIG=/absolute/path/to/server-sessions.yaml
 THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git
 THT_WORKSPACE_GIT_BRANCH=main
@@ -14,8 +14,8 @@ first startup. Keep storage separated:
 ```text
 /srv/thothii/data/                # settings, session data, Pi state as applicable
 /srv/thothii/workspace-registry/  # repo/, snapshots/, state/, locks/
-/srv/thothii/secrets/             # Git and connector secret files, mode 0700
-/srv/thothii/operator/            # untracked Compose/.env, mode 0700
+/srv/thothii/secrets/             # Git and connector secret files, setgid mode 2750
+/srv/thothii/operator/            # untracked Compose/.env, setgid mode 2750
 ```

 Permit outbound TCP only to approved Git/Gitea, DWH, vector, embedding, and bastion endpoints.
@@ -41,8 +41,10 @@ authoring environment for migration.
 ## Git credentials, CA, SSH key, and known-hosts mounts

 Use the secret manager or a protected host-only procedure to create independent regular files under
-`/srv/thothii/secrets`. Set individual mode `0600`, directory mode `0700`, and ownership readable
-by the service account. These path-only variables are mounted read-only by Compose:
+`/srv/thothii/secrets`. As established in the server guide, use owner UID 10001, group
+`thothii-ops`, file mode `0640`, and setgid directory mode `2750`. This lets the UID 10001
+container and the reviewed Docker operator running `thothctl` read the files without making them
+public. These path-only variables are mounted read-only by Compose:

 ```dotenv
 THT_WORKSPACE_GIT_CREDENTIALS_FILE=/srv/thothii/secrets/git-credentials
@@ -43,19 +43,29 @@ sudo useradd --system --uid 10001 --user-group --home-dir /srv/thothii \
  --create-home --shell /usr/sbin/nologin thothii
 ```

-The human operator who runs `thothctl` needs Docker access. On many installations membership in
-the `docker` group is effectively host-root access; grant it only according to site policy. The
-non-login `thothii` account owns application data and secrets but does not itself need Docker
-access.
+Use a dedicated `thothii-ops` group for the small set of human operators. A human who runs
+`thothctl` must be in both `thothii-ops` (to traverse operator paths and read declared secret files
+for output redaction) and the host `docker` group (to invoke Docker). Docker-group membership is
+effectively host-root access; grant both memberships only to reviewed administrators. The
+non-login `thothii` account owns files and writable data but does not need Docker access.
+
+```sh
+sudo groupadd --system thothii-ops
+sudo usermod --append --groups thothii-ops,docker "$USER"
+```
+
+Log out and back in before continuing; `id` must show both groups. Do not run `thothctl` through
+`sudo -u thothii`: that account deliberately lacks Docker access. Do not grant the human direct
+write access to runtime bind trees.

 Create explicit directories. `source` contains the clone; `operator` contains untracked path-only
 configuration; the three writable trees are bind-mounted into `core`; `secrets` contains regular
 files only. Backups are separate from live data.

 ```sh
-sudo install -d -o 10001 -g 10001 -m 0750 /srv/thothii/source
-sudo install -d -o 10001 -g 10001 -m 0700 /srv/thothii/operator
-sudo install -d -o 10001 -g 10001 -m 0700 /srv/thothii/secrets
+sudo install -d -o 10001 -g thothii-ops -m 2750 /srv/thothii/source
+sudo install -d -o 10001 -g thothii-ops -m 2750 /srv/thothii/operator
+sudo install -d -o 10001 -g thothii-ops -m 2750 /srv/thothii/secrets
 sudo install -d -o 10001 -g 10001 -m 0750 /srv/thothii/data
 sudo install -d -o 10001 -g 10001 -m 0750 /srv/thothii/pi-state
 sudo install -d -o 10001 -g 10001 -m 0750 /srv/thothii/workspace-registry
@@ -103,8 +113,50 @@ services:
 ```

 Use `host.docker.internal` in the endpoint binding. The `host-gateway` mapping supplies routing;
-it does not bundle or trust the target service. Keep the target port bound/firewalled for Docker
-host access only. A stable internal DNS record is the preferred alternative.
+it does not bundle or trust the target service. A host service listening only on host
+`127.0.0.1` is **not reachable** through this mapping. Bind that service to the ThothII Docker
+bridge gateway address or to a dedicated private host interface—never to `0.0.0.0` merely to make
+the check pass. A stable internal DNS record routed through an authenticated private listener is
+the preferred alternative.
+
+After the first bounded start attempt, copy the exact core container name from `thothctl status`
+into `CORE_NAME`, then derive—not guess—the network ID, Linux bridge interface, gateway, and
+subnet. Compose networks normally use `br-<first-12-network-id>`; an explicit
+`com.docker.network.bridge.name` option takes precedence:
+
+```sh
+CORE_NAME=replace-with-exact-core-container-name
+NETWORK_ID=$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.NetworkID}}{{end}}' "$CORE_NAME")
+NETWORK_NAME=$(docker network inspect --format '{{.Name}}' "$NETWORK_ID")
+BRIDGE=$(docker network inspect --format '{{index .Options "com.docker.network.bridge.name"}}' "$NETWORK_ID")
+test -n "$BRIDGE" || BRIDGE="br-${NETWORK_ID%${NETWORK_ID#????????????}}"
+GATEWAY=$(docker network inspect --format '{{(index .IPAM.Config 0).Gateway}}' "$NETWORK_ID")
+SUBNET=$(docker network inspect --format '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK_ID")
+printf 'network=%s bridge=%s gateway=%s subnet=%s\n' "$NETWORK_NAME" "$BRIDGE" "$GATEWAY" "$SUBNET"
+ip address show dev "$BRIDGE"
+```
+
+Bind the co-resident service to `$GATEWAY`. In the host firewall `INPUT` chain, allow its exact
+TCP port only when source is `$SUBNET`, input interface is `$BRIDGE`, and destination is
+`$GATEWAY`; reject other sources to that listener and persist the rules using the distribution's
+firewall manager. Docker's `DOCKER-USER` chain governs forwarded/published traffic and does not
+replace this host-input rule. Ask the firewall administrator to implement the equivalent policy
+with nftables when iptables is not the site's source of truth.
+
+For an iptables-managed host, replace the port before applying these reviewed rules; the second
+rule prevents any other interface/source from reaching that gateway listener:
+
+```sh
+EXTERNAL_PORT=replace-with-exact-service-port
+sudo iptables -I INPUT 1 -i "$BRIDGE" -s "$SUBNET" -d "$GATEWAY" -p tcp --dport "$EXTERNAL_PORT" -j ACCEPT
+sudo iptables -I INPUT 2 -d "$GATEWAY" -p tcp --dport "$EXTERNAL_PORT" -j REJECT
+```
+
+Confirm reachability with `thothctl pi test` for the configured LLM/Pi path and with the
+authenticated Workspace Diagnostics action for DWH, vector collection/embedding pairing, and
+embedding endpoints. A timeout paired with `ss -lntp`, `ip address show dev "$BRIDGE"`, and the
+firewall counters distinguishes a loopback bind from a subnet/interface rule failure. Do not add
+a shell to the browser or mount the Docker socket into core for this diagnostic.

 Configure each boundary independently:

@@ -136,7 +188,9 @@ Copy the path-only server environment and installation descriptor:
 sudo -u thothii cp deploy/env/server.env.example /srv/thothii/operator/server.env
 sudo -u thothii cp docs/install/examples/thothii-installation.server.yaml \
  /srv/thothii/operator/thothii-installation.yaml
-sudo chmod 0600 /srv/thothii/operator/server.env \
+sudo chown 10001:thothii-ops /srv/thothii/operator/server.env \
+  /srv/thothii/operator/thothii-installation.yaml
+sudo chmod 0640 /srv/thothii/operator/server.env \
  /srv/thothii/operator/thothii-installation.yaml
 ```

@@ -146,15 +200,16 @@ session-server overlay and generated connector-secret override. Optional host-ga
 image overrides go after them.

 Create each credential as an independent regular file in `/srv/thothii/secrets`, owned by
-UID/GID 10001 and mode `0600`. The operator environment records only absolute `*_FILE` or
+UID 10001, group `thothii-ops`, and mode `0640`. Owner access lets the UID 10001 container read a
+file mounted under `/run/secrets`; group access lets the reviewed human run `thothctl`. The
+operator environment records only absolute `*_FILE` or
 `*_SOURCE` paths. Compose mounts application and connector targets read-only under `/run/secrets`;
 the frontend receives none. Do not print file contents while testing permissions.

 ```sh
-sudo find /srv/thothii/secrets -type f -exec chown 10001:10001 {} +
-sudo find /srv/thothii/secrets -type f -exec chmod 0600 {} +
-sudo find /srv/thothii/secrets -type f ! -user thothii -print
-sudo find /srv/thothii/secrets -type f ! -perm 0600 -print
+sudo find /srv/thothii/secrets -type f -exec chown 10001:thothii-ops {} +
+sudo find /srv/thothii/secrets -type f -exec chmod 0640 {} +
+sudo find /srv/thothii/secrets -type f \( ! -user thothii -o ! -group thothii-ops -o ! -perm 0640 \) -print
 ```

 Add `THT_WORKSPACE_BINDINGS_ENV_FILE=/srv/thothii/operator/workspace-bindings.env` and the matching
@@ -185,13 +240,18 @@ services:
  core:
    build: !reset null
    image: registry.example.com/thothii/core@sha256:<64-lowercase-hex-digits>
+  session-migrate:
+    build: !reset null
+    image: registry.example.com/thothii/core@sha256:<64-lowercase-hex-digits>
  frontend:
    build: !reset null
    image: registry.example.com/thothii/frontend@sha256:<64-lowercase-hex-digits>
 ```

-Add that absolute file last in `overrides`. Both images must come from one compatible release; the
-core image must retain the declared Pi version labels checked by `thothctl pi doctor`. Pull access
+Add that absolute file last in `overrides`. `core` and `session-migrate` must use the exact same
+core digest; neither may retain a local build or `:local` image. Frontend uses its own exact digest.
+Both images must come from one compatible release; the core image must retain the declared Pi
+version labels checked by `thothctl pi doctor`. Pull access
 belongs in the host Docker credential store, not in Compose or the installation descriptor.

 ## Install thothctl
@@ -201,7 +261,7 @@ Build the operator binaries with Docker. No Go installation or Go knowledge is r
 ```sh
 cd /srv/thothii/source/ThothII
 bash scripts/build-thothctl.sh
-sudo install -o 10001 -g 10001 -m 0755 dist/thothctl/thothctl-linux-amd64 \
+sudo install -o root -g thothii-ops -m 0750 dist/thothctl/thothctl-linux-amd64 \
  /srv/thothii/operator/thothctl
 ```

@@ -211,7 +271,7 @@ not source `server.env` as shell code:
 ```sh
 THTCTL=/srv/thothii/operator/thothctl
 INSTALLATION=/srv/thothii/operator/thothii-installation.yaml
-"$THTCTL" --installation "$INSTALLATION" --help
+"$THTCTL" --help
 "$THTCTL" --installation "$INSTALLATION" update --check-only
 ```

@@ -221,7 +281,24 @@ profile, overrides, project identity, and durable current-image selector. The ge

 ## Start and verify readiness

-Keep the TLS proxy stopped or firewalled during bootstrap:
+Keep the TLS proxy stopped or firewalled during bootstrap. First stop the app, run the
+installation-aware session migration, and inspect its pristine JSON. The command activates only
+the `session-migrate` profile/service with `--no-deps --no-TTY`; it derives the migrator image from the
+selected core image after all installation overrides, so this procedure is identical for source
+and pinned modes. It exits nonzero unless both arrays are empty:
+
+```sh
+"$THTCTL" --installation "$INSTALLATION" stop
+"$THTCTL" --installation "$INSTALLATION" sessions migrate --yes
+```
+
+Successful output has this shape (the `applied` list may contain versions on first use):
+
+```json
+{"applied":[],"drifted":[],"pending":[]}
+```
+
+Only after seeing `"pending":[]` and `"drifted":[]`, start and verify:

 ```sh
 "$THTCTL" --installation "$INSTALLATION" start
@@ -303,7 +380,7 @@ BACKUP=/srv/thothii-backups/2026-08-05
 sudo install -d -o root -g root -m 0700 "$BACKUP"
 sudo tar --numeric-owner --xattrs --acls -C /srv/thothii -czf "$BACKUP/runtime-data.tgz" \
  data pi-state workspace-registry
-sudo sha256sum "$BACKUP/runtime-data.tgz" >"$BACKUP/SHA256SUMS"
+sudo sh -ceu 'cd "$1"; sha256sum runtime-data.tgz > SHA256SUMS; sha256sum --check SHA256SUMS' sh "$BACKUP"
 ```

 Back up the installation descriptor, path-only environment, generated overrides, source revision,
@@ -318,7 +395,7 @@ the restored set. This keeps the previous state recoverable:
 ```sh
 RESTORE=/srv/thothii-restore-2026-08-05
 sudo install -d -o root -g root -m 0700 "$RESTORE"
-sudo sha256sum --check /srv/thothii-backups/2026-08-05/SHA256SUMS
+sudo sh -ceu 'cd "$1"; sha256sum --check SHA256SUMS' sh /srv/thothii-backups/2026-08-05
 sudo tar --numeric-owner --xattrs --acls -C "$RESTORE" \
  -xzf /srv/thothii-backups/2026-08-05/runtime-data.tgz
 sudo test -d "$RESTORE/workspace-registry/repo"
@@ -357,9 +434,30 @@ external service identity; and the proxy/identity provider for login failures.

 ## Data-preserving uninstall

-Drain and stop through `thothctl`, take and verify one final backup, disable the TLS proxy route,
-and remove only this installation's stopped `frontend` and `core` containers and optional images
-by their exact Compose project labels. Keep `/srv/thothii/data`, `pi-state`,
+Drain and stop through `thothctl`, take and verify one final backup, and disable the TLS proxy
+route. Set `THT_BACKUP_ROOT=/srv/thothii-backups` in `server.env`; the removal command verifies the
+filesystem identity of that backup root, all three bind trees, and every declared secret before
+and after removing anything.
+
+First run without confirmation. It displays the exact installation project, service, container
+name, container ID, and stopped state, then exits without mutation. Check every target:
+
+```sh
+"$THTCTL" --installation "$INSTALLATION" stop
+"$THTCTL" --installation "$INSTALLATION" remove
+```
+
+If and only if both targets are the expected stopped `frontend` and `core` containers, confirm:
+
+```sh
+"$THTCTL" --installation "$INSTALLATION" remove --yes exact-core-id exact-frontend-id
+```
+
+Replace both example IDs with the values from the immediately preceding dry-run. The command
+refuses confirmation if the current target set differs. The confirmed operation passes only those
+previously displayed immutable container IDs to Docker,
+uses no force or volume option, rejects running/replaced containers, and proves the preservation
+paths still identify the same filesystem objects. Keep `/srv/thothii/data`, `pi-state`,
 `workspace-registry`, `operator`, protected secrets, database backups, and the installation
 descriptor if reinstallation is possible. Do not prune global Docker data.

@@ -20,6 +20,8 @@ for fixture in \
  "Caddy reverse-proxy guide contract" \
  "local installation example rendered from path with spaces" \
  "server installation example rendered from path with spaces" \
+  "server pinned migration image fixture" \
+  "server backup checksum root-only fixture" \
  "local manual canonical base+override references" \
  "server manual canonical base+override references" \
  "canonical local base+override fixture" \
@@ -33,6 +35,32 @@ for fixture in \
  }
 done

+server_guide="$root/docs/install/server.md"
+for required in \
+  'thothii-ops' \
+  'THT_BACKUP_ROOT=/srv/thothii-backups' \
+  'sessions migrate --yes' \
+  '"pending":[]' \
+  '"drifted":[]' \
+  'remove --yes' \
+  'sha256sum --check SHA256SUMS' \
+  'DOCKER-USER' \
+  'iptables -I INPUT' \
+  'com.docker.network.bridge.name'; do
+  grep -Fq -- "$required" "$server_guide" || {
+    echo "server operations guide lacks executable contract: $required" >&2
+    exit 1
+  }
+done
+grep -Fq '"$THTCTL" --help' "$server_guide" || {
+  echo "server guide lacks plain thothctl --help" >&2
+  exit 1
+}
+if grep -Fq '"$THTCTL" --installation "$INSTALLATION" --help' "$server_guide"; then
+  echo "server guide still uses installation-scoped --help" >&2
+  exit 1
+fi
+
 for manual in "$root/docs/install/local-workspace-registry.md"; do
  grep -Fq 'export THT_SOURCE_ROOT=/absolute/path/to/ThothII' "$manual" || {
    echo "installation manual does not publish a self-contained THT_SOURCE_ROOT export: $manual" >&2
@@ -114,6 +142,18 @@ switch (mutation) {
  case "server-coupling":
    changed += "\nAttach core to the omics_portal application network.\n";
    break;
+  case "server-host-loopback":
+    changed += "\nFor host-gateway, keep the external service listening on 127.0.0.1.\n";
+    break;
+  case "server-raw-remove":
+    changed += "\n```sh\ndocker rm thothii-core thothii-frontend\n```\n";
+    break;
+  case "server-pinned-migrator-mismatch":
+    changed += "\n```yaml\nservices:\n  core:\n    image: registry.invalid/core@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n  session-migrate:\n    image: thothii-core:local\n```\n";
+    break;
+  case "server-pinned-frontend-missing":
+    changed = original.replace('  frontend:\n    build: !reset null\n    image: registry.example.com/thothii/frontend@sha256:<64-lowercase-hex-digits>\n', '');
+    break;
  case "nginx-no-auth":
    changed = original.replace("        auth_request /_authenticate;", "        # authentication omitted");
    break;
@@ -123,6 +163,18 @@ switch (mutation) {
  case "nginx-no-sse":
    changed = original.replace("        proxy_buffering off;", "        proxy_buffering on;");
    break;
+  case "nginx-no-issuer-clear":
+    changed = original.replaceAll('proxy_set_header X-Thoth-Principal-Issuer "";', 'proxy_set_header X-Thoth-Principal-Issuer $http_x_thoth_principal_issuer;');
+    break;
+  case "nginx-no-subject-capture":
+    changed = original.replace("auth_request_set $thoth_principal_subject", "# missing auth capture $thoth_principal_subject");
+    break;
+  case "nginx-no-display-map":
+    changed = original.replace("proxy_set_header X-Thoth-Trusted-Principal-Display-Name $thoth_principal_display_name;", "proxy_set_header X-Thoth-Trusted-Principal-Display-Name \"\";");
+    break;
+  case "nginx-no-admin-map":
+    changed = original.replace("proxy_set_header X-Thoth-Trusted-Is-Admin $thoth_is_admin;", "proxy_set_header X-Thoth-Trusted-Is-Admin \"\";");
+    break;
  case "caddy-no-auth":
    changed = original.replace("forward_auth auth-gateway:4180 {", "# forward authentication omitted");
    break;
@@ -132,6 +184,18 @@ switch (mutation) {
  case "caddy-core-upstream":
    changed = original.replaceAll("127.0.0.1:8080", "127.0.0.1:8787");
    break;
+  case "caddy-no-issuer-public-clear":
+    changed = original.replace("request_header -X-Thoth-Principal-Issuer", "request_header X-Thoth-Principal-Issuer {header.X-Thoth-Principal-Issuer}");
+    break;
+  case "caddy-no-subject-trusted-clear":
+    changed = original.replace("request_header -X-Thoth-Trusted-Principal-Subject", "request_header X-Thoth-Trusted-Principal-Subject {header.X-Thoth-Trusted-Principal-Subject}");
+    break;
+  case "caddy-no-display-map":
+    changed = original.replace("X-Thoth-Principal-Display-Name>X-Thoth-Trusted-Principal-Display-Name", "X-Thoth-Principal-Display-Name");
+    break;
+  case "caddy-no-admin-map":
+    changed = original.replace("X-Thoth-Is-Admin>X-Thoth-Trusted-Is-Admin", "X-Thoth-Is-Admin");
+    break;
  case "dirty-source":
    changed = original.replaceAll("git status --porcelain --untracked-files=all", "git status --short");
    break;
@@ -215,6 +279,22 @@ expect_guide_rejected \
  "server application coupling" verify_server_guide \
  "$root/docs/install/server.md" docs/install/server.md server-coupling \
  "server installation guide introduces forbidden application coupling"
+expect_guide_rejected \
+  "server host-gateway loopback listener" verify_server_guide \
+  "$root/docs/install/server.md" docs/install/server.md server-host-loopback \
+  "server host-gateway guidance assumes a host loopback listener"
+expect_guide_rejected \
+  "server raw container removal" verify_server_guide \
+  "$root/docs/install/server.md" docs/install/server.md server-raw-remove \
+  "server uninstall bypasses installation-aware removal"
+expect_guide_rejected \
+  "server pinned migrator differs from core" verify_server_guide \
+  "$root/docs/install/server.md" docs/install/server.md server-pinned-migrator-mismatch \
+  "server pinned migration image must equal the pinned core image"
+expect_guide_rejected \
+  "server pinned frontend is missing" verify_server_guide \
+  "$root/docs/install/server.md" docs/install/server.md server-pinned-frontend-missing \
+  "server pinned image override must pin core, session-migrate, and frontend without builds"
 expect_guide_rejected \
  "Nginx identity without authentication" verify_reverse_proxy_nginx_guide \
  "$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-auth \
@@ -227,6 +307,22 @@ expect_guide_rejected \
  "Nginx buffered SSE" verify_reverse_proxy_nginx_guide \
  "$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-sse \
  "Nginx proxy lacks structural token: proxy_buffering off;"
+expect_guide_rejected \
+  "Nginx issuer inbound claim not cleared" verify_reverse_proxy_nginx_guide \
+  "$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-issuer-clear \
+  "Nginx proxy does not clear inbound issuer identity"
+expect_guide_rejected \
+  "Nginx subject auth response not captured" verify_reverse_proxy_nginx_guide \
+  "$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-subject-capture \
+  "Nginx proxy does not capture authenticated subject identity"
+expect_guide_rejected \
+  "Nginx display identity not mapped to private hop" verify_reverse_proxy_nginx_guide \
+  "$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-display-map \
+  "Nginx proxy does not map authenticated display identity"
+expect_guide_rejected \
+  "Nginx admin identity not mapped to private hop" verify_reverse_proxy_nginx_guide \
+  "$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-admin-map \
+  "Nginx proxy does not map authenticated admin identity"
 expect_guide_rejected \
  "Caddy identity without authentication" verify_reverse_proxy_caddy_guide \
  "$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-auth \
@@ -234,11 +330,27 @@ expect_guide_rejected \
 expect_guide_rejected \
  "Caddy untrusted identity forwarding" verify_reverse_proxy_caddy_guide \
  "$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-client-identity \
-  "Caddy proxy lacks structural token: X-Thoth-Principal-Subject>X-Thoth-Trusted-Principal-Subject"
+  "Caddy proxy does not map authenticated subject identity"
 expect_guide_rejected \
  "Caddy direct core exposure" verify_reverse_proxy_caddy_guide \
  "$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-core-upstream \
  "Caddy proxy must forward only to frontend on 127.0.0.1:8080"
+expect_guide_rejected \
+  "Caddy issuer inbound claim not cleared" verify_reverse_proxy_caddy_guide \
+  "$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-issuer-public-clear \
+  "Caddy proxy does not clear inbound issuer identity"
+expect_guide_rejected \
+  "Caddy subject private-hop claim not cleared" verify_reverse_proxy_caddy_guide \
+  "$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-subject-trusted-clear \
+  "Caddy proxy does not clear inbound trusted subject identity"
+expect_guide_rejected \
+  "Caddy display identity not mapped to private hop" verify_reverse_proxy_caddy_guide \
+  "$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-display-map \
+  "Caddy proxy does not map authenticated display identity"
+expect_guide_rejected \
+  "Caddy admin identity not mapped to private hop" verify_reverse_proxy_caddy_guide \
+  "$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-admin-map \
+  "Caddy proxy does not map authenticated admin identity"

 expect_guide_rejected \
  "dirty or untracked source tree" verify_local_guide \
@@ -512,12 +512,16 @@ verify_server_guide() {
    "frontend" \
    "core" \
    "UID/GID 10001" \
+    "thothii-ops" \
    "/srv/thothii" \
    "example operator root" \
    "/run/secrets" \
    "Git-backed workspace registry is the source of truth" \
    "host.docker.internal" \
    "host-gateway" \
+    "com.docker.network.bridge.name" \
+    "DOCKER-USER" \
+    "iptables -I INPUT" \
    "container 127.0.0.1" \
    "collection" \
    "embedding" \
@@ -525,6 +529,12 @@ verify_server_guide() {
    "@sha256:" \
    "bash scripts/build-thothctl.sh" \
    "thothctl --installation" \
+    "sessions migrate --yes" \
+    '"pending":[]' \
+    '"drifted":[]' \
+    "remove --yes" \
+    "THT_BACKUP_ROOT=/srv/thothii-backups" \
+    "sha256sum --check SHA256SUMS" \
    "curl --fail http://127.0.0.1:8080/health" \
    "https://thoth.example.com" \
    "pi update" \
@@ -564,6 +574,36 @@ for (const line of source.split(/\n/)) {
    throw new Error("server docker compose down --volumes must appear only in an explicit prose prohibition");
  }
 }
+if (/```(?:sh|bash)\n[\s\S]*?\bdocker\s+rm\b[\s\S]*?```/i.test(source)) {
+  throw new Error("server uninstall bypasses installation-aware removal");
+}
+if (/host-gateway[^\n]{0,120}(?:listen|listening|bound)[^\n]{0,80}127\.0\.0\.1|(?:listen|listening|bound)[^\n]{0,80}127\.0\.0\.1[^\n]{0,120}host-gateway/i.test(source)) {
+  throw new Error("server host-gateway guidance assumes a host loopback listener");
+}
+const pinnedStart = source.indexOf("## Build locally or select pinned images");
+const pinnedEnd = source.indexOf("\n## ", pinnedStart + 3);
+const pinnedSection = source.slice(pinnedStart, pinnedEnd < 0 ? source.length : pinnedEnd);
+const pinnedBlock = [...pinnedSection.matchAll(/```yaml\n([\s\S]*?)```/g)].map((match) => match[1])
+  .find((block) => block.includes("session-migrate:")) || "";
+function pinnedService(name) {
+  const match = pinnedBlock.match(new RegExp(`^  ${name}:\\n((?:    [^\\n]*\\n)+)`, "m"));
+  return match ? match[1] : "";
+}
+const pinnedCore = pinnedService("core");
+const pinnedMigrator = pinnedService("session-migrate");
+const pinnedFrontend = pinnedService("frontend");
+const coreImage = pinnedCore.match(/image:\s*(\S+)/)?.[1];
+const migratorImage = pinnedMigrator.match(/image:\s*(\S+)/)?.[1];
+const frontendImage = pinnedFrontend.match(/image:\s*(\S+)/)?.[1];
+if (![pinnedCore, pinnedMigrator, pinnedFrontend].every((block) => block.includes("build: !reset null")) ||
+    !coreImage || coreImage !== migratorImage || !/@sha256:<64-lowercase-hex-digits>$/.test(coreImage) ||
+    !frontendImage || !/@sha256:<64-lowercase-hex-digits>$/.test(frontendImage)) {
+  throw new Error("server pinned image override must pin core, session-migrate, and frontend without builds");
+}
+if (/session-migrate:[\s\S]{0,180}image:\s*thothii-core:local/.test(source) &&
+    /core:[\s\S]{0,180}image:\s*registry\.[^\n]+@sha256:[a-f0-9]{64}/.test(source)) {
+  throw new Error("server pinned migration image must equal the pinned core image");
+}
 if (/```(?:sh|bash)\n[\s\S]*?\bdocker compose\s+(?:up|stop|down|restart|pull|build)\b[\s\S]*?```/i.test(source)) {
  throw new Error("server lifecycle must use thothctl, not raw Docker Compose");
 }
@@ -595,11 +635,9 @@ const block = [...source.matchAll(/```nginx\n([\s\S]*?)```/g)].map((match) => ma
 const tokens = [
  "listen 443 ssl;", "ssl_certificate ", "ssl_certificate_key ",
  "location = /_authenticate {", "internal;", "proxy_pass http://auth-gateway:4180/verify;",
-  "auth_request /_authenticate;", "auth_request_set $thoth_principal_subject",
-  "$upstream_http_x_thoth_principal_subject", "proxy_pass http://127.0.0.1:8080;",
+  "auth_request /_authenticate;", "proxy_pass http://127.0.0.1:8080;",
  "proxy_http_version 1.1;", "proxy_buffering off;", "proxy_cache off;",
-  "proxy_read_timeout 3600s;", "proxy_set_header X-Thoth-Principal-Subject \"\";",
-  "proxy_set_header X-Thoth-Trusted-Principal-Subject $thoth_principal_subject;",
+  "proxy_read_timeout 3600s;",
 ];
 if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(block) || !block.includes("http://127.0.0.1:8080")) {
  throw new Error("Nginx proxy must forward only to frontend on 127.0.0.1:8080");
@@ -610,6 +648,28 @@ for (const token of tokens) {
 if (/proxy_set_header\s+X-Thoth-Trusted-[^;]+\$http_/i.test(block)) {
  throw new Error("Nginx proxy trusts a client-supplied identity header");
 }
+const identities = [
+  ["issuer", "Principal-Issuer", "thoth_principal_issuer", "x_thoth_principal_issuer"],
+  ["subject", "Principal-Subject", "thoth_principal_subject", "x_thoth_principal_subject"],
+  ["display", "Principal-Display-Name", "thoth_principal_display_name", "x_thoth_principal_display_name"],
+  ["admin", "Is-Admin", "thoth_is_admin", "x_thoth_is_admin"],
+];
+function escaped(value) { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); }
+for (const [label, publicName, variable, upstream] of identities) {
+  const trustedName = publicName === "Is-Admin" ? "Is-Admin" : publicName;
+  const publicClears = block.match(new RegExp(`proxy_set_header\\s+X-Thoth-${escaped(publicName)}\\s+"";`, "g")) || [];
+  if (publicClears.length < 2) throw new Error(`Nginx proxy does not clear inbound ${label} identity`);
+  const trustedHeader = `X-Thoth-Trusted-${trustedName}`;
+  if (!new RegExp(`proxy_set_header\\s+${escaped(trustedHeader)}\\s+"";`).test(block)) {
+    throw new Error(`Nginx proxy does not clear inbound trusted ${label} identity`);
+  }
+  if (!new RegExp(`auth_request_set\\s+\\$${variable}\\s+\\$upstream_http_${upstream};`, "m").test(block.replace(/\s+/g, " "))) {
+    throw new Error(`Nginx proxy does not capture authenticated ${label} identity`);
+  }
+  if (!new RegExp(`proxy_set_header\\s+${escaped(trustedHeader)}\\s+\\$${variable};`).test(block)) {
+    throw new Error(`Nginx proxy does not map authenticated ${label} identity`);
+  }
+}
 NODE
  echo "Nginx reverse-proxy guide contract passed"
 }
@@ -637,10 +697,7 @@ const source = fs.readFileSync(process.argv[2], "utf8");
 const block = [...source.matchAll(/```caddyfile\n([\s\S]*?)```/g)].map((match) => match[1]).join("\n");
 const tokens = [
  "thoth.example.com {", "route {",
-  "request_header -X-Thoth-Principal-Subject",
-  "request_header -X-Thoth-Trusted-Principal-Subject",
  "forward_auth auth-gateway:4180 {", "uri /verify", "copy_headers {",
-  "X-Thoth-Principal-Subject>X-Thoth-Trusted-Principal-Subject",
  "reverse_proxy 127.0.0.1:8080 {", "flush_interval -1",
 ];
 if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(block) || !block.includes("127.0.0.1:8080")) {
@@ -649,6 +706,22 @@ if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(block) || !block.includes("127.0.0.1:
 for (const token of tokens) {
  if (!block.includes(token)) throw new Error(`Caddy proxy lacks structural token: ${token}`);
 }
+for (const [label, publicName, trustedName] of [
+  ["issuer", "X-Thoth-Principal-Issuer", "X-Thoth-Trusted-Principal-Issuer"],
+  ["subject", "X-Thoth-Principal-Subject", "X-Thoth-Trusted-Principal-Subject"],
+  ["display", "X-Thoth-Principal-Display-Name", "X-Thoth-Trusted-Principal-Display-Name"],
+  ["admin", "X-Thoth-Is-Admin", "X-Thoth-Trusted-Is-Admin"],
+]) {
+  if (!block.includes(`request_header -${publicName}`)) {
+    throw new Error(`Caddy proxy does not clear inbound ${label} identity`);
+  }
+  if (!block.includes(`request_header -${trustedName}`)) {
+    throw new Error(`Caddy proxy does not clear inbound trusted ${label} identity`);
+  }
+  if (!block.includes(`${publicName}>${trustedName}`)) {
+    throw new Error(`Caddy proxy does not map authenticated ${label} identity`);
+  }
+}
 NODE
  echo "Caddy reverse-proxy guide contract passed"
 }
@@ -821,7 +894,7 @@ verify_server_installation_example() {
    return 1
  }

-  local fixture source_copy operator_dir copied_example connector_override env_file
+  local fixture source_copy operator_dir copied_example connector_override env_file backup_root
  fixture="$(mktemp -d "${TMPDIR%/}/thoth server install.XXXXXX")"
  trap 'rm -rf "$fixture"' RETURN
  [[ "$fixture" == *" "* ]] || {
@@ -830,8 +903,9 @@ verify_server_installation_example() {
  }
  source_copy="$fixture/ThothII server source"
  operator_dir="$fixture/server operator files"
+  backup_root="$fixture/server backups"
  mkdir -p "$source_copy/deploy/pi" "$source_copy/deploy/workspaces" \
-    "$operator_dir/data" "$operator_dir/pi-state" "$operator_dir/workspace-registry"
+    "$operator_dir/data" "$operator_dir/pi-state" "$operator_dir/workspace-registry" "$backup_root"
  cp "$root/compose.yaml" "$source_copy/compose.yaml"
  cp "$root/deploy/compose.server.yaml" "$source_copy/deploy/compose.server.yaml"
  cp "$root/deploy/compose.session-server.yaml.example" \
@@ -869,6 +943,7 @@ verify_server_installation_example() {
    "THT_DATA_ROOT=$operator_dir/data" \
    "THT_PI_STATE_ROOT=$operator_dir/pi-state" \
    "THT_WORKSPACE_REGISTRY_ROOT=$operator_dir/workspace-registry" \
+    "THT_BACKUP_ROOT=$backup_root" \
    "THT_SERVER_WORKSPACE_CONFIG=$source_copy/deploy/workspaces/server-sessions.yaml.example" \
    'THT_LLM_URL=https://llm.example.invalid' \
    'THT_SESSION_DB_HOST=sessions.example.invalid' \
@@ -942,6 +1017,62 @@ for (const secret of [
 }
 NODE
  echo "server installation example rendered from path with spaces passed"
+
+  local migration_rendered="$fixture/server-migration.json"
+  "$root/scripts/compose-with-preflight.sh" --env-file "$descriptor_env" \
+    "${files[@]}" --profile session-migrate config --format json >"$migration_rendered"
+  node - "$migration_rendered" <<'NODE'
+const fs = require("fs");
+const config = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
+const services = config.services || {};
+if (!services.core || !services["session-migrate"]) throw new Error("server migration profile is missing core or session-migrate");
+if (services.core.image !== services["session-migrate"].image) throw new Error("source migration image differs from core");
+if (services["session-migrate"].build) throw new Error("source migration service unexpectedly declares a build");
+NODE
+
+  local pinned_template="$fixture/pinned-template.yaml" pinned_override="$operator_dir/pinned-images.yaml"
+  awk '
+    /^## Build locally or select pinned images$/ { section=1; next }
+    section && /^```yaml$/ { code=1; next }
+    code && /^```$/ { exit }
+    code { print }
+  ' "$root/docs/install/server.md" >"$pinned_template"
+  sed \
+    -e "s#registry.example.com/thothii/core@sha256:<64-lowercase-hex-digits>#registry.example.com/thothii/core@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa#g" \
+    -e "s#registry.example.com/thothii/frontend@sha256:<64-lowercase-hex-digits>#registry.example.com/thothii/frontend@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb#g" \
+    "$pinned_template" >"$pinned_override"
+  chmod 0600 "$pinned_override"
+  local pinned_rendered="$fixture/server-pinned-migration.json"
+  "$root/scripts/compose-with-preflight.sh" --env-file "$descriptor_env" \
+    "${files[@]}" -f "$pinned_override" --profile session-migrate config --format json >"$pinned_rendered"
+  node - "$pinned_rendered" <<'NODE'
+const fs = require("fs");
+const config = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
+const core = config.services?.core;
+const frontend = config.services?.frontend;
+const migrator = config.services?.["session-migrate"];
+if (!core || !frontend || !migrator) throw new Error("pinned migration profile lacks core, frontend, or session-migrate");
+if (core.image !== migrator.image || !/@sha256:[a-f0-9]{64}$/.test(core.image)) {
+  throw new Error("pinned migration image does not equal the exact core digest");
+}
+if (!/@sha256:[a-f0-9]{64}$/.test(frontend.image)) throw new Error("frontend is not pinned by exact digest");
+for (const [name, service] of Object.entries({core, frontend, migrator})) {
+  if (service.build) throw new Error(name + " retained a local build in pinned mode");
+  if (/:local$/.test(service.image || "")) throw new Error(name + " retained a local image in pinned mode");
+}
+NODE
+  echo "server pinned migration image fixture passed"
+
+  local checksum_root="$fixture/root-only-checksum"
+  mkdir -m 0700 "$checksum_root"
+  printf 'fixture backup bytes\n' >"$checksum_root/runtime-data.tgz"
+  /bin/sh -ceu 'cd "$1"; sha256sum runtime-data.tgz > SHA256SUMS; sha256sum --check SHA256SUMS' sh "$checksum_root" >/dev/null
+  printf 'corruption\n' >>"$checksum_root/runtime-data.tgz"
+  if (cd "$checksum_root" && sha256sum --check SHA256SUMS) >/dev/null 2>&1; then
+    echo "corrupted server backup checksum fixture was accepted" >&2
+    return 1
+  fi
+  echo "server backup checksum root-only fixture passed"
 }

 write_private() {
@@ -18,6 +18,7 @@ import (
 	"github.com/aritmolab/thothii/tools/thothctl/internal/config"
 	"github.com/aritmolab/thothii/tools/thothctl/internal/output"
 	"github.com/aritmolab/thothii/tools/thothctl/internal/pi"
+	"github.com/aritmolab/thothii/tools/thothctl/internal/serverops"
 )

 const usage = `Usage: thothctl --installation <absolute-path>/thothii-installation.yaml <command>
@@ -29,6 +30,10 @@ Commands:
  start               Start the installation in the background.
  stop                Stop the installation.
  update --check-only Validate the current installation without changing containers.
+  sessions migrate --yes
+                      Run only the server session migrator and verify pending=[] and drifted=[].
+  remove              Display exact stopped app container IDs without mutation.
+  remove --yes ID...  Remove only the stopped IDs copied from the preceding display.
  pi status           Show the Pi version embedded in core.
  pi doctor           Check Pi preconditions without changing the installation.
  pi test             Run the temporary Pi/core smoke checks.
@@ -113,12 +118,67 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
 		return doctor(ctx, installation, runner, secretValues, stdout, stderr)
 	case "pi":
 		return piCommand(ctx, installation, runner, commandArgs, secretValues, stdout, stderr)
+	case "sessions":
+		if len(commandArgs) != 2 || commandArgs[0] != "migrate" || commandArgs[1] != "--yes" {
+			return commandUsageError(stderr, "sessions migrate requires --yes")
+		}
+		status, operationErr := serverops.MigrateSessions(ctx, installation, runner, true)
+		if operationErr != nil {
+			return serverOperationFailure(stderr, operationErr, secretValues)
+		}
+		if encodeErr := json.NewEncoder(stdout).Encode(status); encodeErr != nil {
+			fmt.Fprintln(stderr, "thothctl: migration status could not be written")
+			return 1
+		}
+		return 0
+	case "remove":
+		var confirmedIDs []string
+		if len(commandArgs) > 0 {
+			if commandArgs[0] != "--yes" || len(commandArgs) < 2 {
+				return commandUsageError(stderr, "remove requires either no arguments or --yes followed by every displayed container ID")
+			}
+			confirmedIDs = commandArgs[1:]
+		}
+		removal, operationErr := serverops.Remove(ctx, installation, runner, confirmedIDs)
+		writeRemovalTargets(stdout, installation.ProjectName(), removal.Targets)
+		if errors.Is(operationErr, serverops.ErrConfirmationRequired) {
+			fmt.Fprint(stderr, "thothctl: inspect the exact targets above, then re-run with remove --yes")
+			for _, target := range removal.Targets {
+				fmt.Fprintf(stderr, " %s", target.ID)
+			}
+			fmt.Fprintln(stderr)
+			return 2
+		}
+		if operationErr != nil {
+			return serverOperationFailure(stderr, operationErr, secretValues)
+		}
+		fmt.Fprintf(stdout, "Removed %d stopped app containers; verified %d preserved paths.\n", len(removal.Targets), removal.Preserved)
+		return 0
 	default:
 		return commandUsageError(stderr, fmt.Sprintf("unknown command %q", command))
 	}
 	return writeResult(result, err, secretValues, stdout, stderr)
 }

+func writeRemovalTargets(outputWriter io.Writer, project string, targets []serverops.Container) {
+	fmt.Fprintf(outputWriter, "Removal targets for installation project %s:\n", project)
+	if len(targets) == 0 {
+		fmt.Fprintln(outputWriter, "  (none)")
+		return
+	}
+	for _, target := range targets {
+		fmt.Fprintf(outputWriter, "  service=%s name=%s id=%s state=%s\n", target.Service, target.Name, target.ID, target.State)
+	}
+}
+
+func serverOperationFailure(stderr io.Writer, err error, secretValues []string) int {
+	fmt.Fprintf(stderr, "thothctl: %s\n", output.Sanitize(err.Error(), secretValues))
+	if errors.Is(err, serverops.ErrConfirmationRequired) || errors.Is(err, serverops.ErrUnsafeState) {
+		return 2
+	}
+	return 1
+}
+
 // installationRunner transforms only Compose invocations into the installation's validated,
 // profile-specific argument list. Direct Docker image commands remain host-side and use arguments.
 type installationRunner struct {
@@ -79,6 +79,8 @@ func TestUsageDocumentsClosedConfigureUpdateSourcesAndMaintenanceRecovery(t *tes
 		"--source pull --image IMAGE@sha256:DIGEST",
 		"pi maintenance status",
 		"pi maintenance recover --yes",
+		"sessions migrate --yes",
+		"remove --yes ID...",
 	} {
 		if !strings.Contains(usage, required) {
 			t.Errorf("usage missing %q", required)
@@ -86,6 +88,48 @@ func TestUsageDocumentsClosedConfigureUpdateSourcesAndMaintenanceRecovery(t *tes
 	}
 }

+func TestRunSessionsMigrateRequiresExplicitConfirmationBeforeDocker(t *testing.T) {
+	fixture := newCLIFixture(t, "")
+	fixture.setProfile(t, "server")
+	fixture.setEnvironment(t)
+	var stdout, stderr bytes.Buffer
+
+	code := run(context.Background(), []string{
+		"--installation", fixture.installationPath, "sessions", "migrate",
+	}, &stdout, &stderr)
+
+	if code != 2 || !strings.Contains(stderr.String(), "sessions migrate requires --yes") {
+		t.Fatalf("exit = %d, stderr = %q", code, stderr.String())
+	}
+	assertDockerNotInvoked(t, fixture)
+}
+
+func TestRunRemoveDisplaysExactInstallationTargetsBeforeConfirmation(t *testing.T) {
+	fixture := newCLIFixture(t, "")
+	fixture.setProfile(t, "server")
+	fixture.setEnvironment(t)
+	t.Setenv("THOTHCTL_FAKE_STOPPED_PS", `[{"ID":"core-id","Name":"exact-core","Service":"core","State":"exited"},{"ID":"front-id","Name":"exact-frontend","Service":"frontend","State":"exited"}]`)
+	var stdout, stderr bytes.Buffer
+
+	code := run(context.Background(), []string{
+		"--installation", fixture.installationPath, "remove",
+	}, &stdout, &stderr)
+
+	if code != 2 || !strings.Contains(stderr.String(), "re-run with remove --yes core-id front-id") {
+		t.Fatalf("exit = %d, stderr = %q", code, stderr.String())
+	}
+	for _, value := range []string{"exact-core", "core-id", "exact-frontend", "front-id", "exited"} {
+		if !strings.Contains(stdout.String(), value) {
+			t.Errorf("target display %q missing %q", stdout.String(), value)
+		}
+	}
+	calls := fixture.invocations(t)
+	if len(calls) != 1 {
+		t.Fatalf("Docker calls = %#v", calls)
+	}
+	assertInvocationContains(t, calls, "ps", "--all", "--format", "json", "core", "frontend")
+}
+
 type wizardRunner struct{ calls []string }

 func (r *wizardRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
@@ -615,6 +659,7 @@ func newCLIFixture(t *testing.T, envTemplate string) cliFixture {
 printf '%s\n' "$@" >> "$THOTHCTL_FAKE_ARGS"
 printf '%s\n' -- >> "$THOTHCTL_FAKE_ARGS"
 case " $* " in
+  *" ps --all --format json core frontend "*) printf '%s\n' "${THOTHCTL_FAKE_STOPPED_PS:-[]}" ;;
  *" config --format json "*) printf '%s\n' '{"volumes":{"settings":{}},"services":{"core":{"image":"thothii-core:local","environment":{"THT_LLM_URL":"https://llm.example.invalid"}}}}' ;;
  *" ps --format json "*) printf '%s\n' '[{"Service":"core","State":"running","Health":"healthy"},{"Service":"frontend","State":"running","Health":"healthy"}]' ;;
  *"io.thothii.pi.version"*) printf '%s\n' '0.80.3' ;;
@@ -661,6 +706,19 @@ func (f cliFixture) setEnvContents(t *testing.T, env string) {
 	t.Setenv("THOTHCTL_FAKE_LOG", "")
 	t.Setenv("THOTHCTL_FAKE_FAILURE", "")
 	t.Setenv("THOTHCTL_FAKE_FAIL_ON", "")
+	t.Setenv("THOTHCTL_FAKE_STOPPED_PS", "[]")
+}
+
+func (f cliFixture) setProfile(t *testing.T, profile string) {
+	t.Helper()
+	composePath := filepath.Join(f.projectDirectory, "deploy", "compose."+profile+".yaml")
+	if err := os.WriteFile(composePath, []byte("services: {}\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	contents := "profile: " + profile + "\nprojectDirectory: " + f.projectDirectory + "\nenvFile: " + f.envFile + "\n"
+	if err := os.WriteFile(f.installationPath, []byte(contents), 0o600); err != nil {
+		t.Fatal(err)
+	}
 }

 func (f cliFixture) invocations(t *testing.T) [][]string {
@@ -148,8 +148,25 @@ func (i Installation) ProjectName() string {

 // ComposeArgs builds Docker Compose arguments without shell quoting or interpolation.
 func (i Installation) ComposeArgs(command ...string) []string {
+	return i.composeArgs(i.ComposeFiles(), command...)
+}
+
+// ComposeArgsWithFinalOverride appends one validated, generated override after every durable
+// installation selector and before the Compose command.
+func (i Installation) ComposeArgsWithFinalOverride(override string, command ...string) ([]string, error) {
+	if filepath.Clean(override) != override || !filepath.IsAbs(override) {
+		return nil, errors.New("final Compose override must be an absolute canonical path")
+	}
+	if err := requireRegularFile(override, "final Compose override"); err != nil {
+		return nil, err
+	}
+	files := append(i.ComposeFiles(), override)
+	return i.composeArgs(files, command...), nil
+}
+
+func (i Installation) composeArgs(files []string, command ...string) []string {
 	args := []string{"compose", "--project-name", i.ProjectName(), "--project-directory", i.ProjectDirectory, "--env-file", i.EnvFile}
-	for _, composeFile := range i.ComposeFiles() {
+	for _, composeFile := range files {
 		args = append(args, "-f", composeFile)
 	}
 	return append(args, command...)
@@ -193,15 +210,75 @@ func (i Installation) SecretFiles() ([]string, error) {
 // EnvironmentValue returns one declared installation value without exposing dotenv parsing to
 // callers. It is used only for operator-visible file locations, never for secret content.
 func (i Installation) EnvironmentValue(name string) (string, error) {
+	values, err := i.environmentValues()
+	if err != nil {
+		return "", err
+	}
+	return values[name], nil
+}
+
+func (i Installation) environmentValues() (map[string]string, error) {
 	contents, err := safeio.ReadCanonicalRegular(i.EnvFile, maxEnvironmentFileBytes)
 	if err != nil {
-		return "", errors.New("installation environment could not be read")
+		return nil, errors.New("installation environment could not be read")
 	}
 	values, err := parseComposeDotenv(contents)
 	if err != nil {
-		return "", errors.New("installation environment could not be read")
+		return nil, errors.New("installation environment could not be read")
 	}
-	return values[name], nil
+	return values, nil
+}
+
+// PreservationPaths returns the server bind roots, backup root, and declared secret files whose
+// filesystem identities must survive a data-preserving removal.
+func (i Installation) PreservationPaths() ([]string, error) {
+	if i.Profile != "server" {
+		return nil, errors.New("data-preserving removal requires a server installation")
+	}
+	values, err := i.environmentValues()
+	if err != nil {
+		return nil, err
+	}
+	paths := make([]string, 0)
+	seen := make(map[string]struct{})
+	for _, name := range []string{
+		"THT_DATA_ROOT", "THT_PI_STATE_ROOT", "THT_WORKSPACE_REGISTRY_ROOT", "THT_BACKUP_ROOT",
+	} {
+		path := values[name]
+		if err := requireCanonicalDirectory(path); err != nil {
+			return nil, fmt.Errorf("%s must identify an existing canonical directory", name)
+		}
+		if _, exists := seen[path]; !exists {
+			paths = append(paths, path)
+			seen[path] = struct{}{}
+		}
+	}
+	secretFiles, err := i.SecretFiles()
+	if err != nil {
+		return nil, err
+	}
+	for _, path := range secretFiles {
+		if _, exists := seen[path]; !exists {
+			paths = append(paths, path)
+			seen[path] = struct{}{}
+		}
+	}
+	return paths, nil
+}
+
+func requireCanonicalDirectory(path string) error {
+	if err := safeio.ValidateCanonicalPath(path); err != nil {
+		return err
+	}
+	resolved, err := filepath.EvalSymlinks(path)
+	if err != nil || resolved != path {
+		return errors.New("directory path is unavailable or contains a symlink")
+	}
+	info, err := os.Stat(path)
+	if err != nil || !info.IsDir() {
+		return errors.New("directory path is unavailable")
+	}
+	return nil
 }

 func parseComposeDotenv(contents []byte) (map[string]string, error) {
@@ -78,6 +78,76 @@ func TestComposeArgsAutomaticallyIncludeTheInstallationCurrentImageOverride(t *t
 	}
 }

+func TestComposeArgsWithFinalOverridePreservesCurrentImagePrecedence(t *testing.T) {
+	installationPath, _, _, _ := writeInstallation(t, "server")
+	seed, err := Load(installationPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.MkdirAll(seed.ControlDirectory(), 0o700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(seed.CurrentImageOverridePath(), []byte("services: {}\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	final := filepath.Join(seed.ControlDirectory(), "migration.yaml")
+	if err := os.WriteFile(final, []byte("services: {}\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	installation, err := Load(installationPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	args, err := installation.ComposeArgsWithFinalOverride(final, "--profile", "session-migrate", "config")
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []string{"-f", installation.CurrentImageOverridePath(), "-f", final, "--profile", "session-migrate", "config"}
+	if !containsSequence(args, want) {
+		t.Fatalf("ComposeArgsWithFinalOverride() = %#v, want %#v", args, want)
+	}
+}
+
+func TestPreservationPathsReturnsCanonicalBindRootsBackupsAndSecretFiles(t *testing.T) {
+	installationPath, _, envFile, _ := writeInstallation(t, "server")
+	root := filepath.Dir(envFile)
+	var wanted []string
+	var lines []string
+	for _, item := range []struct{ key, name string }{
+		{"THT_DATA_ROOT", "data"},
+		{"THT_PI_STATE_ROOT", "pi-state"},
+		{"THT_WORKSPACE_REGISTRY_ROOT", "workspace-registry"},
+		{"THT_BACKUP_ROOT", "backups"},
+	} {
+		path := filepath.Join(root, item.name)
+		if err := os.Mkdir(path, 0o700); err != nil {
+			t.Fatal(err)
+		}
+		wanted = append(wanted, path)
+		lines = append(lines, item.key+"="+path)
+	}
+	secret := filepath.Join(root, "secret")
+	if err := os.WriteFile(secret, []byte("secret"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	wanted = append(wanted, secret)
+	lines = append(lines, "APP_TOKEN_FILE="+secret)
+	if err := os.WriteFile(envFile, []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	installation, err := Load(installationPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	got, err := installation.PreservationPaths()
+	if err != nil {
+		t.Fatal(err)
+	}
+	assertStringsEqual(t, got, wanted)
+}
+
 func TestInstallationControlPathsAreIsolatedForDescriptorsSharingOneCheckout(t *testing.T) {
 	projectDirectory := t.TempDir()
 	first := Installation{Path: filepath.Join(t.TempDir(), installationFileName), ProjectDirectory: projectDirectory}
@@ -0,0 +1,333 @@
+// Package serverops implements bounded, installation-aware server maintenance operations.
+package serverops
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/aritmolab/thothii/tools/thothctl/internal/compose"
+	"github.com/aritmolab/thothii/tools/thothctl/internal/config"
+)
+
+var (
+	ErrConfirmationRequired = errors.New("explicit confirmation is required")
+	ErrUnsafeState          = errors.New("server operation refused in the current state")
+)
+
+type Runner interface {
+	Run(context.Context, []string, io.Reader) (compose.Result, error)
+}
+
+type MigrationStatus struct {
+	Applied []string `json:"applied"`
+	Drifted []string `json:"drifted"`
+	Pending []string `json:"pending"`
+}
+
+type Container struct {
+	ID      string `json:"ID"`
+	Name    string `json:"Name"`
+	Service string `json:"Service"`
+	State   string `json:"State"`
+}
+
+type RemovalResult struct {
+	Targets   []Container
+	Preserved int
+}
+
+// MigrateSessions runs only the one-shot migration service and proves the resulting schema state.
+func MigrateSessions(ctx context.Context, installation config.Installation, runner Runner, confirmed bool) (MigrationStatus, error) {
+	if !confirmed {
+		return MigrationStatus{}, ErrConfirmationRequired
+	}
+	if installation.Profile != "server" {
+		return MigrationStatus{}, fmt.Errorf("%w: session migration requires a server installation", ErrUnsafeState)
+	}
+	containers, err := inspectContainers(ctx, installation, runner)
+	if err != nil {
+		return MigrationStatus{}, err
+	}
+	if err := requireStopped(containers); err != nil {
+		return MigrationStatus{}, err
+	}
+
+	rendered, err := runCompose(ctx, runner, installation.ComposeArgs("--profile", "session-migrate", "config", "--format", "json"))
+	if err != nil {
+		return MigrationStatus{}, err
+	}
+	coreImage, err := selectedCoreImage(rendered.Stdout)
+	if err != nil {
+		return MigrationStatus{}, err
+	}
+	override, cleanup, err := migrationOverride(installation, coreImage)
+	if err != nil {
+		return MigrationStatus{}, err
+	}
+	defer cleanup()
+
+	configArgs, err := installation.ComposeArgsWithFinalOverride(override, "--profile", "session-migrate", "config", "--format", "json")
+	if err != nil {
+		return MigrationStatus{}, err
+	}
+	finalConfig, err := runCompose(ctx, runner, configArgs)
+	if err != nil {
+		return MigrationStatus{}, err
+	}
+	if err := requireMigrationImage(finalConfig.Stdout, coreImage); err != nil {
+		return MigrationStatus{}, err
+	}
+	runArgs, err := installation.ComposeArgsWithFinalOverride(
+		override, "--profile", "session-migrate", "run", "--rm", "--no-deps", "--no-TTY", "session-migrate",
+	)
+	if err != nil {
+		return MigrationStatus{}, err
+	}
+	result, err := runCompose(ctx, runner, runArgs)
+	if err != nil {
+		return MigrationStatus{}, err
+	}
+	status, err := parseMigrationStatus(result.Stdout)
+	if err != nil {
+		return MigrationStatus{}, err
+	}
+	if len(status.Pending) != 0 || len(status.Drifted) != 0 {
+		return status, fmt.Errorf("%w: session migration did not finish cleanly", ErrUnsafeState)
+	}
+	return status, nil
+}
+
+// Remove deletes only the exact stopped core/frontend container IDs displayed by the command.
+// A nil confirmation performs inspection only; a non-nil confirmation must equal every target ID.
+func Remove(ctx context.Context, installation config.Installation, runner Runner, confirmedIDs []string) (RemovalResult, error) {
+	if installation.Profile != "server" {
+		return RemovalResult{}, fmt.Errorf("%w: removal requires a server installation", ErrUnsafeState)
+	}
+	targets, err := inspectContainers(ctx, installation, runner)
+	result := RemovalResult{Targets: targets}
+	if err != nil {
+		return result, err
+	}
+	if err := requireStopped(targets); err != nil {
+		return result, err
+	}
+	if confirmedIDs == nil {
+		return result, ErrConfirmationRequired
+	}
+	if !sameTargetIDs(targets, confirmedIDs) {
+		return result, fmt.Errorf("%w: confirmed container IDs differ from current targets", ErrUnsafeState)
+	}
+	paths, err := installation.PreservationPaths()
+	if err != nil {
+		return result, fmt.Errorf("%w: preservation paths could not be verified", ErrUnsafeState)
+	}
+	snapshots, err := snapshotPaths(paths)
+	if err != nil {
+		return result, err
+	}
+	if len(targets) > 0 {
+		args := []string{"rm"}
+		for _, target := range targets {
+			args = append(args, target.ID)
+		}
+		if _, err := runDocker(ctx, runner, args); err != nil {
+			return result, err
+		}
+	}
+	remaining, err := inspectContainers(ctx, installation, runner)
+	if err != nil {
+		return result, err
+	}
+	if len(remaining) != 0 {
+		return result, fmt.Errorf("%w: installation containers changed during removal", ErrUnsafeState)
+	}
+	if err := verifySnapshots(snapshots); err != nil {
+		return result, err
+	}
+	result.Preserved = len(snapshots)
+	return result, nil
+}
+
+func sameTargetIDs(targets []Container, confirmed []string) bool {
+	if len(targets) != len(confirmed) {
+		return false
+	}
+	wanted := make(map[string]struct{}, len(confirmed))
+	for _, id := range confirmed {
+		if strings.TrimSpace(id) == "" {
+			return false
+		}
+		if _, duplicate := wanted[id]; duplicate {
+			return false
+		}
+		wanted[id] = struct{}{}
+	}
+	for _, target := range targets {
+		if _, exists := wanted[target.ID]; !exists {
+			return false
+		}
+	}
+	return true
+}
+
+func inspectContainers(ctx context.Context, installation config.Installation, runner Runner) ([]Container, error) {
+	result, err := runCompose(ctx, runner, installation.ComposeArgs("ps", "--all", "--format", "json", "core", "frontend"))
+	if err != nil {
+		return nil, err
+	}
+	var containers []Container
+	if err := json.Unmarshal([]byte(result.Stdout), &containers); err != nil {
+		return nil, fmt.Errorf("%w: Compose returned invalid container status", ErrUnsafeState)
+	}
+	seen := make(map[string]struct{})
+	for _, container := range containers {
+		if (container.Service != "core" && container.Service != "frontend") || container.ID == "" || container.Name == "" {
+			return nil, fmt.Errorf("%w: Compose returned an unexpected removal target", ErrUnsafeState)
+		}
+		if _, exists := seen[container.ID]; exists {
+			return nil, fmt.Errorf("%w: Compose returned duplicate container IDs", ErrUnsafeState)
+		}
+		seen[container.ID] = struct{}{}
+	}
+	return containers, nil
+}
+
+func requireStopped(containers []Container) error {
+	for _, container := range containers {
+		if strings.ToLower(container.State) != "exited" {
+			return fmt.Errorf("%w: %s is not stopped", ErrUnsafeState, container.Service)
+		}
+	}
+	return nil
+}
+
+func selectedCoreImage(document string) (string, error) {
+	services, err := renderedServices(document)
+	if err != nil {
+		return "", err
+	}
+	core, exists := services["core"]
+	if !exists || strings.TrimSpace(core.Image) == "" {
+		return "", fmt.Errorf("%w: rendered core image is missing", ErrUnsafeState)
+	}
+	if _, exists := services["session-migrate"]; !exists {
+		return "", fmt.Errorf("%w: rendered migration service is missing", ErrUnsafeState)
+	}
+	return core.Image, nil
+}
+
+type renderedService struct {
+	Image string          `json:"image"`
+	Build json.RawMessage `json:"build"`
+}
+
+func renderedServices(document string) (map[string]renderedService, error) {
+	var configDocument struct {
+		Services map[string]renderedService `json:"services"`
+	}
+	if err := json.Unmarshal([]byte(document), &configDocument); err != nil {
+		return nil, fmt.Errorf("%w: Compose returned invalid rendered configuration", ErrUnsafeState)
+	}
+	return configDocument.Services, nil
+}
+
+func requireMigrationImage(document, coreImage string) error {
+	services, err := renderedServices(document)
+	if err != nil {
+		return err
+	}
+	migrator, exists := services["session-migrate"]
+	if !exists || migrator.Image != coreImage {
+		return fmt.Errorf("%w: migration image differs from selected core image", ErrUnsafeState)
+	}
+	if len(migrator.Build) != 0 && strings.TrimSpace(string(migrator.Build)) != "null" {
+		return fmt.Errorf("%w: migration service unexpectedly declares a build", ErrUnsafeState)
+	}
+	return nil
+}
+
+func migrationOverride(installation config.Installation, image string) (string, func(), error) {
+	control := installation.ControlDirectory()
+	if err := os.MkdirAll(control, 0o700); err != nil {
+		return "", func() {}, errors.New("migration control directory could not be created")
+	}
+	info, err := os.Lstat(control)
+	if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
+		return "", func() {}, errors.New("migration control directory is unsafe")
+	}
+	directory, err := os.MkdirTemp(control, "session-migrate-")
+	if err != nil {
+		return "", func() {}, errors.New("migration override directory could not be created")
+	}
+	cleanup := func() {
+		_ = os.Remove(filepath.Join(directory, "override.yaml"))
+		_ = os.Remove(directory)
+	}
+	path := filepath.Join(directory, "override.yaml")
+	contents := "services:\n  session-migrate:\n    build: !reset null\n    image: " + strconv.Quote(image) + "\n"
+	if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
+		cleanup()
+		return "", func() {}, errors.New("migration override could not be written")
+	}
+	return path, cleanup, nil
+}
+
+func parseMigrationStatus(document string) (MigrationStatus, error) {
+	var status MigrationStatus
+	decoder := json.NewDecoder(strings.NewReader(document))
+	decoder.DisallowUnknownFields()
+	if err := decoder.Decode(&status); err != nil || status.Applied == nil || status.Drifted == nil || status.Pending == nil {
+		return MigrationStatus{}, fmt.Errorf("%w: migration did not return verified JSON status", ErrUnsafeState)
+	}
+	var extra any
+	if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) {
+		return MigrationStatus{}, fmt.Errorf("%w: migration returned trailing output", ErrUnsafeState)
+	}
+	return status, nil
+}
+
+type pathSnapshot struct {
+	path string
+	info os.FileInfo
+}
+
+func snapshotPaths(paths []string) ([]pathSnapshot, error) {
+	snapshots := make([]pathSnapshot, 0, len(paths))
+	for _, path := range paths {
+		info, err := os.Stat(path)
+		if err != nil {
+			return nil, fmt.Errorf("%w: preservation target is unavailable", ErrUnsafeState)
+		}
+		snapshots = append(snapshots, pathSnapshot{path: path, info: info})
+	}
+	return snapshots, nil
+}
+
+func verifySnapshots(snapshots []pathSnapshot) error {
+	for _, snapshot := range snapshots {
+		info, err := os.Stat(snapshot.path)
+		if err != nil || !os.SameFile(snapshot.info, info) {
+			return fmt.Errorf("%w: a preserved path changed during removal", ErrUnsafeState)
+		}
+	}
+	return nil
+}
+
+func runCompose(ctx context.Context, runner Runner, args []string) (compose.Result, error) {
+	return runDocker(ctx, runner, args)
+}
+
+func runDocker(ctx context.Context, runner Runner, args []string) (compose.Result, error) {
+	result, err := runner.Run(ctx, args, nil)
+	if err != nil {
+		return result, errors.New("Docker operation failed")
+	}
+	return result, nil
+}
@@ -0,0 +1,314 @@
+package serverops
+
+import (
+	"context"
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/aritmolab/thothii/tools/thothctl/internal/compose"
+	"github.com/aritmolab/thothii/tools/thothctl/internal/config"
+)
+
+type fakeRunner struct {
+	run func(args []string) (compose.Result, error)
+	all [][]string
+}
+
+func (r *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
+	r.all = append(r.all, append([]string(nil), args...))
+	return r.run(args)
+}
+
+func TestMigrateSessionsUsesOnlyTheMigrationProfileAndSelectedCoreImage(t *testing.T) {
+	for _, image := range []string{
+		"thothii-core:local",
+		"registry.example.invalid/thothii/core@sha256:" + strings.Repeat("a", 64),
+	} {
+		t.Run(image, func(t *testing.T) {
+			installation := testInstallation(t)
+			if err := os.MkdirAll(installation.ControlDirectory(), 0o700); err != nil {
+				t.Fatal(err)
+			}
+			if err := os.WriteFile(installation.CurrentImageOverridePath(), []byte("services:\n  core:\n    image: "+image+"\n"), 0o600); err != nil {
+				t.Fatal(err)
+			}
+
+			var temporaryOverride string
+			configCalls := 0
+			runner := &fakeRunner{run: func(args []string) (compose.Result, error) {
+				switch {
+				case contains(args, "ps", "--all", "--format", "json", "core", "frontend"):
+					return compose.Result{Stdout: `[{"ID":"core-id","Name":"core-name","Service":"core","State":"exited"},{"ID":"front-id","Name":"front-name","Service":"frontend","State":"exited"}]`}, nil
+				case contains(args, "--profile", "session-migrate", "config", "--format", "json"):
+					configCalls++
+					if configCalls == 1 {
+						return compose.Result{Stdout: `{"services":{"core":{"image":"` + image + `"},"session-migrate":{"image":"thothii-core:local"}}}`}, nil
+					}
+					temporaryOverride = lastComposeFile(args)
+					contents, err := os.ReadFile(temporaryOverride)
+					if err != nil {
+						t.Fatal(err)
+					}
+					if !strings.Contains(string(contents), "image: "+strconv.Quote(image)) || !strings.Contains(string(contents), "build: !reset null") {
+						t.Fatalf("migration override = %q", contents)
+					}
+					if indexOf(args, installation.CurrentImageOverridePath()) >= indexOf(args, temporaryOverride) {
+						t.Fatalf("temporary override does not follow durable selector: %#v", args)
+					}
+					return compose.Result{Stdout: `{"services":{"core":{"image":"` + image + `"},"session-migrate":{"image":"` + image + `"}}}`}, nil
+				case contains(args, "--profile", "session-migrate", "run", "--rm", "--no-deps", "--no-TTY", "session-migrate"):
+					return compose.Result{Stdout: `{"applied":["0001"],"drifted":[],"pending":[]}` + "\n"}, nil
+				default:
+					t.Fatalf("unexpected Docker invocation: %#v", args)
+					return compose.Result{}, nil
+				}
+			}}
+
+			status, err := MigrateSessions(context.Background(), installation, runner, true)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !reflect.DeepEqual(status.Pending, []string{}) || !reflect.DeepEqual(status.Drifted, []string{}) {
+				t.Fatalf("status = %#v", status)
+			}
+			if temporaryOverride == "" {
+				t.Fatal("migration override was not inspected")
+			}
+			if _, err := os.Stat(temporaryOverride); !errors.Is(err, os.ErrNotExist) {
+				t.Fatalf("temporary override remains after migration: %v", err)
+			}
+		})
+	}
+}
+
+func TestMigrateSessionsFailsClosedBeforeMutation(t *testing.T) {
+	installation := testInstallation(t)
+	for name, spec := range map[string]struct {
+		confirmed     bool
+		ps            string
+		migrationJSON string
+	}{
+		"confirmation missing": {false, `[]`, `{"applied":[],"drifted":[],"pending":[]}`},
+		"service running":      {true, `[{"ID":"core-id","Name":"core","Service":"core","State":"running"}]`, `{"applied":[],"drifted":[],"pending":[]}`},
+		"pending migration":    {true, `[]`, `{"applied":[],"drifted":[],"pending":["0002"]}`},
+		"drifted migration":    {true, `[]`, `{"applied":[],"drifted":["0001"],"pending":[]}`},
+	} {
+		t.Run(name, func(t *testing.T) {
+			runCalled := false
+			configCalls := 0
+			runner := &fakeRunner{run: func(args []string) (compose.Result, error) {
+				switch {
+				case contains(args, "ps", "--all"):
+					return compose.Result{Stdout: spec.ps}, nil
+				case contains(args, "config", "--format", "json"):
+					configCalls++
+					return compose.Result{Stdout: `{"services":{"core":{"image":"thothii-core:local"},"session-migrate":{"image":"thothii-core:local"}}}`}, nil
+				case contains(args, "run", "--rm", "--no-deps", "--no-TTY", "session-migrate"):
+					runCalled = true
+					return compose.Result{Stdout: spec.migrationJSON}, nil
+				default:
+					t.Fatalf("unexpected Docker invocation: %#v", args)
+					return compose.Result{}, nil
+				}
+			}}
+			_, err := MigrateSessions(context.Background(), installation, runner, spec.confirmed)
+			if err == nil {
+				t.Fatal("MigrateSessions() error = nil")
+			}
+			if !spec.confirmed && len(runner.all) != 0 {
+				t.Fatalf("Docker invoked without confirmation: %#v", runner.all)
+			}
+			if strings.Contains(name, "service running") && (runCalled || configCalls != 0) {
+				t.Fatalf("migration advanced while app was running: %#v", runner.all)
+			}
+		})
+	}
+}
+
+func TestRemovePreservesEveryDeclaredBindSecretAndBackup(t *testing.T) {
+	installation, preserved := removalInstallation(t)
+	psCalls := 0
+	runner := &fakeRunner{run: func(args []string) (compose.Result, error) {
+		switch {
+		case contains(args, "ps", "--all", "--format", "json", "core", "frontend"):
+			psCalls++
+			if psCalls == 1 {
+				return compose.Result{Stdout: `[{"ID":"core-id","Name":"project-core-1","Service":"core","State":"exited"},{"ID":"frontend-id","Name":"project-frontend-1","Service":"frontend","State":"exited"}]`}, nil
+			}
+			return compose.Result{Stdout: `[]`}, nil
+		case reflect.DeepEqual(args, []string{"rm", "core-id", "frontend-id"}):
+			return compose.Result{Stdout: "core-id\nfrontend-id\n"}, nil
+		default:
+			t.Fatalf("unexpected Docker invocation: %#v", args)
+			return compose.Result{}, nil
+		}
+	}}
+
+	result, err := Remove(context.Background(), installation, runner, []string{"core-id", "frontend-id"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if result.Preserved != len(preserved) {
+		t.Fatalf("preserved = %d, want %d", result.Preserved, len(preserved))
+	}
+	if got := result.Targets; len(got) != 2 || got[0].ID != "core-id" || got[1].ID != "frontend-id" {
+		t.Fatalf("targets = %#v", got)
+	}
+	for _, args := range runner.all {
+		joined := strings.Join(args, " ")
+		if strings.Contains(joined, " -v") || strings.Contains(joined, "volume") || strings.Contains(joined, "down") || strings.Contains(joined, "prune") {
+			t.Fatalf("destructive removal invocation: %q", joined)
+		}
+	}
+	for _, path := range preserved {
+		if _, err := os.Stat(path); err != nil {
+			t.Errorf("preserved path %q: %v", path, err)
+		}
+	}
+}
+
+func TestRemoveDisplaysTargetsButDoesNotMutateWithoutConfirmation(t *testing.T) {
+	installation, _ := removalInstallation(t)
+	runner := &fakeRunner{run: func(args []string) (compose.Result, error) {
+		if !contains(args, "ps", "--all") {
+			t.Fatalf("mutation without confirmation: %#v", args)
+		}
+		return compose.Result{Stdout: `[{"ID":"core-id","Name":"project-core-1","Service":"core","State":"exited"}]`}, nil
+	}}
+	result, err := Remove(context.Background(), installation, runner, nil)
+	if !errors.Is(err, ErrConfirmationRequired) {
+		t.Fatalf("Remove() error = %v, want confirmation", err)
+	}
+	if len(result.Targets) != 1 || result.Targets[0].ID != "core-id" {
+		t.Fatalf("targets = %#v", result.Targets)
+	}
+	if len(runner.all) != 1 {
+		t.Fatalf("Docker calls = %#v", runner.all)
+	}
+}
+
+func TestRemoveRejectsRunningOrReplacedContainers(t *testing.T) {
+	for name, spec := range map[string]struct{ first, second string }{
+		"running":  {`[{"ID":"core-id","Name":"core","Service":"core","State":"running"}]`, `[]`},
+		"replaced": {`[{"ID":"core-id","Name":"core","Service":"core","State":"exited"}]`, `[{"ID":"new-id","Name":"core","Service":"core","State":"exited"}]`},
+	} {
+		t.Run(name, func(t *testing.T) {
+			installation, _ := removalInstallation(t)
+			psCalls := 0
+			runner := &fakeRunner{run: func(args []string) (compose.Result, error) {
+				if contains(args, "ps", "--all") {
+					psCalls++
+					if psCalls == 1 {
+						return compose.Result{Stdout: spec.first}, nil
+					}
+					return compose.Result{Stdout: spec.second}, nil
+				}
+				if reflect.DeepEqual(args, []string{"rm", "core-id"}) {
+					return compose.Result{}, nil
+				}
+				t.Fatalf("unexpected Docker invocation: %#v", args)
+				return compose.Result{}, nil
+			}}
+			_, err := Remove(context.Background(), installation, runner, []string{"core-id"})
+			if err == nil {
+				t.Fatal("Remove() error = nil")
+			}
+			if name == "running" && len(runner.all) != 1 {
+				t.Fatalf("running container was mutated: %#v", runner.all)
+			}
+		})
+	}
+}
+
+func TestRemoveRejectsConfirmationForDifferentContainerIDs(t *testing.T) {
+	installation, _ := removalInstallation(t)
+	runner := &fakeRunner{run: func(args []string) (compose.Result, error) {
+		if !contains(args, "ps", "--all") {
+			t.Fatalf("mismatched confirmation caused mutation: %#v", args)
+		}
+		return compose.Result{Stdout: `[{"ID":"replacement-id","Name":"core","Service":"core","State":"exited"}]`}, nil
+	}}
+	result, err := Remove(context.Background(), installation, runner, []string{"previously-displayed-id"})
+	if !errors.Is(err, ErrUnsafeState) || len(result.Targets) != 1 {
+		t.Fatalf("Remove() = %#v, %v", result, err)
+	}
+	if len(runner.all) != 1 {
+		t.Fatalf("Docker calls = %#v", runner.all)
+	}
+}
+
+func testInstallation(t *testing.T) config.Installation {
+	t.Helper()
+	root := t.TempDir()
+	project := filepath.Join(root, "project")
+	if err := os.Mkdir(project, 0o700); err != nil {
+		t.Fatal(err)
+	}
+	return config.Installation{
+		Path: filepath.Join(root, "thothii-installation.yaml"), Profile: "server",
+		ProjectDirectory: project, EnvFile: filepath.Join(root, "server.env"),
+	}
+}
+
+func removalInstallation(t *testing.T) (config.Installation, []string) {
+	t.Helper()
+	installation := testInstallation(t)
+	paths := make([]string, 0, 5)
+	values := map[string]string{}
+	for _, name := range []string{"data", "pi-state", "workspace-registry", "backups"} {
+		path := filepath.Join(filepath.Dir(installation.Path), name)
+		if err := os.Mkdir(path, 0o700); err != nil {
+			t.Fatal(err)
+		}
+		paths = append(paths, path)
+		values[name] = path
+	}
+	secret := filepath.Join(filepath.Dir(installation.Path), "secret")
+	if err := os.WriteFile(secret, []byte("never-log-this"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	paths = append(paths, secret)
+	env := "THT_DATA_ROOT=" + values["data"] + "\n" +
+		"THT_PI_STATE_ROOT=" + values["pi-state"] + "\n" +
+		"THT_WORKSPACE_REGISTRY_ROOT=" + values["workspace-registry"] + "\n" +
+		"THT_BACKUP_ROOT=" + values["backups"] + "\n" +
+		"APP_TOKEN_FILE=" + secret + "\n"
+	if err := os.WriteFile(installation.EnvFile, []byte(env), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	return installation, paths
+}
+
+func contains(values []string, sequence ...string) bool {
+	for start := range values {
+		if start+len(sequence) <= len(values) && reflect.DeepEqual(values[start:start+len(sequence)], sequence) {
+			return true
+		}
+	}
+	return false
+}
+
+func indexOf(values []string, value string) int {
+	for index, candidate := range values {
+		if candidate == value {
+			return index
+		}
+	}
+	return -1
+}
+
+func lastComposeFile(args []string) string {
+	last := ""
+	for index := 0; index+1 < len(args); index++ {
+		if args[index] == "-f" {
+			last = args[index+1]
+		}
+	}
+	return last
+}