From ece9cfda5033ffcfc4fa5ccfb5b0f48b2d9d7ab8 Mon Sep 17 00:00:00 2001 From: mptyl Date: Wed, 5 Aug 2026 15:15:06 +0200 Subject: [PATCH] fix: validate deployment rollback and server topology --- .github/workflows/deployment.yml | 5 +- PROJECT_STATE.md | 18 ++- README.md | 27 +++- deploy/compose.server.yaml | 29 +++- docs/install/server-workspace-registry.md | 18 +++ docs/install/server.md | 14 ++ scripts/prepare-server-pi-state.sh | 72 ++++++++++ scripts/task13-runtime-fixture-check.ts | 105 ++++++++++++++ scripts/test-canonical-install-compose.sh | 1 + scripts/test-server-operator-permissions.sh | 8 ++ scripts/test-server-pi-state-topology.sh | 98 +++++++++++++ scripts/test-task13-runtime-fixtures.sh | 129 ++++++++++++++++++ scripts/test-verify-workspace-install-docs.sh | 8 ++ scripts/unified-deployment-smoke.sh | 74 +++++----- scripts/verify-workspace-install-docs.sh | 2 + tools/thothctl/cmd/thothctl/main_test.go | 15 ++ 16 files changed, 571 insertions(+), 52 deletions(-) create mode 100755 scripts/prepare-server-pi-state.sh create mode 100644 scripts/task13-runtime-fixture-check.ts create mode 100755 scripts/test-server-pi-state-topology.sh create mode 100755 scripts/test-task13-runtime-fixtures.sh diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 96f8a674..d3b684f3 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -44,11 +44,14 @@ jobs: bash scripts/test-compose-secret-policy.sh bash scripts/test-no-deployment-coupling.sh bash scripts/test-verify-workspace-install-docs.sh - bash scripts/unified-deployment-smoke.sh --self-test git diff --check - name: Install backend dependencies working-directory: backend run: npm ci + - name: Verify Task 13 clean-install and runtime fixtures + run: | + bash scripts/test-server-pi-state-topology.sh + bash scripts/unified-deployment-smoke.sh --self-test - name: Test and type-check backend working-directory: backend run: | diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index 1a7dc859..fd5cb107 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -33,13 +33,17 @@ Review round 1 ran each Docker smoke exactly once without retry. Unified (`103.86s`) and update-only (`46.45s`) passed build/start, core/Pi/registry/persistence setup and the stopped candidate preflight, but `thothctl` stopped before mutation at its active-session inventory gate. - A test-first fix now scopes local inventory to `mine` and supplies the fixture's missing direct - DWH/vector/embedding runtime bindings; the final rollback path was not rerun, so compensation - and all-sentinel preservation remain unproven. Server-profile execution (`12.88s`) built both - images but Docker Desktop/VirtioFS rejected the real profile's parent Pi-state bind plus nested - tracked agent-file binds before service startup. Every run's exact labelled cleanup passed. - Native-Linux server startup and native Windows PowerShell/Docker execution remain explicit - CI/manual release gates; no local success is claimed for either platform. + Round 2 replaces presence-only fixture checks with generated Compose renders plus the production + workspace resolver; this found and fixed missing explicit direct transport selections. The + clean-server preflight now atomically initializes the three hidden Pi-agent targets under the + writable parent bind while protected/tracked sources remain separate read-only mounts. Clean + empty-root render/setup and wrong-service/value/mount mutations are green. The corrected server + one-shot built and started both healthy services from an empty Pi-state root, then stopped at an + incorrectly addressed authenticated frontend hop; the trusted-hop fixture correction is + deterministic-only. The corrected rollback one-shot passed runtime/Pi/registry/persistence and + stopped-candidate preflight, then stopped at active-session inventory before mutation. Exact + cleanup passed for both. Full server behavior, compensation/all-sentinel preservation, and + native Windows PowerShell/Docker execution remain explicit release gates. ## Portable deployment decoupling — LIVE 2026-08-05 diff --git a/README.md b/README.md index 68702c12..fcad9967 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,17 @@ contains its Pi runtime; no host `pi` executable is used. For a server installat ```sh cp deploy/env/server.env.example deploy/env/server.env # Edit all absolute storage, Pi/secret/session files, and endpoint paths. +sudo scripts/prepare-server-pi-state.sh /srv/thothii/pi-state 10001 10001 docker compose --env-file deploy/env/server.env \ -f compose.yaml -f deploy/compose.server.yaml \ -f deploy/compose.session-server.yaml.example up --build -d ``` +The initializer is required for an empty or restored server Pi-state bind. It atomically creates +the three regular targets hidden below the writable parent bind; protected Pi auth and tracked +model/settings sources remain separate read-only mounts. See the server manual before substituting +a root other than `/srv/thothii/pi-state`. + Workspace descriptors come from the Git remote configured by `THT_WORKSPACE_GIT_REMOTE`; their runtime endpoint and secret bindings remain installation-local. Open (set `THOTH_HTTP_PORT` in `deploy/env/local.env` to choose another @@ -124,15 +130,24 @@ secret files, upstream-auth checks, and a fail-closed `503` assertion for its de unavailable disposable session endpoint. No real provider, database credential, or repository secret is required. +For a clean server bind, `scripts/prepare-server-pi-state.sh` creates the hidden regular +`agent/auth.json`, `agent/models.json`, and `agent/settings.json` mount targets atomically before +Compose. The server smoke starts from an empty Pi-state root and applies this same preflight; the +real protected/tracked sources remain separate read-only mounts. Deterministic fixture tests render +both profiles, verify that bindings stay on `core`, check mount readability, and run the production +workspace resolver. Wrong-service, wrong-value, and broken-secret-mount mutations must fail. + Each public smoke has its own 30-minute process-group supervisor with TERM/KILL cleanup; CI retains an independent 32-minute outer timeout and does not retry a failed command. -Current release status (2026-08-05): deterministic contracts are green, but the complete rollback -fixture has not passed end to end after its runtime-binding correction. The one observed local -server-profile run also stopped before startup because Docker Desktop/VirtioFS rejected the -profile's parent Pi-state bind with nested tracked agent-file binds. A fresh single rollback run, -native-Linux server-profile run, and native Windows Docker Desktop/WSL2 run remain release gates; -the project does not claim those criteria green. +Current release status (2026-08-05): clean-root render/setup and the production runtime-binding +resolver contracts are green. The single corrected server-profile run proved image build, +clean-root startup, and core/frontend health, then stopped at a fixture-authenticated frontend +request; its trusted-hop headers are corrected deterministically but were not rerun. The single +corrected rollback run reached runtime/Pi/registry/persistence checks and the stopped-candidate +preflight, then stopped at active-session inventory before mutation. Full server behavior and +bad-Pi compensation with unchanged state therefore remain release gates. Native Windows Docker +Desktop/WSL2 remains a separate manual/self-hosted gate. The deterministic native Windows contract is: diff --git a/deploy/compose.server.yaml b/deploy/compose.server.yaml index 789f061c..a3a9fdf6 100644 --- a/deploy/compose.server.yaml +++ b/deploy/compose.server.yaml @@ -5,13 +5,30 @@ services: THOTH_PUBLIC_EXPOSURE: "true" THT_DATA_ROOT: /data THT_WORKSPACE_INSTALLATION_ID: server + # prepare-server-pi-state.sh creates the regular child targets before this parent bind is used. + # The real configuration sources still remain separate read-only mounts. volumes: !override - - ${THT_DATA_ROOT:?set THT_DATA_ROOT}:/data - - ${THT_PI_STATE_ROOT:?set THT_PI_STATE_ROOT}:/home/thoth/.pi - - ${PI_AUTH_FILE:?set PI_AUTH_FILE}:/home/thoth/.pi/agent/auth.json:ro - - ./deploy/pi/models.json:/home/thoth/.pi/agent/models.json:ro - - ./deploy/pi/settings.json:/home/thoth/.pi/agent/settings.json:ro - - ${THT_WORKSPACE_REGISTRY_ROOT:?set THT_WORKSPACE_REGISTRY_ROOT}:/data/workspace-registry + - type: bind + source: ${THT_DATA_ROOT:?set THT_DATA_ROOT} + target: /data + - type: bind + source: ${THT_PI_STATE_ROOT:?set THT_PI_STATE_ROOT} + target: /home/thoth/.pi + - type: bind + source: ${PI_AUTH_FILE:?set PI_AUTH_FILE} + target: /home/thoth/.pi/agent/auth.json + read_only: true + - type: bind + source: ./deploy/pi/models.json + target: /home/thoth/.pi/agent/models.json + read_only: true + - type: bind + source: ./deploy/pi/settings.json + target: /home/thoth/.pi/agent/settings.json + read_only: true + - type: bind + source: ${THT_WORKSPACE_REGISTRY_ROOT:?set THT_WORKSPACE_REGISTRY_ROOT} + target: /data/workspace-registry restart: unless-stopped frontend: diff --git a/docs/install/server-workspace-registry.md b/docs/install/server-workspace-registry.md index 8c33e8d6..b07a831e 100644 --- a/docs/install/server-workspace-registry.md +++ b/docs/install/server-workspace-registry.md @@ -18,6 +18,20 @@ first startup. Keep storage separated: /srv/thothii/operator/ # untracked operator files, setgid mode 2770 ``` +After cloning the source and before the first render/start, initialize the empty Pi-state root with +the repository setup command: + +```sh +sudo /srv/thothii/source/ThothII/scripts/prepare-server-pi-state.sh \ + /srv/thothii/pi-state 10001 10001 +``` + +The active server profile mounts that writable parent at `/home/thoth/.pi` and overlays three +read-only files beneath `agent/`. The setup command atomically creates the required hidden regular +targets with runtime ownership without copying secret or tracked file contents into writable +state. Rerun it after a restore and before Compose or `thothctl` startup; it is idempotent and does +not overwrite existing targets. + Permit outbound TCP only to approved Git/Gitea, DWH, vector, embedding, and bastion endpoints. Allow inbound traffic only from the reverse proxy/Docker network. Do not give the runtime service account Gitea administration, database-superuser rights, or a shell in the Git host. @@ -171,6 +185,10 @@ THT_WORKSPACE_BINDINGS_ENV_FILE=/srv/thothii/operator/workspace-bindings.env THT_CONNECTOR_OVERRIDE=/srv/thothii/operator/connector-secrets.server.yaml THTCTL=/srv/thothii/operator/thothctl INSTALLATION=/srv/thothii/operator/thothii-installation.yaml +sudo "$THT_SOURCE_ROOT/scripts/prepare-server-pi-state.sh" /srv/thothii/pi-state 10001 10001 +"$THT_SOURCE_ROOT/scripts/compose-with-preflight.sh" --env-file "$THT_OPERATOR_ENV" \ + -f "$THT_SOURCE_ROOT/compose.yaml" -f "$THT_SOURCE_ROOT/deploy/compose.server.yaml" \ + -f "$THT_SOURCE_ROOT/deploy/compose.session-server.yaml.example" config --quiet "$THT_SOURCE_ROOT/scripts/generate-connector-secrets-override.sh" \ --bindings-env "$THT_WORKSPACE_BINDINGS_ENV_FILE" \ --operator-env "$THT_OPERATOR_ENV" --output "$THT_CONNECTOR_OVERRIDE" diff --git a/docs/install/server.md b/docs/install/server.md index a0d0b458..ee28d427 100644 --- a/docs/install/server.md +++ b/docs/install/server.md @@ -184,8 +184,17 @@ sudo -u thothii git -c core.autocrlf=false clone \ cd /srv/thothii/source/ThothII sudo -u thothii git config --local core.autocrlf false bash scripts/verify-line-endings.sh +sudo /srv/thothii/source/ThothII/scripts/prepare-server-pi-state.sh \ + /srv/thothii/pi-state 10001 10001 ``` +The last command is a mandatory clean-install and restore preflight. The server profile bind-mounts +the writable Pi-state root and then overlays protected `auth.json` plus tracked `models.json` and +`settings.json` read-only below it. Docker requires those three hidden target files to exist under +the host parent bind before startup. The initializer creates them atomically with UID/GID 10001, +mode `0600`, rejects symlink roots or targets, and never overwrites existing contents. It is safe to rerun +after restoring `pi-state`; run it before any `thothctl start`, Compose render/start, or Pi update. + Copy the path-only server environment and installation descriptor: ```sh @@ -414,6 +423,11 @@ sudo test -d "$RESTORE/workspace-registry/repo" sudo test -d "$RESTORE/workspace-registry/snapshots" ``` +After placing the restored `pi-state` tree and before the first start, rerun +`sudo /srv/thothii/source/ThothII/scripts/prepare-server-pi-state.sh /srv/thothii/pi-state 10001 10001`. +It validates or recreates only the hidden regular mount targets; it does not alter restored Pi +state or any protected configuration source. + During the reviewed restore window, move each old tree to a timestamped sibling, move the matching restored tree into `/srv/thothii`, restore the PostgreSQL session backup from the same recovery point, and keep the proxy closed. Run `update --check-only`, `start`, `doctor`, `pi test`, registry diff --git a/scripts/prepare-server-pi-state.sh b/scripts/prepare-server-pi-state.sh new file mode 100755 index 00000000..fc039532 --- /dev/null +++ b/scripts/prepare-server-pi-state.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Prepare the nested targets required beneath the server profile's writable Pi-state parent bind. +set -euo pipefail + +fail() { + printf 'prepare-server-pi-state: %s\n' "$*" >&2 + exit 2 +} + +[[ $# -ge 1 && $# -le 3 ]] \ + || fail "usage: $0 ABSOLUTE_PI_STATE_ROOT [NUMERIC_UID [NUMERIC_GID]]" + +pi_state_root="$1" +owner="${2:-$(id -u)}" +group="${3:-$(id -g)}" +[[ "$pi_state_root" == /* && "$pi_state_root" != / && "$pi_state_root" != */ \ + && "$pi_state_root" != *//* && "$pi_state_root/" != */../* \ + && "$pi_state_root/" != */./* ]] \ + || fail "Pi-state root must be an absolute canonical non-root path" +[[ "$owner" =~ ^[0-9]+$ && "$group" =~ ^[0-9]+$ ]] \ + || fail "owner and group must be numeric" +[[ ! -L "$pi_state_root" ]] || fail "Pi-state root must not be a symlink" + +if [[ "$(id -u)" -ne 0 && ( "$owner" != "$(id -u)" || "$group" != "$(id -g)" ) ]]; then + fail "non-root execution may prepare only its own UID/GID" +fi + +ensure_directory() { + local path="$1" mode="$2" + if [[ -e "$path" && ( ! -d "$path" || -L "$path" ) ]]; then + fail "expected a real directory: $path" + fi + mkdir -p "$path" + chmod "$mode" "$path" + if [[ "$(id -u)" -eq 0 ]]; then + chown "$owner:$group" "$path" + fi +} + +ensure_target() { + local target="$1" temporary="" + if [[ -e "$target" || -L "$target" ]]; then + [[ -f "$target" && ! -L "$target" ]] || fail "expected a regular target file: $target" + else + temporary="$(mktemp "${target%/*}/.${target##*/}.XXXXXX")" + trap '[[ -z "${temporary:-}" ]] || rm -f "$temporary"' RETURN + chmod 0600 "$temporary" + if [[ "$(id -u)" -eq 0 ]]; then + chown "$owner:$group" "$temporary" + fi + if ! ln "$temporary" "$target" 2>/dev/null; then + [[ -f "$target" && ! -L "$target" ]] \ + || fail "could not atomically create target: $target" + fi + rm -f "$temporary" + temporary="" + trap - RETURN + fi + chmod 0600 "$target" + if [[ "$(id -u)" -eq 0 ]]; then + chown "$owner:$group" "$target" + fi +} + +ensure_directory "$pi_state_root" 0750 +ensure_directory "$pi_state_root/agent" 0700 +for name in auth.json models.json settings.json; do + ensure_target "$pi_state_root/agent/$name" +done + +printf 'Prepared server Pi-state targets under %s for %s:%s.\n' \ + "$pi_state_root" "$owner" "$group" diff --git a/scripts/task13-runtime-fixture-check.ts b/scripts/task13-runtime-fixture-check.ts new file mode 100644 index 00000000..fd16f367 --- /dev/null +++ b/scripts/task13-runtime-fixture-check.ts @@ -0,0 +1,105 @@ +import { constants, accessSync, readFileSync, statSync } from "node:fs"; +import { basename, dirname, join } from "node:path"; +import { createRequire } from "node:module"; +import { resolveRuntimeBindings } from "../backend/src/workspaces/bindings.js"; +import { renderRuntimeConfig } from "../backend/src/workspaces/runtime-renderer.js"; + +const requireFromBackend = createRequire(new URL("../backend/package.json", import.meta.url)); +const { parse } = requireFromBackend("yaml") as { parse: (value: string) => any }; + +const [renderedPath, workspacePath, profile] = process.argv.slice(2); +if (!renderedPath || !workspacePath || (profile !== "local" && profile !== "server")) { + throw new Error("usage: task13-runtime-fixture-check RENDERED_JSON WORKSPACE_YAML local|server"); +} + +const config = JSON.parse(readFileSync(renderedPath, "utf8")); +const workspace = parse(readFileSync(workspacePath, "utf8")); +const core = config.services?.core; +const frontend = config.services?.frontend; +if (!core || !frontend) throw new Error("fixture render must contain core and frontend"); + +const expected = { + THT_WS_TASK13_SMOKE_DWH_TRANSPORT: "postgres_direct", + THT_WS_TASK13_SMOKE_DWH_HOST: "dwh.task13.invalid", + THT_WS_TASK13_SMOKE_DWH_PORT: "5432", + THT_WS_TASK13_SMOKE_DWH_USER: "task13_reader", + THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE: "/run/secrets/thothii.secrets", + THT_WS_TASK13_SMOKE_VECTOR_TRANSPORT: "pgvector_direct", + THT_WS_TASK13_SMOKE_VECTOR_HOST: "vector.task13.invalid", + THT_WS_TASK13_SMOKE_VECTOR_PORT: "5432", + THT_WS_TASK13_SMOKE_VECTOR_USER: "task13_vector_reader", + THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE: "/run/secrets/thothii.secrets", + THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL: profile === "local" + ? `http://${config.name}-llm:9000` + : "https://embedding.task13.invalid", +}; +for (const [name, value] of Object.entries(expected)) { + if (core.environment?.[name] !== value) { + throw new Error(`core runtime binding ${name} is ${JSON.stringify(core.environment?.[name])}, want ${JSON.stringify(value)}`); + } + if (Object.hasOwn(frontend.environment || {}, name)) { + throw new Error(`runtime binding escaped to frontend: ${name}`); + } +} + +const bundle = config.secrets?.thothii_secrets; +const bundleSource = bundle?.file; +if (typeof bundleSource !== "string" || !statSync(bundleSource).isFile()) { + throw new Error("fixture secret bundle source is not a regular file"); +} +accessSync(bundleSource, constants.R_OK); +const coreBundle = (core.secrets || []).filter( + (secret: any) => secret.source === "thothii_secrets" && secret.target === "thothii.secrets", +); +if (coreBundle.length !== 1) throw new Error("core lacks exactly one runtime secret bundle mount"); +if ((frontend.secrets || []).length !== 0) throw new Error("frontend received a runtime secret"); + +const mounts = core.volumes || []; +for (const target of [ + "/home/thoth/.pi/agent/auth.json", + "/home/thoth/.pi/agent/models.json", + "/home/thoth/.pi/agent/settings.json", +]) { + const selected = mounts.filter((mount: any) => mount.target === target); + if (selected.length !== 1 || selected[0].type !== "bind" || !selected[0].read_only) { + throw new Error(`Pi fixture mount is not one read-only bind: ${target}`); + } + accessSync(selected[0].source, constants.R_OK); + if (profile === "server") { + const parent = mounts.find((mount: any) => mount.target === "/home/thoth/.pi"); + const hidden = join(parent.source, "agent", basename(target)); + if (!statSync(hidden).isFile()) throw new Error(`server parent root lacks ${hidden}`); + } +} + +const resolverEnvironment = { ...core.environment }; +resolverEnvironment.THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE = bundleSource; +resolverEnvironment.THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE = bundleSource; +const bindings = resolveRuntimeBindings(workspace, resolverEnvironment, [dirname(bundleSource)]); +for (const [role, binding] of Object.entries(bindings)) { + if ((binding as any).missing.length !== 0) { + throw new Error(`workspace resolver reports missing ${role} bindings: ${(binding as any).missing.join(",")}`); + } +} +const runtime = parse(renderRuntimeConfig(workspace, bindings, { + sessions: "/data/sessions", + artifacts: "/data/artifacts", + indexes: "/data/indexes", +})); +if (runtime.database.host !== expected.THT_WS_TASK13_SMOKE_DWH_HOST + || runtime.database.user !== expected.THT_WS_TASK13_SMOKE_DWH_USER + || runtime.database.password_file !== bundleSource) { + throw new Error("workspace resolver produced the wrong DWH runtime"); +} +if (runtime.vector_db.host !== expected.THT_WS_TASK13_SMOKE_VECTOR_HOST + || runtime.vector_db.user !== expected.THT_WS_TASK13_SMOKE_VECTOR_USER + || runtime.vector_db.password_file !== bundleSource) { + throw new Error("workspace resolver produced the wrong vector runtime"); +} +if (runtime.embeddings.base_url !== expected.THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL) { + throw new Error("workspace resolver produced the wrong embedding runtime"); +} +const secret = readFileSync(bundleSource, "utf8").trim(); +if (JSON.stringify(config).includes(secret) || JSON.stringify(runtime).includes(secret)) { + throw new Error("fixture render or resolver output leaked secret content"); +} diff --git a/scripts/test-canonical-install-compose.sh b/scripts/test-canonical-install-compose.sh index e516bf30..e44cd1ac 100755 --- a/scripts/test-canonical-install-compose.sh +++ b/scripts/test-canonical-install-compose.sh @@ -21,6 +21,7 @@ printf '%s\n' '{}' >"$tmp/pi-auth.json" printf '%s\n' 'THT_MODEL_API_KEY=fixture-model-api-key' >"$tmp/thothii.secrets" chmod 0600 "$tmp/pi-auth.json" "$tmp/thothii.secrets" mkdir -p "$tmp/data" "$tmp/pi-state" "$tmp/workspace-registry" +"$root/scripts/prepare-server-pi-state.sh" "$tmp/pi-state" "$(id -u)" "$(id -g)" >/dev/null printf '%s\n' 'fixture-session-password' >"$tmp/session-runtime-password" printf '%s\n' 'fixture-session-migrator-password' >"$tmp/session-migrator-password" printf '%s\n' 'fixture-session-ca' >"$tmp/session-ca.pem" diff --git a/scripts/test-server-operator-permissions.sh b/scripts/test-server-operator-permissions.sh index 4b8ab817..97da1d57 100755 --- a/scripts/test-server-operator-permissions.sh +++ b/scripts/test-server-operator-permissions.sh @@ -23,6 +23,8 @@ install -d -o 10001 -g 10001 -m 0750 /srv/thothii/data /srv/thothii/pi-state /sr install -d -o 10001 -g 20002 -m 2750 /srv/thothii/source/ThothII /srv/thothii/source/ThothII/scripts install -o 10001 -g 20002 -m 0750 /repository/scripts/build-thothctl.sh /srv/thothii/source/ThothII/scripts/build-thothctl.sh install -o 10001 -g 20002 -m 0750 /repository/scripts/generate-connector-secrets-override.sh /srv/thothii/source/ThothII/scripts/generate-connector-secrets-override.sh +install -o 10001 -g 20002 -m 0750 /repository/scripts/prepare-server-pi-state.sh /srv/thothii/source/ThothII/scripts/prepare-server-pi-state.sh +/srv/thothii/source/ThothII/scripts/prepare-server-pi-state.sh /srv/thothii/pi-state 10001 10001 printf "%s\n" "PLACEHOLDER=replace-me" "THT_WS_TEST_DWH_PASSWORD_SOURCE=/srv/thothii/secrets/dwh-password" > /srv/thothii/operator/server.env printf "%s\n" "projectDirectory: replace-me" > /srv/thothii/operator/thothii-installation.yaml @@ -77,6 +79,12 @@ test "$(stat -c %u:%g /srv/thothii/operator/connector-secrets.server.yaml)" = 20 test "$(stat -c %a /srv/thothii/operator/connector-secrets.server.yaml)" = 660 test "$(stat -c %u:%g /srv/thothii)" = 10001:20002 test "$(stat -c %a /srv/thothii)" = 2750 +test "$(stat -c %u:%g /srv/thothii/pi-state/agent)" = 10001:10001 +test "$(stat -c %a /srv/thothii/pi-state/agent)" = 700 +for target in auth.json models.json settings.json; do + test "$(stat -c %u:%g /srv/thothii/pi-state/agent/$target)" = 10001:10001 + test "$(stat -c %a /srv/thothii/pi-state/agent/$target)" = 600 +done test "$(stat -c %u:%g /srv/thothii/operator/build-output/thothctl-linux-amd64)" = 20001:20002 test "$(stat -c %a /srv/thothii/operator/build-output/thothctl-linux-amd64)" = 750 test -f /srv/thothii/operator/start.marker diff --git a/scripts/test-server-pi-state-topology.sh b/scripts/test-server-pi-state-topology.sh new file mode 100755 index 00000000..59f07e22 --- /dev/null +++ b/scripts/test-server-pi-state-topology.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# Clean-install contract for the server Pi-state parent bind and its read-only child mounts. +set -euo pipefail + +root="$(cd "$(dirname "$0")/.." && pwd -P)" +tmp_parent="${TMPDIR:-/tmp}" +tmp_parent="${tmp_parent%/}" +fixture="$(mktemp -d "$tmp_parent/thoth-server-pi-state.XXXXXX")" +trap 'rm -rf "$fixture"' EXIT HUP INT TERM + +pi_state="$fixture/empty pi state" +mkdir -p "$pi_state" +"$root/scripts/prepare-server-pi-state.sh" "$pi_state" "$(id -u)" "$(id -g)" + +for target in auth.json models.json settings.json; do + path="$pi_state/agent/$target" + [[ -f "$path" && ! -L "$path" ]] || { + echo "server Pi-state initializer did not create regular target: $target" >&2 + exit 1 + } +done + +printf '%s\n' preserved-placeholder >"$pi_state/agent/models.json" +"$root/scripts/prepare-server-pi-state.sh" "$pi_state" "$(id -u)" "$(id -g)" +[[ "$(cat "$pi_state/agent/models.json")" == preserved-placeholder ]] || { + echo "server Pi-state initializer overwrote an existing target" >&2 + exit 1 +} + +printf '{}\n' >"$fixture/pi-auth.json" +printf 'THT_MODEL_API_KEY=fixture-model-key\n' >"$fixture/thothii.secrets" +printf 'fixture-session-password\n' >"$fixture/session-runtime-password" +printf 'fixture-session-migrator-password\n' >"$fixture/session-migrator-password" +printf 'fixture-session-ca\n' >"$fixture/session-ca.pem" +cp "$root/deploy/workspaces/server-sessions.yaml.example" "$fixture/server-sessions.yaml" +chmod 0600 "$fixture"/*.json "$fixture"/*.secrets "$fixture"/*password "$fixture"/*.pem + +cat >"$fixture/server.env" <"$fixture/rendered.json" + +node - "$fixture/rendered.json" "$pi_state" "$fixture/pi-auth.json" <<'NODE' +const fs = require("fs"); +const path = require("path"); + +const [renderedPath, piState, authSource] = process.argv.slice(2); +const config = JSON.parse(fs.readFileSync(renderedPath, "utf8")); +const core = config.services?.core; +if (!core) throw new Error("server render lacks core"); +const mounts = core.volumes || []; +const parent = mounts.find((mount) => mount.target === "/home/thoth/.pi"); +if (!parent || parent.type !== "bind" || parent.source !== piState || parent.read_only) { + throw new Error("server Pi-state parent bind is not the expected writable root"); +} +const children = new Map(mounts + .filter((mount) => mount.target?.startsWith("/home/thoth/.pi/agent/")) + .map((mount) => [path.basename(mount.target), mount])); +for (const name of ["auth.json", "models.json", "settings.json"]) { + const mount = children.get(name); + if (!mount || mount.type !== "bind" || !mount.read_only) { + throw new Error(`server Pi agent child is not one read-only bind: ${name}`); + } + const hiddenTarget = path.join(piState, "agent", name); + if (!fs.statSync(hiddenTarget).isFile()) { + throw new Error(`server Pi-state root lacks nested target: ${name}`); + } +} +if (children.get("auth.json").source !== authSource) { + throw new Error("server Pi auth source changed while preparing nested targets"); +} +if (JSON.stringify(config).includes("fixture-model-key")) { + throw new Error("server render leaked a secret value"); +} +NODE + +echo "clean empty-root server Pi-state render contract passed." diff --git a/scripts/test-task13-runtime-fixtures.sh b/scripts/test-task13-runtime-fixtures.sh new file mode 100755 index 00000000..0f44b1ca --- /dev/null +++ b/scripts/test-task13-runtime-fixtures.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Generate Task 13 fixtures and validate rendered bindings with the production workspace resolver. +set -euo pipefail + +profile="${1:-}" +[[ "$profile" == local || "$profile" == server ]] || { + echo "usage: $0 local|server" >&2 + exit 2 +} +root="$(cd "$(dirname "$0")/.." && pwd -P)" +tmp_parent="${TMPDIR:-/tmp}" +tmp_parent="${tmp_parent%/}" +fixture="$(mktemp -d "$tmp_parent/thoth-task13-runtime-$profile.XXXXXX")" +trap 'rm -rf "$fixture"' EXIT HUP INT TERM + +# shellcheck source=./unified-deployment-smoke.sh +source "$root/scripts/unified-deployment-smoke.sh" +TASK13_ROOT="$root" +TASK13_TMP="$fixture" +TASK13_RUN_ID="fixture-$profile" +TASK13_PROJECT="thothii-task13-$profile" +TASK13_CORE_IMAGE="task13-core-$profile:fixture" +TASK13_FRONTEND_IMAGE="task13-frontend-$profile:fixture" +TASK13_SECRET_VALUE="task13-runtime-secret-$profile" +TASK13_BRANCH=main +TASK13_ENV_FILE="$fixture/operator.env" +TASK13_OVERRIDE="$fixture/compose.task13.yaml" +TASK13_LOG="$fixture/task13.log" +: >"$TASK13_LOG" +TASK13_INSTALLATION="$fixture/thothii-installation.yaml" +TASK13_PI_AUTH="$fixture/pi-auth.json" +TASK13_SECRETS="$fixture/thothii.secrets" +TASK13_PI_MODELS="$fixture/models.json" +TASK13_PI_SETTINGS="$fixture/settings.json" +TASK13_LLM_SERVER="$fixture/fake-llm.mjs" +TASK13_LLM_CONTAINER="$TASK13_PROJECT-llm" +TASK13_REMOTE="$fixture/remote.git" +mkdir -p "$TASK13_REMOTE" + +workspace="$fixture/task13-smoke.yaml" +cat >"$workspace" <<'EOF' +workspace: + schema_version: 2 + id: task13-smoke + name: Task 13 Smoke + language: en +dwh: + engine: postgres + database: warehouse + schema: analytics + supported_transports: [postgres_direct] +semantic_index: + vector_store: + engine: pgvector + database: vectors + schema: public + collection: task13_documents + dimensions: 8 + distance: cosine + supported_transports: [pgvector_direct] + embedding: + provider: ollama_compatible + model: task13-embedding + dimensions: 8 +llm_policy: + default: local-qwen/task13-smoke + allowed: [local-qwen/task13-smoke] +EOF + +if [[ "$profile" == local ]]; then + task13_write_fixture_files + task13_write_environment /fixtures/remote.git + compose_files=(-f "$root/compose.yaml" -f "$root/deploy/compose.local.yaml" -f "$TASK13_OVERRIDE") +else + TASK13_SERVER_DATA="$fixture/Server Data" + TASK13_SERVER_PI_STATE="$fixture/Server Pi State" + TASK13_SERVER_REGISTRY="$fixture/Server Registry" + TASK13_SERVER_WORKSPACE_CONFIG="$fixture/server-sessions.yaml" + TASK13_SESSION_RUNTIME_PASSWORD="$fixture/session-runtime-password" + TASK13_SESSION_MIGRATOR_PASSWORD_FILE="$fixture/session-migrator-password" + TASK13_SESSION_CA="$fixture/session-ca.pem" + TASK13_SESSION_PASSWORD="task13-runtime-$profile" + TASK13_SESSION_MIGRATOR_PASSWORD="task13-migrator-$profile" + task13_write_server_fixture_files + compose_files=( + -f "$root/compose.yaml" + -f "$root/deploy/compose.server.yaml" + -f "$root/deploy/compose.session-server.yaml.example" + -f "$TASK13_OVERRIDE" + ) +fi + +rendered="$fixture/rendered.json" +docker compose --project-name "$TASK13_PROJECT" --project-directory "$root" \ + --env-file "$TASK13_ENV_FILE" "${compose_files[@]}" config --format json >"$rendered" +tsx_loader="$root/backend/node_modules/tsx/dist/loader.mjs" +checker=(node --import "$tsx_loader" "$root/scripts/task13-runtime-fixture-check.ts") +[[ -f "$tsx_loader" ]] || { + echo "backend dependencies are required for the Task 13 runtime fixture contract" >&2 + exit 2 +} +"${checker[@]}" "$rendered" "$workspace" "$profile" + +for mutation in wrong-service wrong-value wrong-secret-mount; do + mutated="$fixture/$mutation.json" + node - "$rendered" "$mutated" "$mutation" <<'NODE' +const fs = require("fs"); +const [source, destination, mutation] = process.argv.slice(2); +const config = JSON.parse(fs.readFileSync(source, "utf8")); +if (mutation === "wrong-service") { + const name = "THT_WS_TASK13_SMOKE_DWH_HOST"; + config.services.frontend.environment ||= {}; + config.services.frontend.environment[name] = config.services.core.environment[name]; + delete config.services.core.environment[name]; +} else if (mutation === "wrong-value") { + config.services.core.environment.THT_WS_TASK13_SMOKE_DWH_HOST = "wrong.task13.invalid"; +} else { + config.secrets.thothii_secrets.file = source + ".missing"; +} +fs.writeFileSync(destination, JSON.stringify(config)); +NODE + if "${checker[@]}" "$mutated" "$workspace" "$profile" \ + >"$fixture/$mutation.out" 2>"$fixture/$mutation.err"; then + echo "runtime fixture checker accepted mutation: $mutation" >&2 + exit 1 + fi +done + +echo "Task 13 $profile rendered runtime fixture contract passed." diff --git a/scripts/test-verify-workspace-install-docs.sh b/scripts/test-verify-workspace-install-docs.sh index 92e96cd0..8be2621a 100755 --- a/scripts/test-verify-workspace-install-docs.sh +++ b/scripts/test-verify-workspace-install-docs.sh @@ -36,6 +36,14 @@ for fixture in \ done server_guide="$root/docs/install/server.md" +grep -Fq 'scripts/prepare-server-pi-state.sh /srv/thothii/pi-state 10001 10001' "$server_guide" || { + echo "server guide does not initialize nested Pi-state targets before Compose" >&2 + exit 1 +} +grep -Fq 'prepare-server-pi-state.sh' "$root/docs/install/server-workspace-registry.md" || { + echo "server workspace-registry guide omits the Pi-state clean-install precondition" >&2 + exit 1 +} grep -Eq '^sudo install -d -o 10001 -g thothii-ops -m 2750 /srv/thothii$' "$server_guide" || { echo "server operations guide does not set the parent traversal boundary" >&2 exit 1 diff --git a/scripts/unified-deployment-smoke.sh b/scripts/unified-deployment-smoke.sh index 23c488db..8b17d8e4 100755 --- a/scripts/unified-deployment-smoke.sh +++ b/scripts/unified-deployment-smoke.sh @@ -279,10 +279,12 @@ services: PI_THINKING: low THT_WORKSPACE_INSTALLATION_ID: task13-smoke THT_WORKSPACE_REGISTRY_ROOT: /data/workspace-registry + THT_WS_TASK13_SMOKE_DWH_TRANSPORT: postgres_direct THT_WS_TASK13_SMOKE_DWH_HOST: dwh.task13.invalid THT_WS_TASK13_SMOKE_DWH_PORT: "5432" THT_WS_TASK13_SMOKE_DWH_USER: task13_reader THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE: /run/secrets/thothii.secrets + THT_WS_TASK13_SMOKE_VECTOR_TRANSPORT: pgvector_direct THT_WS_TASK13_SMOKE_VECTOR_HOST: vector.task13.invalid THT_WS_TASK13_SMOKE_VECTOR_PORT: "5432" THT_WS_TASK13_SMOKE_VECTOR_USER: task13_vector_reader @@ -372,7 +374,10 @@ EOF chmod 0600 "$TASK13_SERVER_WORKSPACE_CONFIG" mkdir -p "$TASK13_SERVER_DATA" "$TASK13_SERVER_PI_STATE" "$TASK13_SERVER_REGISTRY" - chmod 0777 "$TASK13_SERVER_DATA" "$TASK13_SERVER_PI_STATE" "$TASK13_SERVER_REGISTRY" + "$TASK13_ROOT/scripts/prepare-server-pi-state.sh" \ + "$TASK13_SERVER_PI_STATE" "$(id -u)" "$(id -g)" >>"$TASK13_LOG" + chmod 0777 "$TASK13_SERVER_DATA" "$TASK13_SERVER_PI_STATE" \ + "$TASK13_SERVER_PI_STATE/agent" "$TASK13_SERVER_REGISTRY" data_root="$TASK13_SERVER_DATA" pi_root="$TASK13_SERVER_PI_STATE" registry_root="$TASK13_SERVER_REGISTRY" @@ -387,10 +392,12 @@ services: labels: io.thothii.task13.run: "$TASK13_RUN_ID" environment: + THT_WS_TASK13_SMOKE_DWH_TRANSPORT: postgres_direct THT_WS_TASK13_SMOKE_DWH_HOST: dwh.task13.invalid THT_WS_TASK13_SMOKE_DWH_PORT: "5432" THT_WS_TASK13_SMOKE_DWH_USER: task13_reader THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE: /run/secrets/thothii.secrets + THT_WS_TASK13_SMOKE_VECTOR_TRANSPORT: pgvector_direct THT_WS_TASK13_SMOKE_VECTOR_HOST: vector.task13.invalid THT_WS_TASK13_SMOKE_VECTOR_PORT: "5432" THT_WS_TASK13_SMOKE_VECTOR_USER: task13_vector_reader @@ -602,6 +609,14 @@ task13_assert_runtime() { task13_run_logged "thothctl Pi doctor" "$TASK13_THOTHCTL" --installation "$TASK13_INSTALLATION" pi doctor } +task13_server_auth_headers() { + TASK13_SERVER_AUTH_HEADERS=( + -H 'x-thoth-trusted-principal-issuer: task13-proxy' + -H 'x-thoth-trusted-principal-subject: task13-user' + -H 'x-thoth-trusted-principal-display-name: Task 13 User' + ) +} + task13_assert_server_runtime() { local frontend unauthenticated authenticated session_status core_id frontend_id local expected_core_image expected_frontend_image @@ -641,11 +656,10 @@ task13_assert_server_runtime() { "http://$frontend/api/workspaces")" [[ "$unauthenticated" == 401 ]] || task13_fail "server profile did not enforce upstream auth" authenticated="$TASK13_TMP/server-workspaces.out" + task13_server_auth_headers curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ --fail --silent --show-error \ - -H 'x-thoth-principal-issuer: task13-proxy' \ - -H 'x-thoth-principal-subject: task13-user' \ - -H 'x-thoth-principal-display-name: Task 13 User' \ + "${TASK13_SERVER_AUTH_HEADERS[@]}" \ "http://$frontend/api/workspaces" >"$authenticated" grep -Fq 'Task 13 Smoke' "$authenticated" \ || task13_fail "authenticated server route did not expose the disposable registry" @@ -653,8 +667,7 @@ task13_assert_server_runtime() { session_status="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \ --max-time "$TASK13_CURL_MAX_TIME" --silent --output "$TASK13_TMP/server-sessions.out" \ --write-out '%{http_code}' \ - -H 'x-thoth-principal-issuer: task13-proxy' \ - -H 'x-thoth-principal-subject: task13-user' \ + "${TASK13_SERVER_AUTH_HEADERS[@]}" \ "http://$frontend/api/sessions")" [[ "$session_status" == 503 ]] \ || task13_fail "disposable unavailable session dependency did not fail closed with 503" @@ -1162,35 +1175,15 @@ task13_self_test_rollback_fixture_contract() { } task13_self_test_runtime_binding_fixture() { - local fixture_source - fixture_source="$(declare -f task13_write_fixture_files)" - for variable in \ - THT_WS_TASK13_SMOKE_DWH_HOST \ - THT_WS_TASK13_SMOKE_DWH_PORT \ - THT_WS_TASK13_SMOKE_DWH_USER \ - THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE \ - THT_WS_TASK13_SMOKE_VECTOR_HOST \ - THT_WS_TASK13_SMOKE_VECTOR_PORT \ - THT_WS_TASK13_SMOKE_VECTOR_USER \ - THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE \ - THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL; do - grep -Fq "$variable" <<<"$fixture_source" \ - || task13_fail "rollback fixture lacks runtime binding: $variable" - done + local root + root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" + "$root/scripts/test-task13-runtime-fixtures.sh" local } task13_self_test_server_runtime_binding_fixture() { - local fixture_source - fixture_source="$(declare -f task13_write_server_fixture_files)" - for variable in \ - THT_WS_TASK13_SMOKE_DWH_HOST \ - THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE \ - THT_WS_TASK13_SMOKE_VECTOR_HOST \ - THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE \ - THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL; do - grep -Fq "$variable" <<<"$fixture_source" \ - || task13_fail "server fixture lacks runtime binding: $variable" - done + local root + root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" + "$root/scripts/test-task13-runtime-fixtures.sh" server } task13_self_test_stopped_project_containers() { @@ -1325,6 +1318,21 @@ task13_self_test_server_release_contract() { || task13_fail "workflow lacks an outer timeout for the Linux server smoke" } +task13_self_test_server_auth_hop_contract() { + local joined + task13_server_auth_headers + joined="${TASK13_SERVER_AUTH_HEADERS[*]}" + for header in \ + x-thoth-trusted-principal-issuer \ + x-thoth-trusted-principal-subject \ + x-thoth-trusted-principal-display-name; do + [[ "$joined" == *"$header:"* ]] \ + || task13_fail "server smoke omits trusted frontend hop header: $header" + done + [[ "$joined" != *'x-thoth-principal-issuer:'* ]] \ + || task13_fail "server smoke sends public identity headers to the frontend hop" +} + task13_self_test_source_contract() { local root host_network push_command registry_function workflow uses_count pinned_uses_count root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" @@ -1385,6 +1393,7 @@ task13_self_test() { task13_self_test_public_timeout_contract task13_self_test_windows_release_contract task13_self_test_server_release_contract + task13_self_test_server_auth_hop_contract task13_self_test_source_contract printf 'Task 13 smoke safety contracts passed.\n' } @@ -1400,6 +1409,7 @@ task13_self_test_case() { timeout-public) task13_self_test_public_timeout_contract ;; windows) task13_self_test_windows_release_contract ;; server) task13_self_test_server_release_contract ;; + server-auth) task13_self_test_server_auth_hop_contract ;; *) task13_fail "unknown Task 13 self-test case: $1" ;; esac } diff --git a/scripts/verify-workspace-install-docs.sh b/scripts/verify-workspace-install-docs.sh index 35435519..a936d516 100755 --- a/scripts/verify-workspace-install-docs.sh +++ b/scripts/verify-workspace-install-docs.sh @@ -1158,6 +1158,8 @@ verify_server_installation_example() { backup_root="$fixture/server backups" mkdir -p "$source_copy/deploy/pi" "$source_copy/deploy/workspaces" \ "$operator_dir/data" "$operator_dir/pi-state" "$operator_dir/workspace-registry" "$backup_root" + "$root/scripts/prepare-server-pi-state.sh" \ + "$operator_dir/pi-state" "$(id -u)" "$(id -g)" >/dev/null cp "$root/compose.yaml" "$source_copy/compose.yaml" cp "$root/deploy/compose.server.yaml" "$source_copy/deploy/compose.server.yaml" cp "$root/deploy/compose.session-server.yaml.example" \ diff --git a/tools/thothctl/cmd/thothctl/main_test.go b/tools/thothctl/cmd/thothctl/main_test.go index 5ecf540a..630abc1d 100644 --- a/tools/thothctl/cmd/thothctl/main_test.go +++ b/tools/thothctl/cmd/thothctl/main_test.go @@ -17,6 +17,21 @@ import ( "github.com/aritmolab/thothii/tools/thothctl/internal/testsupport" ) +func TestInstallationRunnerMapsProfileToSessionInventoryScope(t *testing.T) { + for _, test := range []struct { + profile string + want string + }{ + {profile: "local", want: "mine"}, + {profile: "server", want: "all"}, + } { + runner := installationRunner{installation: config.Installation{Profile: test.profile}} + if got := runner.SessionInventoryScope(); got != test.want { + t.Fatalf("profile %q maps to session scope %q, want %q", test.profile, got, test.want) + } + } +} + // Catches interactive configuration prompts that use retired model-only data instead of the // provider, model, and reasoning choices supplied by the dedicated Pi Management API. func TestResolvePiConfigureUsesNumberedClosedChoicesOnlyForTTY(t *testing.T) {