diff --git a/backend/src/operator-command.ts b/backend/src/operator-command.ts index 326ca2fd..30648074 100644 --- a/backend/src/operator-command.ts +++ b/backend/src/operator-command.ts @@ -18,7 +18,8 @@ import { WorkspaceRegistry } from "./workspaces/registry.js"; import { WorkspaceSecretStore } from "./workspaces/secret-store.js"; type OperatorAction = "maintenance-activate" | "maintenance-deactivate" | "maintenance-status" - | "session-inventory" | "workflow-doctor" | "pi-options" | "pi-test" | "effective-settings"; + | "session-inventory" | "workflow-doctor" | "workspace-integrity" + | "pi-options" | "pi-test" | "effective-settings"; const lifecyclePrincipal: PrincipalContext = { issuer: "tht-operator-command", @@ -86,6 +87,15 @@ async function workflowDiagnostics(config: AppConfig): Promise<{ ready: true; wo return { ready: true, workspaces: revisions.length }; } +async function workspaceIntegrity(config: AppConfig): Promise<{ + ready: true; + state: "uninitialized" | "active"; + workspaces: number; +}> { + const integrity = await new WorkspaceRegistry(config.workspaceRegistry).verifyStoredState(); + return { ready: true, ...integrity }; +} + export async function runOperatorAction( action: OperatorAction, config: AppConfig, @@ -98,6 +108,7 @@ export async function runOperatorAction( } if (action === "session-inventory") return await sessionInventory(config); if (action === "workflow-doctor") return await workflowDiagnostics(config); + if (action === "workspace-integrity") return await workspaceIntegrity(config); if (action === "effective-settings") return effectiveSettings(config, loadSettings(config)); const service = createPiManagement(config, { listModels: createPiModelLister(config) }); if (action === "pi-options") return await service.options(); @@ -109,7 +120,7 @@ async function main(): Promise { const action = process.argv[2] as OperatorAction | undefined; if (!action || ![ "maintenance-activate", "maintenance-deactivate", "maintenance-status", "session-inventory", - "workflow-doctor", "pi-options", "pi-test", "effective-settings", + "workflow-doctor", "workspace-integrity", "pi-options", "pi-test", "effective-settings", ].includes(action)) throw new Error("invalid operator action"); const result = await runOperatorAction(action, loadConfig(process.env)); process.stdout.write(`${JSON.stringify(result)}\n`); diff --git a/backend/src/workspaces/registry.ts b/backend/src/workspaces/registry.ts index ea466ff9..cacd72e4 100644 --- a/backend/src/workspaces/registry.ts +++ b/backend/src/workspaces/registry.ts @@ -31,6 +31,11 @@ export interface WorkspaceRevision { snapshotPath: string; } +export interface StoredWorkspaceIntegrity { + state: "uninitialized" | "active"; + workspaces: number; +} + export interface SessionRevisionLease { workspace: WorkspaceDescriptor; revision: WorkspaceRevision; @@ -180,6 +185,48 @@ export class WorkspaceRegistry { }); } + /** + * Validate only persisted local registry state. Restore uses this path while the installation + * is stopped: it must neither contact Git nor turn a never-used registry into initialized state. + */ + async verifyStoredState(): Promise { + await this.repository.ensureLayout(); + return await this.lock.run(async () => { + try { + const stateEntries = await readdir(this.repository.statePath, { withFileTypes: true }); + if (stateEntries.some((entry) => ( + entry.name !== "active.json" || !entry.isFile() || entry.isSymbolicLink() + ))) throw new Error("workspace state directory is partial"); + + const snapshotEntries = await readdir(this.repository.snapshotsPath, { withFileTypes: true }); + for (const entry of snapshotEntries) { + const isRuntime = entry.name === "runtime"; + const isSnapshot = /^[0-9a-f]{40}$/.test(entry.name); + if ((!isRuntime && !isSnapshot) || !entry.isDirectory() || entry.isSymbolicLink()) { + throw new Error("workspace snapshot directory is partial"); + } + } + + const hasActiveState = stateEntries.length === 1; + if (!hasActiveState) { + if (snapshotEntries.some((entry) => entry.name !== "runtime") || this.storedPathExists(this.repository.repoPath)) { + throw new Error("workspace registry is partially initialized"); + } + return { state: "uninitialized", workspaces: 0 }; + } + + const active = await this.activeState(); + for (const entry of snapshotEntries) { + if (entry.name === "runtime" || entry.name === active.head) continue; + await this.snapshotState(entry.name); + } + return { state: "active", workspaces: active.revisions.length }; + } catch (error) { + throw workspaceError(error); + } + }); + } + async read(id: string): Promise<{ workspace: WorkspaceDescriptor; revision: WorkspaceRevision }> { const state = await this.activeState(); const revision = state.revisions.find((candidate) => candidate.id === id); @@ -737,6 +784,20 @@ export class WorkspaceRegistry { } } + private storedPathExists(path: string): boolean { + try { + const entry = lstatSync(path); + if (!entry.isDirectory() || entry.isSymbolicLink()) { + throw new WorkspaceRegistryError("workspace_invalid", "Workspace registry path is invalid"); + } + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + if (error instanceof WorkspaceRegistryError) throw error; + throw new WorkspaceRegistryError("workspace_invalid", "Workspace registry path is unavailable"); + } + } + private pathIsMissing(path: string): boolean { try { lstatSync(path); diff --git a/backend/test/workspace-registry.test.ts b/backend/test/workspace-registry.test.ts index 58b1db10..99fbb976 100644 --- a/backend/test/workspace-registry.test.ts +++ b/backend/test/workspace-registry.test.ts @@ -338,6 +338,41 @@ function persistedState(root: string, commit: string): { active: any; manifest: }; } +test("verifies a never-initialized registry without contacting its remote", async () => { + const root = mkdtempSync(join(tmpdir(), "thoth-workspace-registry-uninitialized-")); + temporaryRoots.push(root); + const registryRoot = join(root, "registry"); + const registry = new WorkspaceRegistry(config( + registryRoot, + join(root, "missing-remote.git"), + )); + + await expect(registry.verifyStoredState()).resolves.toEqual({ + state: "uninitialized", + workspaces: 0, + }); + expect(existsSync(join(registryRoot, "repo"))).toBe(false); + expect(readdirSync(join(registryRoot, "state"))).toEqual([]); +}); + +test("verifies initialized snapshots and rejects partial or malformed persisted state", async () => { + const remote = await fixture(); + const registryRoot = join(remote.root, "registry"); + const registry = new WorkspaceRegistry(config(registryRoot, remote.remote)); + await registry.bootstrap(); + + await expect(registry.verifyStoredState()).resolves.toEqual({ + state: "active", + workspaces: 1, + }); + + rmSync(join(registryRoot, "state", "active.json")); + await expect(registry.verifyStoredState()).rejects.toMatchObject({ code: "workspace_invalid" }); + + writeFileSync(join(registryRoot, "state", "active.json"), "{malformed"); + await expect(registry.verifyStoredState()).rejects.toMatchObject({ code: "workspace_invalid" }); +}); + test("rejects a catalog entry without a descriptor instead of creating a bootstrap slot", async () => { const remote = await contentOnlyFixture(); const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote)); diff --git a/scripts/unified-deployment-smoke.sh b/scripts/unified-deployment-smoke.sh index 93e14e54..24a93f6c 100755 --- a/scripts/unified-deployment-smoke.sh +++ b/scripts/unified-deployment-smoke.sh @@ -983,7 +983,8 @@ PY } task13_assert_server_oidc_restore_verification() { - local archive frontend status diagnostics + local archive frontend status diagnostics active_state_before restore_output restore_rc restore_cause + local rollback_output rollback_rc rollback_sentinel provider_label checkpoint_leftover archive="$TASK13_TMP/server-oidc-restore-source.zip" frontend="$(task13_frontend_address)" task13_compose_logged "seed valid server authentication runtime excluded from restore" exec -T core node --input-type=module -e ' @@ -1003,14 +1004,88 @@ task13_assert_server_oidc_restore_verification() { browserTransactionDigest: "d".repeat(64), browserTransactionTransport: "https", }); ' + active_state_before="$(task13_compose exec -T core node -e ' + const fs = require("node:fs"); + process.stdout.write(fs.existsSync("/data/workspace-registry/state/active.json") ? "present" : "absent"); + ')" + [[ "$active_state_before" == present || "$active_state_before" == absent ]] \ + || task13_fail "server restore fixture returned an invalid registry state diagnostic" task13_compose_logged "stop server stack for OIDC restore" stop + rollback_sentinel="$TASK13_SERVER_DATA/task13-restore-rollback" + printf 'backup-state\n' >"$rollback_sentinel" task13_run_logged "create real server default-custody backup" "$TASK13_THT" \ --installation "$TASK13_INSTALLATION" backup --output "$archive" - task13_run_logged "perform real stopped OIDC production restore verification" "$TASK13_THT" \ - --installation "$TASK13_INSTALLATION" restore "$archive" --yes + printf 'current-state\n' >"$rollback_sentinel" + + provider_label="$(docker container inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$TASK13_OIDC_CONTAINER")" + [[ "$provider_label" == "$TASK13_RUN_ID" ]] || task13_fail "fake OIDC provider ownership changed before rollback injection" + task13_run_logged "inject post-mutation OIDC verification failure" docker stop "$TASK13_OIDC_CONTAINER" + rollback_output="$TASK13_TMP/server-oidc-rollback.out" + set +e + "$TASK13_THT" --installation "$TASK13_INSTALLATION" restore "$archive" --yes \ + >"$rollback_output" 2>&1 + rollback_rc=$? + set -e + [[ "$rollback_rc" -ne 0 ]] || task13_fail "server restore unexpectedly passed with its OIDC provider unavailable" + grep -Eq 'restore verification doctor:|authentication diagnostics did not pass after restore' "$rollback_output" \ + || task13_fail "server restore rollback injection did not reach post-mutation authentication verification" + if grep -Fq "$TASK13_OIDC_CLIENT_SECRET" "$rollback_output" \ + || grep -Fq "$TASK13_AUTHENTIK_API_TOKEN" "$rollback_output"; then + task13_fail "failed server restore exposed fake-provider custody values" + fi + [[ "$(cat "$rollback_sentinel")" == current-state ]] \ + || task13_fail "failed server restore did not roll back the server data bind" + checkpoint_leftover="$(find "$TASK13_CONTROL_DIR" -maxdepth 1 -name 'restore-checkpoint-*.zip' -print -quit)" + [[ -z "$checkpoint_leftover" ]] || task13_fail "failed server restore retained its private recovery checkpoint" + task13_compose_logged "verify failed restore cleared authentication runtime" \ + run --rm --no-deps --no-TTY core sh -ceu ' + test "$(stat -c %a /data/auth)" = 700 + test "$(stat -c %a /data/auth/sessions)" = 700 + test "$(stat -c %a /data/auth/oidc)" = 700 + test "$(stat -c %u /data/auth)" = "$(id -u)" + test -z "$(find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit)" + ' + task13_run_logged "restore scoped fake OIDC provider after failure injection" docker start "$TASK13_OIDC_CONTAINER" + for _attempt in $(seq 1 30); do + if docker exec "$TASK13_OIDC_CONTAINER" node -e \ + "fetch('https://task13-fake-oidc:9443/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \ + >>"$TASK13_LOG" 2>&1; then + break + fi + sleep 1 + done + docker exec "$TASK13_OIDC_CONTAINER" node -e \ + "fetch('https://task13-fake-oidc:9443/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \ + >>"$TASK13_LOG" 2>&1 || task13_log_failure "restored scoped fake OIDC provider readiness" + printf 'Task 13 server rollback injection passed: exit=%s; recovery checkpoint removed.\n' "$rollback_rc" + + restore_output="$TASK13_TMP/server-oidc-restore.out" + set +e + "$TASK13_THT" --installation "$TASK13_INSTALLATION" restore "$archive" --yes \ + >"$restore_output" 2>&1 + restore_rc=$? + set -e + if [[ "$restore_rc" -ne 0 ]]; then + restore_cause=restore-failed + if grep -Fq 'restore verification workspace: validate restored workspace registry' "$restore_output"; then + restore_cause=workspace-validator-rejected + fi + printf 'Task 13 stopped restore diagnostic: exit=%s active-state-before=%s cause=%s\n' \ + "$restore_rc" "$active_state_before" "$restore_cause" >&2 + task13_sanitize <"$restore_output" | tail -n 8 >&2 + return "$restore_rc" + fi + checkpoint_leftover="$(find "$TASK13_CONTROL_DIR" -maxdepth 1 -name 'restore-checkpoint-*.zip' -print -quit)" + [[ -z "$checkpoint_leftover" ]] || task13_fail "successful server restore retained its private recovery checkpoint" task13_compose_start_logged "start restored server stack" up --detach --wait --wait-timeout 120 core frontend - status="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ - --silent --output /dev/null --write-out '%{http_code}' "http://$frontend/api/me")" + frontend="$(task13_frontend_address)" + status="" + for _attempt in $(seq 1 30); do + status="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ + --silent --output /dev/null --write-out '%{http_code}' "http://$frontend/api/me" || true)" + [[ "$status" == 401 ]] && break + sleep 1 + done [[ "$status" == 401 ]] || task13_fail "OIDC restore did not require browser reauthentication" task13_compose_logged "verify private empty server authentication state" exec -T core sh -ceu ' test "$(stat -c %a /data/auth)" = 700 diff --git a/tools/tht/internal/backup/restore_host.go b/tools/tht/internal/backup/restore_host.go index 33dc45bb..0f6d6cba 100644 --- a/tools/tht/internal/backup/restore_host.go +++ b/tools/tht/internal/backup/restore_host.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "crypto/sha256" "encoding/hex" + "encoding/json" "errors" "fmt" "io" @@ -407,14 +408,35 @@ func verifyRestoreWorkspace(ctx context.Context, installation config.Installatio if err != nil { return err } - command := []string{"exec", "-T", "core", "node", "-e"} + command := []string{"exec", "-T", "core", "node", "/app/backend/dist/operator-command.js", "workspace-integrity"} if !running { - command = []string{"run", "--rm", "--no-deps", "--no-TTY", "core", "node", "-e"} + command = []string{"run", "--rm", "--no-deps", "--no-TTY", "core", "node", "/app/backend/dist/operator-command.js", "workspace-integrity"} } - command = append(command, `const fs=require("node:fs");const p="/data/workspace-registry/state/active.json";const s=JSON.parse(fs.readFileSync(p,"utf8"));const h=/^[0-9a-f]{40}$/;if(!h.test(s.head)||!Array.isArray(s.revisions)||s.revisions.some(r=>!r||typeof r.id!=="string"||!r.id||!h.test(r.commit)||!h.test(r.blob)))process.exit(1);for(const r of s.revisions)fs.accessSync("/data/workspace-registry/snapshots/"+r.commit+"/"+r.id+".yaml",fs.constants.R_OK)`) result, err := runner.Run(ctx, installation.ComposeArgs(command...), nil) - if err != nil { + if err != nil || result.ExitCode != 0 { return dockerError("validate restored workspace registry", result, err) } + if len(result.Stdout) == 0 || len(result.Stdout) > 4096 { + return errors.New("restored workspace registry returned an invalid result") + } + var payload struct { + Ready bool `json:"ready"` + State string `json:"state"` + Workspaces int `json:"workspaces"` + } + decoder := json.NewDecoder(strings.NewReader(result.Stdout)) + decoder.DisallowUnknownFields() + if decodeErr := decoder.Decode(&payload); decodeErr != nil { + return errors.New("restored workspace registry returned an invalid result") + } + var trailing any + if decodeErr := decoder.Decode(&trailing); !errors.Is(decodeErr, io.EOF) { + return errors.New("restored workspace registry returned an invalid result") + } + if !payload.Ready || payload.Workspaces < 0 || + (payload.State != "active" && payload.State != "uninitialized") || + (payload.State == "uninitialized" && payload.Workspaces != 0) { + return errors.New("restored workspace registry did not pass integrity validation") + } return nil } diff --git a/tools/tht/internal/backup/restore_test.go b/tools/tht/internal/backup/restore_test.go index 34270c0c..c4541e43 100644 --- a/tools/tht/internal/backup/restore_test.go +++ b/tools/tht/internal/backup/restore_test.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "errors" + "fmt" "io" "os" "path/filepath" @@ -511,6 +512,90 @@ func (runner *authenticationStateResetRunner) Stream(context.Context, []string, func (*authenticationStateResetRunner) SessionInventoryScope() string { return "mine" } +type workspaceVerificationRunner struct { + running bool + result compose.Result + err error + calls []string +} + +func (runner *workspaceVerificationRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) { + command := strings.Join(args, " ") + runner.calls = append(runner.calls, command) + if strings.Contains(command, " ps --all --format json") { + if runner.running { + return compose.Result{Stdout: healthyServicesPayload()}, nil + } + return compose.Result{}, nil + } + if strings.Contains(command, "operator-command.js workspace-integrity") { + return runner.result, runner.err + } + return compose.Result{}, fmt.Errorf("unexpected workspace verification command: %s", command) +} + +func (*workspaceVerificationRunner) Stream(context.Context, []string, io.Reader, io.Writer) (compose.Result, error) { + return compose.Result{}, errors.New("workspace verification must not stream") +} + +func (*workspaceVerificationRunner) SessionInventoryScope() string { return "mine" } + +func TestVerifyRestoreWorkspaceUsesFixedNonNetworkOperatorPath(t *testing.T) { + installation := preflightTestInstallation(t) + for _, test := range []struct { + name string + running bool + payload string + prefix string + }{ + {name: "stopped uninitialized", payload: `{"ready":true,"state":"uninitialized","workspaces":0}`, prefix: "run --rm --no-deps --no-TTY core"}, + {name: "running active", running: true, payload: `{"ready":true,"state":"active","workspaces":1}`, prefix: "exec -T core"}, + } { + t.Run(test.name, func(t *testing.T) { + runner := &workspaceVerificationRunner{ + running: test.running, + result: compose.Result{Stdout: test.payload}, + } + if err := verifyRestoreWorkspace(context.Background(), installation, runner); err != nil { + t.Fatalf("verify restored workspace: %v", err) + } + if len(runner.calls) != 2 || !strings.Contains(runner.calls[1], test.prefix+" node /app/backend/dist/operator-command.js workspace-integrity") { + t.Fatalf("workspace verification calls = %#v", runner.calls) + } + for _, call := range runner.calls { + if strings.Contains(call, "curl") || strings.Contains(call, "-e const") || strings.Contains(strings.ToLower(call), "header") { + t.Fatalf("workspace verification used an unsafe command: %s", call) + } + } + }) + } +} + +func TestVerifyRestoreWorkspaceRejectsInvalidOperatorResults(t *testing.T) { + installation := preflightTestInstallation(t) + for _, test := range []struct { + name string + result compose.Result + err error + }{ + {name: "empty"}, + {name: "malformed", result: compose.Result{Stdout: `{malformed`}}, + {name: "trailing document", result: compose.Result{Stdout: `{"ready":true,"state":"active","workspaces":1}{}`}}, + {name: "unknown field", result: compose.Result{Stdout: `{"ready":true,"state":"active","workspaces":1,"detail":"unsafe"}`}}, + {name: "not ready", result: compose.Result{Stdout: `{"ready":false,"state":"uninitialized","workspaces":0}`}}, + {name: "unknown state", result: compose.Result{Stdout: `{"ready":true,"state":"unknown","workspaces":0}`}}, + {name: "inconsistent count", result: compose.Result{Stdout: `{"ready":true,"state":"uninitialized","workspaces":1}`}}, + {name: "nonzero", result: compose.Result{ExitCode: 2}, err: errors.New("exit status 2")}, + } { + t.Run(test.name, func(t *testing.T) { + runner := &workspaceVerificationRunner{result: test.result, err: test.err} + if err := verifyRestoreWorkspace(context.Background(), installation, runner); err == nil { + t.Fatal("invalid workspace verification result was accepted") + } + }) + } +} + func (lock fakeRestoreLock) Release() error { if lock.release != nil { lock.release()