From dee17893b41af1b55220237ea8ef0db395332648 Mon Sep 17 00:00:00 2001 From: mptyl Date: Tue, 18 Aug 2026 00:58:12 +0200 Subject: [PATCH] fix(auth): harden restore verification transaction --- backend/src/operator-command.ts | 1 + backend/src/workspaces/registry.ts | 207 +++++++++++++----- backend/test/workspace-registry.test.ts | 122 ++++++++++- scripts/unified-deployment-smoke.sh | 118 +++++++++- tools/tht/internal/backup/restore.go | 28 +++ .../internal/backup/restore_file_windows.go | 60 ++--- .../restore_file_windows_policy_test.go | 31 +++ .../backup/restore_file_windows_test.go | 83 +++++++ tools/tht/internal/backup/restore_host.go | 9 +- tools/tht/internal/backup/restore_test.go | 154 ++++++++++++- 10 files changed, 698 insertions(+), 115 deletions(-) create mode 100644 tools/tht/internal/backup/restore_file_windows_policy_test.go create mode 100644 tools/tht/internal/backup/restore_file_windows_test.go diff --git a/backend/src/operator-command.ts b/backend/src/operator-command.ts index 30648074..db820c45 100644 --- a/backend/src/operator-command.ts +++ b/backend/src/operator-command.ts @@ -91,6 +91,7 @@ async function workspaceIntegrity(config: AppConfig): Promise<{ ready: true; state: "uninitialized" | "active"; workspaces: number; + fingerprint: string; }> { const integrity = await new WorkspaceRegistry(config.workspaceRegistry).verifyStoredState(); return { ready: true, ...integrity }; diff --git a/backend/src/workspaces/registry.ts b/backend/src/workspaces/registry.ts index cacd72e4..e69b4e87 100644 --- a/backend/src/workspaces/registry.ts +++ b/backend/src/workspaces/registry.ts @@ -1,5 +1,5 @@ import { createHash, randomUUID } from "node:crypto"; -import { lstatSync, readFileSync } from "node:fs"; +import { lstatSync, readdirSync, readFileSync } from "node:fs"; import { mkdir, readdir, readFile, rename, rm, writeFile } from "node:fs/promises"; import { isAbsolute, join } from "node:path"; import { buildInstallationContract, renderWorkspaceDocs } from "./contracts.js"; @@ -34,6 +34,7 @@ export interface WorkspaceRevision { export interface StoredWorkspaceIntegrity { state: "uninitialized" | "active"; workspaces: number; + fingerprint: string; } export interface SessionRevisionLease { @@ -188,43 +189,137 @@ export class WorkspaceRegistry { /** * Validate only persisted local registry state. Restore uses this path while the installation * is stopped: it must neither contact Git nor turn a never-used registry into initialized state. + * The only never-initialized shape is an existing empty root. An initialized root is exactly + * repo/, snapshots/, state/, and locks/: locks contains repository.lock plus an empty + * empty-hooks/, state contains active.json plus an optional empty revision-leases/, and + * snapshots contains exact immutable commit snapshots plus an optional empty runtime/. + * Descendants may contain only ordinary directories and regular files; links and special files + * are rejected by the stable whole-tree fingerprint before any shape is accepted. */ async verifyStoredState(): Promise { - await this.repository.ensureLayout(); - return await this.lock.run(async () => { - try { - const stateEntries = await readdir(this.repository.statePath, { withFileTypes: true }); - if (stateEntries.some((entry) => ( - entry.name !== "active.json" || !entry.isFile() || entry.isSymbolicLink() - ))) throw new Error("workspace state directory is partial"); - - const snapshotEntries = await readdir(this.repository.snapshotsPath, { withFileTypes: true }); - for (const entry of snapshotEntries) { - const isRuntime = entry.name === "runtime"; - const isSnapshot = /^[0-9a-f]{40}$/.test(entry.name); - if ((!isRuntime && !isSnapshot) || !entry.isDirectory() || entry.isSymbolicLink()) { - throw new Error("workspace snapshot directory is partial"); - } - } - - const hasActiveState = stateEntries.length === 1; - if (!hasActiveState) { - if (snapshotEntries.some((entry) => entry.name !== "runtime") || this.storedPathExists(this.repository.repoPath)) { - throw new Error("workspace registry is partially initialized"); - } - return { state: "uninitialized", workspaces: 0 }; - } - - const active = await this.activeState(); - for (const entry of snapshotEntries) { - if (entry.name === "runtime" || entry.name === active.head) continue; - await this.snapshotState(entry.name); - } - return { state: "active", workspaces: active.revisions.length }; - } catch (error) { - throw workspaceError(error); + try { + const before = await this.storedStateFingerprint(); + const rootEntries = await readdir(this.repository.root, { withFileTypes: true }); + if (rootEntries.length === 0) { + const after = await this.storedStateFingerprint(); + if (after !== before) throw new Error("workspace registry changed during inspection"); + return { state: "uninitialized", workspaces: 0, fingerprint: `sha256:${before}` }; } - }); + this.assertExactDirectoryEntries(rootEntries, { + locks: "directory", repo: "directory", snapshots: "directory", state: "directory", + }); + + this.assertExactDirectoryEntries( + await readdir(this.repository.locksPath, { withFileTypes: true }), + { "empty-hooks": "directory", "repository.lock": "file" }, + ); + this.assertExactDirectoryEntries( + await readdir(join(this.repository.locksPath, "empty-hooks"), { withFileTypes: true }), + {}, + ); + + const stateEntries = await readdir(this.repository.statePath, { withFileTypes: true }); + const stateShape: Record = { "active.json": "file" }; + if (stateEntries.some((entry) => entry.name === "revision-leases")) { + stateShape["revision-leases"] = "directory"; + } + this.assertExactDirectoryEntries(stateEntries, stateShape); + if (stateShape["revision-leases"] !== undefined) { + this.assertExactDirectoryEntries( + await readdir(join(this.repository.statePath, "revision-leases"), { withFileTypes: true }), + {}, + ); + } + + const active = this.decodeActiveState(JSON.parse(await readFile( + join(this.repository.statePath, "active.json"), "utf8", + ))); + const snapshotEntries = await readdir(this.repository.snapshotsPath, { withFileTypes: true }); + if (snapshotEntries.length === 0) throw new Error("workspace snapshots are unavailable"); + let activeSnapshotFound = false; + for (const entry of snapshotEntries) { + if (entry.name === "runtime") { + if (!entry.isDirectory() || entry.isSymbolicLink()) throw new Error("workspace runtime path is invalid"); + this.assertExactDirectoryEntries( + await readdir(join(this.repository.snapshotsPath, "runtime"), { withFileTypes: true }), + {}, + ); + continue; + } + if (!/^[0-9a-f]{40}$/.test(entry.name) || !entry.isDirectory() || entry.isSymbolicLink()) { + throw new Error("workspace snapshot path is invalid"); + } + const state = entry.name === active.head ? active : await this.readStoredSnapshotState(entry.name); + await this.assertSnapshotIntegrity(state, false, true); + if (entry.name === active.head) activeSnapshotFound = true; + } + if (!activeSnapshotFound) throw new Error("active workspace snapshot is unavailable"); + + const after = await this.storedStateFingerprint(); + if (after !== before) throw new Error("workspace registry changed during inspection"); + return { state: "active", workspaces: active.revisions.length, fingerprint: `sha256:${before}` }; + } catch (error) { + throw workspaceError(error); + } + } + + private assertExactDirectoryEntries( + entries: Array<{ name: string; isFile(): boolean; isDirectory(): boolean; isSymbolicLink(): boolean }>, + expected: Record, + ): void { + if (entries.length !== Object.keys(expected).length) throw new Error("workspace registry shape is invalid"); + for (const entry of entries) { + const kind = expected[entry.name]; + if (kind === undefined || entry.isSymbolicLink() + || (kind === "file" && !entry.isFile()) + || (kind === "directory" && !entry.isDirectory())) { + throw new Error("workspace registry shape is invalid"); + } + } + } + + private async storedStateFingerprint(): Promise { + const records: string[] = []; + let entries = 0; + let totalBytes = 0; + const visit = async (path: string, relative: string): Promise => { + const before = lstatSync(path); + if (before.isSymbolicLink()) throw new Error("workspace registry link is invalid"); + const metadata = [ + before.dev, before.ino, before.mode, before.uid, before.gid, + before.size, before.mtimeMs, before.ctimeMs, + ].join(":"); + entries += 1; + if (entries > 65_536) throw new Error("workspace registry contains too many entries"); + if (before.isDirectory()) { + records.push(`directory:${relative}:${metadata}`); + const children = await readdir(path); + children.sort(); + for (const name of children) { + await visit(join(path, name), relative === "." ? name : `${relative}/${name}`); + } + } else if (before.isFile()) { + if (before.size > 256 * 1024 * 1024) throw new Error("workspace registry file is too large"); + totalBytes += before.size; + if (totalBytes > 2 * 1024 * 1024 * 1024) throw new Error("workspace registry is too large"); + const contents = await readFile(path); + records.push(`file:${relative}:${metadata}:${contents.length}:${digest(contents)}`); + } else { + throw new Error("workspace registry entry is invalid"); + } + const after = lstatSync(path); + if (before.dev !== after.dev || before.ino !== after.ino || before.mode !== after.mode + || before.uid !== after.uid || before.gid !== after.gid || before.size !== after.size + || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs) { + throw new Error("workspace registry changed during inspection"); + } + }; + await visit(this.repository.root, "."); + return digest(records.join("\n")); + } + + private async readStoredSnapshotState(head: string): Promise { + return this.decodeSnapshotManifest(await this.readSnapshotManifest(safeCommit(head))); } async read(id: string): Promise<{ workspace: WorkspaceDescriptor; revision: WorkspaceRevision }> { @@ -696,7 +791,11 @@ export class WorkspaceRegistry { return state; } - private async assertSnapshotIntegrity(state: ActiveState): Promise { + private async assertSnapshotIntegrity( + state: ActiveState, + verifyGitEvidence = true, + exactStoredShape = false, + ): Promise { const directory = join(this.repository.snapshotsPath, state.head); try { const manifest = this.decodeSnapshotManifest(await this.readSnapshotManifest(state.head)); @@ -705,13 +804,33 @@ export class WorkspaceRegistry { throw new Error("manifest state does not match active state"); } await this.assertManifestFiles(directory, manifest.files, this.expectedSnapshotFiles(state, directory)); - await this.assertSnapshotEvidenceContexts(state); + if (exactStoredShape) this.assertStoredSnapshotShape(directory, state); + if (verifyGitEvidence) await this.assertSnapshotEvidenceContexts(state); } catch (error) { if (error instanceof WorkspaceRegistryError) throw error; throw new WorkspaceRegistryError("workspace_invalid", "Workspace snapshot integrity check failed"); } } + private assertStoredSnapshotShape(directory: string, state: ActiveState): void { + const expected: Record = { "snapshot.json": "file" }; + for (const revision of state.revisions) { + expected[`${revision.id}.yaml`] = "file"; + expected[`${revision.id}.env.example`] = "file"; + expected[`${revision.id}.md`] = "file"; + const workspace = parseWorkspaceYaml(readFileSync(join(directory, `${revision.id}.yaml`), "utf8")); + if (workspace.evidence?.source.type === "filesystem") expected[revision.id] = "directory"; + } + this.assertExactDirectoryEntries(readdirSync(directory, { withFileTypes: true }), expected); + for (const revision of state.revisions) { + if (expected[revision.id] !== "directory") continue; + this.assertExactDirectoryEntries( + readdirSync(join(directory, revision.id), { withFileTypes: true }), + { evidence: "directory", "evidence.manifest.json": "file" }, + ); + } + } + private expectedSnapshotFiles(state: ActiveState, directory: string): string[] { return state.revisions.flatMap((revision) => { const names = [`${revision.id}.yaml`, `${revision.id}.env.example`, `${revision.id}.md`]; @@ -784,20 +903,6 @@ export class WorkspaceRegistry { } } - private storedPathExists(path: string): boolean { - try { - const entry = lstatSync(path); - if (!entry.isDirectory() || entry.isSymbolicLink()) { - throw new WorkspaceRegistryError("workspace_invalid", "Workspace registry path is invalid"); - } - return true; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; - if (error instanceof WorkspaceRegistryError) throw error; - throw new WorkspaceRegistryError("workspace_invalid", "Workspace registry path is unavailable"); - } - } - private pathIsMissing(path: string): boolean { try { lstatSync(path); diff --git a/backend/test/workspace-registry.test.ts b/backend/test/workspace-registry.test.ts index 99fbb976..08b9163b 100644 --- a/backend/test/workspace-registry.test.ts +++ b/backend/test/workspace-registry.test.ts @@ -1,7 +1,7 @@ import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { - chmodSync, existsSync, mkdtempSync, mkdirSync, readdirSync, readFileSync, rmSync, symlinkSync, writeFileSync, + chmodSync, existsSync, lstatSync, mkdtempSync, mkdirSync, readdirSync, readFileSync, rmSync, symlinkSync, utimesSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -338,21 +338,70 @@ function persistedState(root: string, commit: string): { active: any; manifest: }; } +function filesystemFingerprint(root: string): string { + if (!existsSync(root)) return "absent"; + const records: string[] = []; + const visit = (path: string, relative: string): void => { + const entry = lstatSync(path); + const metadata = [ + entry.dev, entry.ino, entry.mode, entry.uid, entry.gid, + entry.size, entry.mtimeMs, entry.ctimeMs, + ].join(":"); + if (entry.isSymbolicLink()) { + records.push(`link:${relative}:${metadata}`); + return; + } + if (entry.isDirectory()) { + records.push(`directory:${relative}:${metadata}`); + for (const name of readdirSync(path).sort()) visit(join(path, name), relative === "." ? name : `${relative}/${name}`); + return; + } + if (entry.isFile()) { + const contents = readFileSync(path); + records.push(`file:${relative}:${metadata}:${contents.length}:${createHash("sha256").update(contents).digest("hex")}`); + return; + } + records.push(`other:${relative}:${metadata}`); + }; + visit(root, "."); + return createHash("sha256").update(records.join("\n")).digest("hex"); +} + test("verifies a never-initialized registry without contacting its remote", async () => { const root = mkdtempSync(join(tmpdir(), "thoth-workspace-registry-uninitialized-")); temporaryRoots.push(root); const registryRoot = join(root, "registry"); + mkdirSync(registryRoot, { mode: 0o700 }); const registry = new WorkspaceRegistry(config( registryRoot, join(root, "missing-remote.git"), )); + const before = filesystemFingerprint(registryRoot); + await expect(registry.verifyStoredState()).resolves.toEqual({ state: "uninitialized", workspaces: 0, + fingerprint: `sha256:${before}`, }); + expect(filesystemFingerprint(registryRoot)).toBe(before); expect(existsSync(join(registryRoot, "repo"))).toBe(false); - expect(readdirSync(join(registryRoot, "state"))).toEqual([]); + expect(readdirSync(registryRoot)).toEqual([]); +}); + +test("never-initialized inspection refuses an absent or partial root without creating it", async () => { + const root = mkdtempSync(join(tmpdir(), "thoth-workspace-registry-absent-")); + temporaryRoots.push(root); + const registryRoot = join(root, "registry"); + const registry = new WorkspaceRegistry(config(registryRoot, join(root, "missing-remote.git"))); + + await expect(registry.verifyStoredState()).rejects.toMatchObject({ code: "workspace_invalid" }); + expect(filesystemFingerprint(registryRoot)).toBe("absent"); + + mkdirSync(join(registryRoot, "state"), { recursive: true }); + const partial = filesystemFingerprint(registryRoot); + await expect(registry.verifyStoredState()).rejects.toMatchObject({ code: "workspace_invalid" }); + expect(filesystemFingerprint(registryRoot)).toBe(partial); }); test("verifies initialized snapshots and rejects partial or malformed persisted state", async () => { @@ -361,10 +410,25 @@ test("verifies initialized snapshots and rejects partial or malformed persisted const registry = new WorkspaceRegistry(config(registryRoot, remote.remote)); await registry.bootstrap(); - await expect(registry.verifyStoredState()).resolves.toEqual({ - state: "active", - workspaces: 1, - }); + const before = filesystemFingerprint(registryRoot); + const savedPath = process.env.PATH; + process.env.PATH = join(remote.root, "no-executables"); + try { + await expect(registry.verifyStoredState()).resolves.toEqual({ + state: "active", + workspaces: 1, + fingerprint: `sha256:${before}`, + }); + } finally { + if (savedPath === undefined) delete process.env.PATH; + else process.env.PATH = savedPath; + } + expect(filesystemFingerprint(registryRoot)).toBe(before); + + const firstIntegrity = await registry.verifyStoredState(); + utimesSync(join(registryRoot, "repo"), new Date(1_000), new Date(1_000)); + const metadataIntegrity = await registry.verifyStoredState(); + expect(metadataIntegrity.fingerprint).not.toBe(firstIntegrity.fingerprint); rmSync(join(registryRoot, "state", "active.json")); await expect(registry.verifyStoredState()).rejects.toMatchObject({ code: "workspace_invalid" }); @@ -373,6 +437,52 @@ test("verifies initialized snapshots and rejects partial or malformed persisted await expect(registry.verifyStoredState()).rejects.toMatchObject({ code: "workspace_invalid" }); }); +test("read-only inspection rejects extra components, links, and ephemeral runtime contents", async () => { + const remote = await fixture(); + const registryRoot = join(remote.root, "registry"); + const registry = new WorkspaceRegistry(config(registryRoot, remote.remote)); + const status = await registry.bootstrap(); + const assertHostile = async (setup: () => void, cleanup: () => void): Promise => { + setup(); + const before = filesystemFingerprint(registryRoot); + await expect(registry.verifyStoredState()).rejects.toMatchObject({ code: "workspace_invalid" }); + expect(filesystemFingerprint(registryRoot)).toBe(before); + cleanup(); + }; + + await assertHostile( + () => mkdirSync(join(registryRoot, "unexpected")), + () => rmSync(join(registryRoot, "unexpected"), { recursive: true }), + ); + await assertHostile( + () => rmSync(join(registryRoot, "locks", "empty-hooks"), { recursive: true }), + () => mkdirSync(join(registryRoot, "locks", "empty-hooks")), + ); + await assertHostile( + () => writeFileSync(join(registryRoot, "state", "unexpected.json"), "{}"), + () => rmSync(join(registryRoot, "state", "unexpected.json")), + ); + await assertHostile( + () => writeFileSync(join(registryRoot, "snapshots", status.head!, "unexpected"), "extra"), + () => rmSync(join(registryRoot, "snapshots", status.head!, "unexpected")), + ); + await assertHostile( + () => { + mkdirSync(join(registryRoot, "snapshots", "runtime"), { recursive: true }); + writeFileSync(join(registryRoot, "snapshots", "runtime", "restored-secret.yaml"), "secret: forbidden"); + }, + () => rmSync(join(registryRoot, "snapshots", "runtime"), { recursive: true }), + ); + await assertHostile( + () => symlinkSync(join(registryRoot, "state", "active.json"), join(registryRoot, "linked-active.json")), + () => rmSync(join(registryRoot, "linked-active.json")), + ); + await assertHostile( + () => symlinkSync(join(registryRoot, "state", "active.json"), join(registryRoot, "repo", "linked-active.json")), + () => rmSync(join(registryRoot, "repo", "linked-active.json")), + ); +}); + test("rejects a catalog entry without a descriptor instead of creating a bootstrap slot", async () => { const remote = await contentOnlyFixture(); const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote)); diff --git a/scripts/unified-deployment-smoke.sh b/scripts/unified-deployment-smoke.sh index 24a93f6c..2c53b83e 100755 --- a/scripts/unified-deployment-smoke.sh +++ b/scripts/unified-deployment-smoke.sh @@ -29,6 +29,71 @@ task13_sha256_text() { fi } +task13_registry_filesystem_fingerprint() { + python3 - "$1" <<'PY' +import hashlib +import os +import stat +import sys + +root = os.path.abspath(sys.argv[1]) +records = [] +entries = 0 +total_bytes = 0 + +def snapshot(value): + return ( + value.st_dev, value.st_ino, value.st_mode, value.st_uid, value.st_gid, + value.st_size, value.st_mtime_ns, value.st_ctime_ns, + ) + +def visit(path, relative): + global entries, total_bytes + before = os.lstat(path) + entries += 1 + if entries > 65536: + raise SystemExit("registry fingerprint entry bound exceeded") + metadata = snapshot(before) + if stat.S_ISLNK(before.st_mode): + raise SystemExit("registry fingerprint rejected a symbolic link") + if stat.S_ISDIR(before.st_mode): + records.append(("directory", relative, metadata)) + with os.scandir(path) as listing: + children = sorted((item.name for item in listing)) + for name in children: + visit(os.path.join(path, name), name if relative == "." else relative + "/" + name) + elif stat.S_ISREG(before.st_mode): + if before.st_size > 256 * 1024 * 1024: + raise SystemExit("registry fingerprint file bound exceeded") + total_bytes += before.st_size + if total_bytes > 2 * 1024 * 1024 * 1024: + raise SystemExit("registry fingerprint total bound exceeded") + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + opened = os.fstat(descriptor) + if snapshot(opened) != metadata: + raise SystemExit("registry changed while fingerprinting") + digest = hashlib.sha256() + while True: + chunk = os.read(descriptor, 1024 * 1024) + if not chunk: + break + digest.update(chunk) + finally: + os.close(descriptor) + records.append(("file", relative, metadata, digest.hexdigest())) + else: + raise SystemExit("registry fingerprint rejected a special file") + if snapshot(os.lstat(path)) != metadata: + raise SystemExit("registry changed while fingerprinting") + +visit(root, ".") +encoded = repr(records).encode("utf-8") +print("sha256:" + hashlib.sha256(encoded).hexdigest()) +PY +} + task13_sanitize() { local line while IFS= read -r line || [[ -n "$line" ]]; do @@ -983,8 +1048,9 @@ PY } task13_assert_server_oidc_restore_verification() { - local archive frontend status diagnostics active_state_before restore_output restore_rc restore_cause + local archive frontend status diagnostics restore_output restore_rc restore_cause local rollback_output rollback_rc rollback_sentinel provider_label checkpoint_leftover + local registry_before registry_after integrity_output archive="$TASK13_TMP/server-oidc-restore-source.zip" frontend="$(task13_frontend_address)" task13_compose_logged "seed valid server authentication runtime excluded from restore" exec -T core node --input-type=module -e ' @@ -1004,12 +1070,6 @@ task13_assert_server_oidc_restore_verification() { browserTransactionDigest: "d".repeat(64), browserTransactionTransport: "https", }); ' - active_state_before="$(task13_compose exec -T core node -e ' - const fs = require("node:fs"); - process.stdout.write(fs.existsSync("/data/workspace-registry/state/active.json") ? "present" : "absent"); - ')" - [[ "$active_state_before" == present || "$active_state_before" == absent ]] \ - || task13_fail "server restore fixture returned an invalid registry state diagnostic" task13_compose_logged "stop server stack for OIDC restore" stop rollback_sentinel="$TASK13_SERVER_DATA/task13-restore-rollback" printf 'backup-state\n' >"$rollback_sentinel" @@ -1059,6 +1119,22 @@ task13_assert_server_oidc_restore_verification() { >>"$TASK13_LOG" 2>&1 || task13_log_failure "restored scoped fake OIDC provider readiness" printf 'Task 13 server rollback injection passed: exit=%s; recovery checkpoint removed.\n' "$rollback_rc" + registry_before="$(task13_registry_filesystem_fingerprint "$TASK13_SERVER_REGISTRY")" + integrity_output="$TASK13_TMP/server-workspace-integrity.json" + if ! task13_compose run --rm --no-deps --no-TTY core \ + node /app/backend/dist/operator-command.js workspace-integrity \ + >"$integrity_output" 2>>"$TASK13_LOG"; then + task13_log_failure "stopped read-only workspace registry verification" + fi + node -e ' + const value=JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); + const keys=Object.keys(value).sort().join(","); + if(keys!=="fingerprint,ready,state,workspaces"||value.ready!==true||value.state!=="uninitialized"||value.workspaces!==0||!/^sha256:[0-9a-f]{64}$/.test(value.fingerprint)) process.exit(1); + ' "$integrity_output" || task13_fail "stopped registry inspector returned malformed or additional output" + registry_after="$(task13_registry_filesystem_fingerprint "$TASK13_SERVER_REGISTRY")" + [[ "$registry_after" == "$registry_before" ]] \ + || task13_fail "stopped restore verification mutated the workspace registry filesystem" + restore_output="$TASK13_TMP/server-oidc-restore.out" set +e "$TASK13_THT" --installation "$TASK13_INSTALLATION" restore "$archive" --yes \ @@ -1070,8 +1146,8 @@ task13_assert_server_oidc_restore_verification() { if grep -Fq 'restore verification workspace: validate restored workspace registry' "$restore_output"; then restore_cause=workspace-validator-rejected fi - printf 'Task 13 stopped restore diagnostic: exit=%s active-state-before=%s cause=%s\n' \ - "$restore_rc" "$active_state_before" "$restore_cause" >&2 + printf 'Task 13 stopped restore diagnostic: exit=%s cause=%s\n' \ + "$restore_rc" "$restore_cause" >&2 task13_sanitize <"$restore_output" | tail -n 8 >&2 return "$restore_rc" fi @@ -1086,7 +1162,8 @@ task13_assert_server_oidc_restore_verification() { [[ "$status" == 401 ]] && break sleep 1 done - [[ "$status" == 401 ]] || task13_fail "OIDC restore did not require browser reauthentication" + [[ "$status" == 401 ]] \ + || task13_fail "OIDC restore did not require browser reauthentication (HTTP ${status:-unavailable})" task13_compose_logged "verify private empty server authentication state" exec -T core sh -ceu ' test "$(stat -c %a /data/auth)" = 700 test "$(stat -c %a /data/auth/sessions)" = 700 @@ -2030,6 +2107,26 @@ task13_self_test_server_release_contract() { || task13_fail "workflow lacks an outer timeout for the Linux server smoke" } +task13_self_test_registry_fingerprint() { + local fixture fixture_escaped before after linked + fixture="$(mktemp -d "${TMPDIR:-/tmp}/thothii-task13-registry-fingerprint.XXXXXX")" + printf -v fixture_escaped '%q' "$fixture" + trap "rm -rf -- $fixture_escaped; trap - RETURN" RETURN + before="$(task13_registry_filesystem_fingerprint "$fixture")" + [[ "$before" =~ ^sha256:[0-9a-f]{64}$ ]] \ + || task13_fail "registry fingerprint did not return a bounded digest" + [[ "$(task13_registry_filesystem_fingerprint "$fixture")" == "$before" ]] \ + || task13_fail "registry fingerprint changed without a filesystem mutation" + mkdir "$fixture/locks" + after="$(task13_registry_filesystem_fingerprint "$fixture")" + [[ "$after" != "$before" ]] || task13_fail "registry fingerprint ignored a new directory" + linked="$fixture/linked" + ln -s "$fixture/locks" "$linked" 2>/dev/null || return 0 + if task13_registry_filesystem_fingerprint "$fixture" >/dev/null 2>&1; then + task13_fail "registry fingerprint accepted a symbolic link" + fi +} + task13_self_test_source_contract() { local root host_network push_command registry_function workflow uses_count pinned_uses_count root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" @@ -2094,6 +2191,7 @@ task13_self_test() { task13_self_test_internal_semantic_offline_contract task13_self_test_windows_release_contract task13_self_test_server_release_contract + task13_self_test_registry_fingerprint task13_self_test_source_contract printf 'Task 13 smoke safety contracts passed.\n' } diff --git a/tools/tht/internal/backup/restore.go b/tools/tht/internal/backup/restore.go index 6ff619e5..e8de5d86 100644 --- a/tools/tht/internal/backup/restore.go +++ b/tools/tht/internal/backup/restore.go @@ -104,20 +104,43 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati return result, err } mutated := false + maintenanceAttempted := false + stopAttempted := false defer func() { if resultErr != nil && mutated { if recoveryErr := deps.recover(context.Background(), installation, recovery, wasRunning); recoveryErr != nil { resultErr = errors.Join(resultErr, fmt.Errorf("restore recovery checkpoint: %w", recoveryErr)) + } else { + stopAttempted = false + } + } + if maintenanceAttempted { + cleanupContext := context.Background() + if wasRunning && stopAttempted { + if startErr := composeStartAndVerify(cleanupContext, installation, deps.runner); startErr != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("restore maintenance cleanup restart: %w", startErr)) + } else { + stopAttempted = false + } + } + if deactivateErr := maintenance(cleanupContext, installation, deps.runner, false); deactivateErr != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("restore maintenance cleanup: %w", deactivateErr)) } } }() if wasRunning { + // The activation command may take effect even when its response is lost. Track the attempt, + // not merely a successful return, so every subsequent path removes the admissions barrier. + maintenanceAttempted = true if err := maintenance(ctx, installation, deps.runner, true); err != nil { return result, err } if err := waitForNoActiveSessions(ctx, installation, deps.runner, request.Drain, deps.sleep); err != nil { return result, err } + // Compose may stop the core and then lose its response. Cleanup must therefore restart after + // any stop attempt, including a command that returns an error. + stopAttempted = true if err := runCompose(ctx, installation, deps.runner, "stop"); err != nil { return result, err } @@ -133,7 +156,12 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati if err := composeStartAndVerify(ctx, installation, deps.runner); err != nil { return result, err } + stopAttempted = false result.Restarted = true + if err := maintenance(ctx, installation, deps.runner, false); err != nil { + return result, err + } + maintenanceAttempted = false } for _, name := range []string{"health", "doctor", "pi", "workspace"} { check := deps.verify[name] diff --git a/tools/tht/internal/backup/restore_file_windows.go b/tools/tht/internal/backup/restore_file_windows.go index 71572789..898e813d 100644 --- a/tools/tht/internal/backup/restore_file_windows.go +++ b/tools/tht/internal/backup/restore_file_windows.go @@ -3,57 +3,45 @@ package backup import ( - "errors" "os" "path/filepath" "github.com/aritmolab/thothii/tools/tht/internal/safeio" - "golang.org/x/sys/windows" ) -func replaceRestoreFile(target string, contents []byte, mode os.FileMode) error { - if safeio.ValidateCanonicalPath(target) != nil || mode&os.ModeType != 0 || mode.Perm() == 0 { +// replaceRestoreFile deliberately supports only owner-private Windows parents. The retained +// native directory handle pins every ancestor, rejects reparse components, creates an owner-only +// temporary file, and publishes it by an NT RootDirectory-relative atomic rename. Installations +// whose restore targets do not satisfy that custody contract fail closed instead of falling back +// to a path-based replacement. +func replaceRestoreFile(target string, contents []byte, mode os.FileMode) (resultErr error) { + if safeio.ValidateCanonicalPath(target) != nil || mode&os.ModeType != 0 || mode.Perm() == 0 || len(contents) == 0 { return safeio.ErrUnsafeFile } - parent := filepath.Dir(target) - resolved, err := filepath.EvalSymlinks(parent) - if err != nil || resolved != parent || !safeWindowsRestoreTarget(target) { + parent, found, err := safeio.OpenPrivateDirectory(filepath.Dir(target), false) + if err != nil || !found || parent == nil { return safeio.ErrUnsafeFile } - temporary, err := os.CreateTemp(parent, ".tht-restore-*.tmp") + defer func() { + if closeErr := parent.Close(); closeErr != nil { + resultErr = safeio.ErrUnsafeFile + } + }() + safeio.NotifyPrivateDirectoryTestHookForTest("after-restore-parent-open") + if parent.Validate() != nil { + return safeio.ErrUnsafeFile + } + created, err := parent.CreateRegular(filepath.Base(target), contents) if err != nil { return safeio.ErrUnsafeFile } - temporaryPath := temporary.Name() - defer os.Remove(temporaryPath) - if err := temporary.Chmod(mode.Perm()); err == nil { - _, err = temporary.Write(contents) + if !created { + if err := parent.ReplaceRegular(filepath.Base(target), contents); err != nil { + return safeio.ErrUnsafeFile + } } - if err == nil { - err = temporary.Sync() - } - closeErr := temporary.Close() - if err == nil { - err = closeErr - } - if err != nil || !safeWindowsRestoreTarget(target) { - return safeio.ErrUnsafeFile - } - from, fromErr := windows.UTF16PtrFromString(temporaryPath) - to, toErr := windows.UTF16PtrFromString(target) - if fromErr != nil || toErr != nil { - return safeio.ErrUnsafeFile - } - if err := windows.MoveFileEx(from, to, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH); err != nil { + if parent.Validate() != nil { return safeio.ErrUnsafeFile } return nil } - -func safeWindowsRestoreTarget(target string) bool { - info, err := os.Lstat(target) - if errors.Is(err, os.ErrNotExist) { - return true - } - return err == nil && info.Mode().IsRegular() && info.Mode()&os.ModeSymlink == 0 -} diff --git a/tools/tht/internal/backup/restore_file_windows_policy_test.go b/tools/tht/internal/backup/restore_file_windows_policy_test.go new file mode 100644 index 00000000..f2265f74 --- /dev/null +++ b/tools/tht/internal/backup/restore_file_windows_policy_test.go @@ -0,0 +1,31 @@ +package backup + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestWindowsRestoreReplacementUsesPinnedHandleRelativeSafeIO(t *testing.T) { + _, current, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("cannot locate Windows restore implementation") + } + contents, err := os.ReadFile(filepath.Join(filepath.Dir(current), "restore_file_windows.go")) + if err != nil { + t.Fatal(err) + } + source := string(contents) + for _, forbidden := range []string{"EvalSymlinks", "MoveFileEx", "CreateTemp"} { + if strings.Contains(source, forbidden) { + t.Fatalf("Windows restore replacement retains forbidden path-based primitive %q", forbidden) + } + } + for _, required := range []string{"OpenPrivateDirectory", "CreateRegular", "ReplaceRegular", "after-restore-parent-open"} { + if !strings.Contains(source, required) { + t.Fatalf("Windows restore replacement omits pinned safe-I/O primitive %q", required) + } + } +} diff --git a/tools/tht/internal/backup/restore_file_windows_test.go b/tools/tht/internal/backup/restore_file_windows_test.go new file mode 100644 index 00000000..160cff5a --- /dev/null +++ b/tools/tht/internal/backup/restore_file_windows_test.go @@ -0,0 +1,83 @@ +//go:build windows + +package backup + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/aritmolab/thothii/tools/tht/internal/safeio" +) + +func TestReplaceRestoreFileWindowsCreatesAndReplacesOwnerOnlyRegular(t *testing.T) { + parent := filepath.Join(t.TempDir(), "private") + if err := os.Mkdir(parent, 0o700); err != nil { + t.Fatal(err) + } + if err := safeio.ProtectPrivateDirectory(parent); err != nil { + t.Fatal(err) + } + target := filepath.Join(parent, "auth.yaml") + if err := replaceRestoreFile(target, []byte("created"), 0o600); err != nil { + t.Fatal(err) + } + if err := safeio.ValidatePrivateRegular(target); err != nil { + t.Fatalf("created restore file is not owner-only: %v", err) + } + if err := replaceRestoreFile(target, []byte("replacement"), 0o600); err != nil { + t.Fatal(err) + } + if err := safeio.ValidatePrivateRegular(target); err != nil { + t.Fatalf("replacement restore file is not owner-only: %v", err) + } + contents, err := os.ReadFile(target) + if err != nil || string(contents) != "replacement" { + t.Fatalf("replacement contents = %q, error = %v", contents, err) + } +} + +func TestReplaceRestoreFileWindowsPinsParentAndRejectsReparsePath(t *testing.T) { + root := t.TempDir() + parent := filepath.Join(root, "private") + if err := os.Mkdir(parent, 0o700); err != nil { + t.Fatal(err) + } + if err := safeio.ProtectPrivateDirectory(parent); err != nil { + t.Fatal(err) + } + target := filepath.Join(parent, "users.yaml") + if err := os.WriteFile(target, []byte("old"), 0o600); err != nil { + t.Fatal(err) + } + if err := safeio.ProtectPrivateRegular(target); err != nil { + t.Fatal(err) + } + renamed := parent + "-renamed" + var hookCalled bool + restoreHook := safeio.SetPrivateDirectoryTestHookForTest(func(stage string) { + if stage != "after-restore-parent-open" { + return + } + hookCalled = true + if err := os.Rename(parent, renamed); err == nil { + t.Fatal("parent rename succeeded while restore retained its directory handles") + } + }) + defer restoreHook() + if err := replaceRestoreFile(target, []byte("new"), 0o600); err != nil { + t.Fatal(err) + } + if !hookCalled { + t.Fatal("parent-pinning race hook was not reached") + } + + link := filepath.Join(root, "reparse-parent") + if err := os.Symlink(parent, link); err != nil { + t.Skipf("Windows host does not permit symlink creation: %v", err) + } + if err := replaceRestoreFile(filepath.Join(link, "users.yaml"), []byte("unsafe"), 0o600); !errors.Is(err, safeio.ErrUnsafeFile) { + t.Fatalf("reparse-parent restore error = %v, want ErrUnsafeFile", err) + } +} diff --git a/tools/tht/internal/backup/restore_host.go b/tools/tht/internal/backup/restore_host.go index 0f6d6cba..e66e0dce 100644 --- a/tools/tht/internal/backup/restore_host.go +++ b/tools/tht/internal/backup/restore_host.go @@ -11,6 +11,7 @@ import ( "io" "os" "path/filepath" + "regexp" "strconv" "strings" "time" @@ -420,9 +421,10 @@ func verifyRestoreWorkspace(ctx context.Context, installation config.Installatio return errors.New("restored workspace registry returned an invalid result") } var payload struct { - Ready bool `json:"ready"` - State string `json:"state"` - Workspaces int `json:"workspaces"` + Ready bool `json:"ready"` + State string `json:"state"` + Workspaces int `json:"workspaces"` + Fingerprint string `json:"fingerprint"` } decoder := json.NewDecoder(strings.NewReader(result.Stdout)) decoder.DisallowUnknownFields() @@ -434,6 +436,7 @@ func verifyRestoreWorkspace(ctx context.Context, installation config.Installatio return errors.New("restored workspace registry returned an invalid result") } if !payload.Ready || payload.Workspaces < 0 || + !regexp.MustCompile(`^sha256:[0-9a-f]{64}$`).MatchString(payload.Fingerprint) || (payload.State != "active" && payload.State != "uninitialized") || (payload.State == "uninitialized" && payload.Workspaces != 0) { return errors.New("restored workspace registry did not pass integrity validation") diff --git a/tools/tht/internal/backup/restore_test.go b/tools/tht/internal/backup/restore_test.go index c4541e43..60d8a2b0 100644 --- a/tools/tht/internal/backup/restore_test.go +++ b/tools/tht/internal/backup/restore_test.go @@ -466,8 +466,109 @@ func TestRestoreRefusesActiveSessionsWithoutDrain(t *testing.T) { if !errors.Is(err, ErrActiveSessions) { t.Fatalf("restore error = %v, want active-session refusal", err) } - if restoredFiles != 0 || runner.stopCount != 0 || runner.startCount != 0 || !runner.running { - t.Fatalf("active-session refusal mutated target: files=%d stops=%d starts=%d running=%t", restoredFiles, runner.stopCount, runner.startCount, runner.running) + if restoredFiles != 0 || runner.stopCount != 0 || runner.startCount != 0 || !runner.running || runner.maintenance { + t.Fatalf("active-session refusal changed lifecycle state: files=%d stops=%d starts=%d running=%t maintenance=%t", restoredFiles, runner.stopCount, runner.startCount, runner.running, runner.maintenance) + } +} + +func TestRestoreCleansMaintenanceAfterActivationFailure(t *testing.T) { + installation := preflightTestInstallation(t) + backing := newBackupRunner(installation, true) + activationErr := errors.New("activation response lost") + runner := &restoreFailureRunner{ + fakeBackupRunner: backing, + failContains: "operator-command.js maintenance-activate", + err: activationErr, + beforeFailure: func() { backing.maintenance = true }, + } + deps := restoreTestDependencies(t, runner) + + _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: restoreArchive(t), Confirm: true}, deps) + if !errors.Is(err, activationErr) { + t.Fatalf("restore error = %v, want activation failure", err) + } + if backing.maintenance || !backing.running || runner.matchCount != 1 { + t.Fatalf("activation cleanup state: maintenance=%t running=%t matches=%d", backing.maintenance, backing.running, runner.matchCount) + } +} + +func TestRestoreRestartsAndCleansMaintenanceAfterStopFailure(t *testing.T) { + installation := preflightTestInstallation(t) + backing := newBackupRunner(installation, true) + stopErr := errors.New("stop response lost") + runner := &restoreFailureRunner{ + fakeBackupRunner: backing, + failSuffix: " stop", + err: stopErr, + beforeFailure: func() { + backing.stopCount++ + backing.running, backing.coreRunning = false, false + }, + } + deps := restoreTestDependencies(t, runner) + + _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: restoreArchive(t), Confirm: true}, deps) + if !errors.Is(err, stopErr) { + t.Fatalf("restore error = %v, want stop failure", err) + } + if backing.maintenance || !backing.running || backing.startCount != 1 { + t.Fatalf("stop cleanup state: maintenance=%t running=%t starts=%d", backing.maintenance, backing.running, backing.startCount) + } +} + +func TestRestoreCleansMaintenanceAfterMutationAndRollbackFailures(t *testing.T) { + installation := preflightTestInstallation(t) + for _, test := range []struct { + name string + recoveryErr error + }{ + {name: "mutation"}, + {name: "rollback", recoveryErr: errors.New("rollback failed")}, + } { + t.Run(test.name, func(t *testing.T) { + backing := newBackupRunner(installation, true) + deps := restoreTestDependencies(t, backing) + mutationErr := errors.New("mutation failed") + deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { + return mutationErr + } + deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error { + if test.recoveryErr == nil { + backing.running, backing.coreRunning = true, true + } + return test.recoveryErr + } + + _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: restoreArchive(t), Confirm: true}, deps) + if !errors.Is(err, mutationErr) || (test.recoveryErr != nil && !errors.Is(err, test.recoveryErr)) { + t.Fatalf("restore error = %v, want mutation and rollback failures", err) + } + if backing.maintenance || !backing.running { + t.Fatalf("failure cleanup state: maintenance=%t running=%t", backing.maintenance, backing.running) + } + }) + } +} + +func TestRestorePreservesDrainAndMaintenanceCleanupFailures(t *testing.T) { + installation := preflightTestInstallation(t) + backing := newBackupRunner(installation, true) + backing.sessionResponses = []string{`[{"status":"running","archived":false}]`} + cleanupErr := errors.New("maintenance cleanup failed") + runner := &restoreFailureRunner{ + fakeBackupRunner: backing, + failContains: "operator-command.js maintenance-deactivate", + err: cleanupErr, + beforeFailure: func() { backing.maintenance = false }, + } + deps := restoreTestDependencies(t, runner) + + _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: restoreArchive(t), Confirm: true}, deps) + if !errors.Is(err, ErrActiveSessions) || !errors.Is(err, cleanupErr) { + t.Fatalf("restore error = %v, want drain and cleanup failures", err) + } + if backing.maintenance || !backing.running { + t.Fatalf("cleanup failure state: maintenance=%t running=%t", backing.maintenance, backing.running) } } @@ -491,6 +592,37 @@ func (runner failStartRestoreRunner) Run(ctx context.Context, args []string, std return runner.fakeBackupRunner.Run(ctx, args, stdin) } +type restoreFailureRunner struct { + *fakeBackupRunner + failContains string + failSuffix string + err error + beforeFailure func() + matchCount int +} + +func (runner *restoreFailureRunner) Run(ctx context.Context, args []string, stdin io.Reader) (compose.Result, error) { + command := strings.Join(args, " ") + matches := runner.failContains != "" && strings.Contains(command, runner.failContains) + matches = matches || runner.failSuffix != "" && strings.HasSuffix(command, runner.failSuffix) + if matches { + runner.matchCount++ + if runner.beforeFailure != nil { + runner.beforeFailure() + } + return compose.Result{}, runner.err + } + return runner.fakeBackupRunner.Run(ctx, args, stdin) +} + +func (runner *restoreFailureRunner) Stream(ctx context.Context, args []string, stdin io.Reader, stdout io.Writer) (compose.Result, error) { + return runner.fakeBackupRunner.Stream(ctx, args, stdin, stdout) +} + +func (runner *restoreFailureRunner) SessionInventoryScope() string { + return runner.fakeBackupRunner.SessionInventoryScope() +} + type fakeRestoreLock struct { release func() } @@ -542,14 +674,15 @@ func (*workspaceVerificationRunner) SessionInventoryScope() string { return "min func TestVerifyRestoreWorkspaceUsesFixedNonNetworkOperatorPath(t *testing.T) { installation := preflightTestInstallation(t) + fingerprint := "sha256:" + strings.Repeat("a", 64) for _, test := range []struct { name string running bool payload string prefix string }{ - {name: "stopped uninitialized", payload: `{"ready":true,"state":"uninitialized","workspaces":0}`, prefix: "run --rm --no-deps --no-TTY core"}, - {name: "running active", running: true, payload: `{"ready":true,"state":"active","workspaces":1}`, prefix: "exec -T core"}, + {name: "stopped uninitialized", payload: fmt.Sprintf(`{"ready":true,"state":"uninitialized","workspaces":0,"fingerprint":%q}`, fingerprint), prefix: "run --rm --no-deps --no-TTY core"}, + {name: "running active", running: true, payload: fmt.Sprintf(`{"ready":true,"state":"active","workspaces":1,"fingerprint":%q}`, fingerprint), prefix: "exec -T core"}, } { t.Run(test.name, func(t *testing.T) { runner := &workspaceVerificationRunner{ @@ -573,6 +706,7 @@ func TestVerifyRestoreWorkspaceUsesFixedNonNetworkOperatorPath(t *testing.T) { func TestVerifyRestoreWorkspaceRejectsInvalidOperatorResults(t *testing.T) { installation := preflightTestInstallation(t) + valid := `{"ready":true,"state":"active","workspaces":1,"fingerprint":"sha256:` + strings.Repeat("a", 64) + `"}` for _, test := range []struct { name string result compose.Result @@ -580,11 +714,13 @@ func TestVerifyRestoreWorkspaceRejectsInvalidOperatorResults(t *testing.T) { }{ {name: "empty"}, {name: "malformed", result: compose.Result{Stdout: `{malformed`}}, - {name: "trailing document", result: compose.Result{Stdout: `{"ready":true,"state":"active","workspaces":1}{}`}}, - {name: "unknown field", result: compose.Result{Stdout: `{"ready":true,"state":"active","workspaces":1,"detail":"unsafe"}`}}, - {name: "not ready", result: compose.Result{Stdout: `{"ready":false,"state":"uninitialized","workspaces":0}`}}, - {name: "unknown state", result: compose.Result{Stdout: `{"ready":true,"state":"unknown","workspaces":0}`}}, - {name: "inconsistent count", result: compose.Result{Stdout: `{"ready":true,"state":"uninitialized","workspaces":1}`}}, + {name: "trailing document", result: compose.Result{Stdout: valid + `{}`}}, + {name: "unknown field", result: compose.Result{Stdout: strings.TrimSuffix(valid, "}") + `,"detail":"unsafe"}`}}, + {name: "not ready", result: compose.Result{Stdout: strings.Replace(valid, `"ready":true`, `"ready":false`, 1)}}, + {name: "unknown state", result: compose.Result{Stdout: strings.Replace(valid, `"state":"active"`, `"state":"unknown"`, 1)}}, + {name: "inconsistent count", result: compose.Result{Stdout: strings.Replace(valid, `"state":"active"`, `"state":"uninitialized"`, 1)}}, + {name: "missing fingerprint", result: compose.Result{Stdout: `{"ready":true,"state":"active","workspaces":1}`}}, + {name: "malformed fingerprint", result: compose.Result{Stdout: `{"ready":true,"state":"active","workspaces":1,"fingerprint":"sha256:not-a-digest"}`}}, {name: "nonzero", result: compose.Result{ExitCode: 2}, err: errors.New("exit status 2")}, } { t.Run(test.name, func(t *testing.T) {