fix(auth): harden restore verification transaction

This commit is contained in:
2026-08-18 00:58:12 +02:00
parent 0651f3316f
commit dee17893b4
10 changed files with 698 additions and 115 deletions
+1
View File
@@ -91,6 +91,7 @@ async function workspaceIntegrity(config: AppConfig): Promise<{
ready: true;
state: "uninitialized" | "active";
workspaces: number;
fingerprint: string;
}> {
const integrity = await new WorkspaceRegistry(config.workspaceRegistry).verifyStoredState();
return { ready: true, ...integrity };
+156 -51
View File
@@ -1,5 +1,5 @@
import { createHash, randomUUID } from "node:crypto";
import { lstatSync, readFileSync } from "node:fs";
import { lstatSync, readdirSync, readFileSync } from "node:fs";
import { mkdir, readdir, readFile, rename, rm, writeFile } from "node:fs/promises";
import { isAbsolute, join } from "node:path";
import { buildInstallationContract, renderWorkspaceDocs } from "./contracts.js";
@@ -34,6 +34,7 @@ export interface WorkspaceRevision {
export interface StoredWorkspaceIntegrity {
state: "uninitialized" | "active";
workspaces: number;
fingerprint: string;
}
export interface SessionRevisionLease {
@@ -188,43 +189,137 @@ export class WorkspaceRegistry {
/**
* Validate only persisted local registry state. Restore uses this path while the installation
* is stopped: it must neither contact Git nor turn a never-used registry into initialized state.
* The only never-initialized shape is an existing empty root. An initialized root is exactly
* repo/, snapshots/, state/, and locks/: locks contains repository.lock plus an empty
* empty-hooks/, state contains active.json plus an optional empty revision-leases/, and
* snapshots contains exact immutable commit snapshots plus an optional empty runtime/.
* Descendants may contain only ordinary directories and regular files; links and special files
* are rejected by the stable whole-tree fingerprint before any shape is accepted.
*/
async verifyStoredState(): Promise<StoredWorkspaceIntegrity> {
await this.repository.ensureLayout();
return await this.lock.run(async () => {
try {
const stateEntries = await readdir(this.repository.statePath, { withFileTypes: true });
if (stateEntries.some((entry) => (
entry.name !== "active.json" || !entry.isFile() || entry.isSymbolicLink()
))) throw new Error("workspace state directory is partial");
const snapshotEntries = await readdir(this.repository.snapshotsPath, { withFileTypes: true });
for (const entry of snapshotEntries) {
const isRuntime = entry.name === "runtime";
const isSnapshot = /^[0-9a-f]{40}$/.test(entry.name);
if ((!isRuntime && !isSnapshot) || !entry.isDirectory() || entry.isSymbolicLink()) {
throw new Error("workspace snapshot directory is partial");
}
}
const hasActiveState = stateEntries.length === 1;
if (!hasActiveState) {
if (snapshotEntries.some((entry) => entry.name !== "runtime") || this.storedPathExists(this.repository.repoPath)) {
throw new Error("workspace registry is partially initialized");
}
return { state: "uninitialized", workspaces: 0 };
}
const active = await this.activeState();
for (const entry of snapshotEntries) {
if (entry.name === "runtime" || entry.name === active.head) continue;
await this.snapshotState(entry.name);
}
return { state: "active", workspaces: active.revisions.length };
} catch (error) {
throw workspaceError(error);
try {
const before = await this.storedStateFingerprint();
const rootEntries = await readdir(this.repository.root, { withFileTypes: true });
if (rootEntries.length === 0) {
const after = await this.storedStateFingerprint();
if (after !== before) throw new Error("workspace registry changed during inspection");
return { state: "uninitialized", workspaces: 0, fingerprint: `sha256:${before}` };
}
});
this.assertExactDirectoryEntries(rootEntries, {
locks: "directory", repo: "directory", snapshots: "directory", state: "directory",
});
this.assertExactDirectoryEntries(
await readdir(this.repository.locksPath, { withFileTypes: true }),
{ "empty-hooks": "directory", "repository.lock": "file" },
);
this.assertExactDirectoryEntries(
await readdir(join(this.repository.locksPath, "empty-hooks"), { withFileTypes: true }),
{},
);
const stateEntries = await readdir(this.repository.statePath, { withFileTypes: true });
const stateShape: Record<string, "file" | "directory"> = { "active.json": "file" };
if (stateEntries.some((entry) => entry.name === "revision-leases")) {
stateShape["revision-leases"] = "directory";
}
this.assertExactDirectoryEntries(stateEntries, stateShape);
if (stateShape["revision-leases"] !== undefined) {
this.assertExactDirectoryEntries(
await readdir(join(this.repository.statePath, "revision-leases"), { withFileTypes: true }),
{},
);
}
const active = this.decodeActiveState(JSON.parse(await readFile(
join(this.repository.statePath, "active.json"), "utf8",
)));
const snapshotEntries = await readdir(this.repository.snapshotsPath, { withFileTypes: true });
if (snapshotEntries.length === 0) throw new Error("workspace snapshots are unavailable");
let activeSnapshotFound = false;
for (const entry of snapshotEntries) {
if (entry.name === "runtime") {
if (!entry.isDirectory() || entry.isSymbolicLink()) throw new Error("workspace runtime path is invalid");
this.assertExactDirectoryEntries(
await readdir(join(this.repository.snapshotsPath, "runtime"), { withFileTypes: true }),
{},
);
continue;
}
if (!/^[0-9a-f]{40}$/.test(entry.name) || !entry.isDirectory() || entry.isSymbolicLink()) {
throw new Error("workspace snapshot path is invalid");
}
const state = entry.name === active.head ? active : await this.readStoredSnapshotState(entry.name);
await this.assertSnapshotIntegrity(state, false, true);
if (entry.name === active.head) activeSnapshotFound = true;
}
if (!activeSnapshotFound) throw new Error("active workspace snapshot is unavailable");
const after = await this.storedStateFingerprint();
if (after !== before) throw new Error("workspace registry changed during inspection");
return { state: "active", workspaces: active.revisions.length, fingerprint: `sha256:${before}` };
} catch (error) {
throw workspaceError(error);
}
}
private assertExactDirectoryEntries(
entries: Array<{ name: string; isFile(): boolean; isDirectory(): boolean; isSymbolicLink(): boolean }>,
expected: Record<string, "file" | "directory">,
): void {
if (entries.length !== Object.keys(expected).length) throw new Error("workspace registry shape is invalid");
for (const entry of entries) {
const kind = expected[entry.name];
if (kind === undefined || entry.isSymbolicLink()
|| (kind === "file" && !entry.isFile())
|| (kind === "directory" && !entry.isDirectory())) {
throw new Error("workspace registry shape is invalid");
}
}
}
private async storedStateFingerprint(): Promise<string> {
const records: string[] = [];
let entries = 0;
let totalBytes = 0;
const visit = async (path: string, relative: string): Promise<void> => {
const before = lstatSync(path);
if (before.isSymbolicLink()) throw new Error("workspace registry link is invalid");
const metadata = [
before.dev, before.ino, before.mode, before.uid, before.gid,
before.size, before.mtimeMs, before.ctimeMs,
].join(":");
entries += 1;
if (entries > 65_536) throw new Error("workspace registry contains too many entries");
if (before.isDirectory()) {
records.push(`directory:${relative}:${metadata}`);
const children = await readdir(path);
children.sort();
for (const name of children) {
await visit(join(path, name), relative === "." ? name : `${relative}/${name}`);
}
} else if (before.isFile()) {
if (before.size > 256 * 1024 * 1024) throw new Error("workspace registry file is too large");
totalBytes += before.size;
if (totalBytes > 2 * 1024 * 1024 * 1024) throw new Error("workspace registry is too large");
const contents = await readFile(path);
records.push(`file:${relative}:${metadata}:${contents.length}:${digest(contents)}`);
} else {
throw new Error("workspace registry entry is invalid");
}
const after = lstatSync(path);
if (before.dev !== after.dev || before.ino !== after.ino || before.mode !== after.mode
|| before.uid !== after.uid || before.gid !== after.gid || before.size !== after.size
|| before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs) {
throw new Error("workspace registry changed during inspection");
}
};
await visit(this.repository.root, ".");
return digest(records.join("\n"));
}
private async readStoredSnapshotState(head: string): Promise<ActiveState> {
return this.decodeSnapshotManifest(await this.readSnapshotManifest(safeCommit(head)));
}
async read(id: string): Promise<{ workspace: WorkspaceDescriptor; revision: WorkspaceRevision }> {
@@ -696,7 +791,11 @@ export class WorkspaceRegistry {
return state;
}
private async assertSnapshotIntegrity(state: ActiveState): Promise<void> {
private async assertSnapshotIntegrity(
state: ActiveState,
verifyGitEvidence = true,
exactStoredShape = false,
): Promise<void> {
const directory = join(this.repository.snapshotsPath, state.head);
try {
const manifest = this.decodeSnapshotManifest(await this.readSnapshotManifest(state.head));
@@ -705,13 +804,33 @@ export class WorkspaceRegistry {
throw new Error("manifest state does not match active state");
}
await this.assertManifestFiles(directory, manifest.files, this.expectedSnapshotFiles(state, directory));
await this.assertSnapshotEvidenceContexts(state);
if (exactStoredShape) this.assertStoredSnapshotShape(directory, state);
if (verifyGitEvidence) await this.assertSnapshotEvidenceContexts(state);
} catch (error) {
if (error instanceof WorkspaceRegistryError) throw error;
throw new WorkspaceRegistryError("workspace_invalid", "Workspace snapshot integrity check failed");
}
}
private assertStoredSnapshotShape(directory: string, state: ActiveState): void {
const expected: Record<string, "file" | "directory"> = { "snapshot.json": "file" };
for (const revision of state.revisions) {
expected[`${revision.id}.yaml`] = "file";
expected[`${revision.id}.env.example`] = "file";
expected[`${revision.id}.md`] = "file";
const workspace = parseWorkspaceYaml(readFileSync(join(directory, `${revision.id}.yaml`), "utf8"));
if (workspace.evidence?.source.type === "filesystem") expected[revision.id] = "directory";
}
this.assertExactDirectoryEntries(readdirSync(directory, { withFileTypes: true }), expected);
for (const revision of state.revisions) {
if (expected[revision.id] !== "directory") continue;
this.assertExactDirectoryEntries(
readdirSync(join(directory, revision.id), { withFileTypes: true }),
{ evidence: "directory", "evidence.manifest.json": "file" },
);
}
}
private expectedSnapshotFiles(state: ActiveState, directory: string): string[] {
return state.revisions.flatMap((revision) => {
const names = [`${revision.id}.yaml`, `${revision.id}.env.example`, `${revision.id}.md`];
@@ -784,20 +903,6 @@ export class WorkspaceRegistry {
}
}
private storedPathExists(path: string): boolean {
try {
const entry = lstatSync(path);
if (!entry.isDirectory() || entry.isSymbolicLink()) {
throw new WorkspaceRegistryError("workspace_invalid", "Workspace registry path is invalid");
}
return true;
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") return false;
if (error instanceof WorkspaceRegistryError) throw error;
throw new WorkspaceRegistryError("workspace_invalid", "Workspace registry path is unavailable");
}
}
private pathIsMissing(path: string): boolean {
try {
lstatSync(path);
+116 -6
View File
@@ -1,7 +1,7 @@
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import {
chmodSync, existsSync, mkdtempSync, mkdirSync, readdirSync, readFileSync, rmSync, symlinkSync, writeFileSync,
chmodSync, existsSync, lstatSync, mkdtempSync, mkdirSync, readdirSync, readFileSync, rmSync, symlinkSync, utimesSync, writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -338,21 +338,70 @@ function persistedState(root: string, commit: string): { active: any; manifest:
};
}
function filesystemFingerprint(root: string): string {
if (!existsSync(root)) return "absent";
const records: string[] = [];
const visit = (path: string, relative: string): void => {
const entry = lstatSync(path);
const metadata = [
entry.dev, entry.ino, entry.mode, entry.uid, entry.gid,
entry.size, entry.mtimeMs, entry.ctimeMs,
].join(":");
if (entry.isSymbolicLink()) {
records.push(`link:${relative}:${metadata}`);
return;
}
if (entry.isDirectory()) {
records.push(`directory:${relative}:${metadata}`);
for (const name of readdirSync(path).sort()) visit(join(path, name), relative === "." ? name : `${relative}/${name}`);
return;
}
if (entry.isFile()) {
const contents = readFileSync(path);
records.push(`file:${relative}:${metadata}:${contents.length}:${createHash("sha256").update(contents).digest("hex")}`);
return;
}
records.push(`other:${relative}:${metadata}`);
};
visit(root, ".");
return createHash("sha256").update(records.join("\n")).digest("hex");
}
test("verifies a never-initialized registry without contacting its remote", async () => {
const root = mkdtempSync(join(tmpdir(), "thoth-workspace-registry-uninitialized-"));
temporaryRoots.push(root);
const registryRoot = join(root, "registry");
mkdirSync(registryRoot, { mode: 0o700 });
const registry = new WorkspaceRegistry(config(
registryRoot,
join(root, "missing-remote.git"),
));
const before = filesystemFingerprint(registryRoot);
await expect(registry.verifyStoredState()).resolves.toEqual({
state: "uninitialized",
workspaces: 0,
fingerprint: `sha256:${before}`,
});
expect(filesystemFingerprint(registryRoot)).toBe(before);
expect(existsSync(join(registryRoot, "repo"))).toBe(false);
expect(readdirSync(join(registryRoot, "state"))).toEqual([]);
expect(readdirSync(registryRoot)).toEqual([]);
});
test("never-initialized inspection refuses an absent or partial root without creating it", async () => {
const root = mkdtempSync(join(tmpdir(), "thoth-workspace-registry-absent-"));
temporaryRoots.push(root);
const registryRoot = join(root, "registry");
const registry = new WorkspaceRegistry(config(registryRoot, join(root, "missing-remote.git")));
await expect(registry.verifyStoredState()).rejects.toMatchObject({ code: "workspace_invalid" });
expect(filesystemFingerprint(registryRoot)).toBe("absent");
mkdirSync(join(registryRoot, "state"), { recursive: true });
const partial = filesystemFingerprint(registryRoot);
await expect(registry.verifyStoredState()).rejects.toMatchObject({ code: "workspace_invalid" });
expect(filesystemFingerprint(registryRoot)).toBe(partial);
});
test("verifies initialized snapshots and rejects partial or malformed persisted state", async () => {
@@ -361,10 +410,25 @@ test("verifies initialized snapshots and rejects partial or malformed persisted
const registry = new WorkspaceRegistry(config(registryRoot, remote.remote));
await registry.bootstrap();
await expect(registry.verifyStoredState()).resolves.toEqual({
state: "active",
workspaces: 1,
});
const before = filesystemFingerprint(registryRoot);
const savedPath = process.env.PATH;
process.env.PATH = join(remote.root, "no-executables");
try {
await expect(registry.verifyStoredState()).resolves.toEqual({
state: "active",
workspaces: 1,
fingerprint: `sha256:${before}`,
});
} finally {
if (savedPath === undefined) delete process.env.PATH;
else process.env.PATH = savedPath;
}
expect(filesystemFingerprint(registryRoot)).toBe(before);
const firstIntegrity = await registry.verifyStoredState();
utimesSync(join(registryRoot, "repo"), new Date(1_000), new Date(1_000));
const metadataIntegrity = await registry.verifyStoredState();
expect(metadataIntegrity.fingerprint).not.toBe(firstIntegrity.fingerprint);
rmSync(join(registryRoot, "state", "active.json"));
await expect(registry.verifyStoredState()).rejects.toMatchObject({ code: "workspace_invalid" });
@@ -373,6 +437,52 @@ test("verifies initialized snapshots and rejects partial or malformed persisted
await expect(registry.verifyStoredState()).rejects.toMatchObject({ code: "workspace_invalid" });
});
test("read-only inspection rejects extra components, links, and ephemeral runtime contents", async () => {
const remote = await fixture();
const registryRoot = join(remote.root, "registry");
const registry = new WorkspaceRegistry(config(registryRoot, remote.remote));
const status = await registry.bootstrap();
const assertHostile = async (setup: () => void, cleanup: () => void): Promise<void> => {
setup();
const before = filesystemFingerprint(registryRoot);
await expect(registry.verifyStoredState()).rejects.toMatchObject({ code: "workspace_invalid" });
expect(filesystemFingerprint(registryRoot)).toBe(before);
cleanup();
};
await assertHostile(
() => mkdirSync(join(registryRoot, "unexpected")),
() => rmSync(join(registryRoot, "unexpected"), { recursive: true }),
);
await assertHostile(
() => rmSync(join(registryRoot, "locks", "empty-hooks"), { recursive: true }),
() => mkdirSync(join(registryRoot, "locks", "empty-hooks")),
);
await assertHostile(
() => writeFileSync(join(registryRoot, "state", "unexpected.json"), "{}"),
() => rmSync(join(registryRoot, "state", "unexpected.json")),
);
await assertHostile(
() => writeFileSync(join(registryRoot, "snapshots", status.head!, "unexpected"), "extra"),
() => rmSync(join(registryRoot, "snapshots", status.head!, "unexpected")),
);
await assertHostile(
() => {
mkdirSync(join(registryRoot, "snapshots", "runtime"), { recursive: true });
writeFileSync(join(registryRoot, "snapshots", "runtime", "restored-secret.yaml"), "secret: forbidden");
},
() => rmSync(join(registryRoot, "snapshots", "runtime"), { recursive: true }),
);
await assertHostile(
() => symlinkSync(join(registryRoot, "state", "active.json"), join(registryRoot, "linked-active.json")),
() => rmSync(join(registryRoot, "linked-active.json")),
);
await assertHostile(
() => symlinkSync(join(registryRoot, "state", "active.json"), join(registryRoot, "repo", "linked-active.json")),
() => rmSync(join(registryRoot, "repo", "linked-active.json")),
);
});
test("rejects a catalog entry without a descriptor instead of creating a bootstrap slot", async () => {
const remote = await contentOnlyFixture();
const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote));
+108 -10
View File
@@ -29,6 +29,71 @@ task13_sha256_text() {
fi
}
task13_registry_filesystem_fingerprint() {
python3 - "$1" <<'PY'
import hashlib
import os
import stat
import sys
root = os.path.abspath(sys.argv[1])
records = []
entries = 0
total_bytes = 0
def snapshot(value):
return (
value.st_dev, value.st_ino, value.st_mode, value.st_uid, value.st_gid,
value.st_size, value.st_mtime_ns, value.st_ctime_ns,
)
def visit(path, relative):
global entries, total_bytes
before = os.lstat(path)
entries += 1
if entries > 65536:
raise SystemExit("registry fingerprint entry bound exceeded")
metadata = snapshot(before)
if stat.S_ISLNK(before.st_mode):
raise SystemExit("registry fingerprint rejected a symbolic link")
if stat.S_ISDIR(before.st_mode):
records.append(("directory", relative, metadata))
with os.scandir(path) as listing:
children = sorted((item.name for item in listing))
for name in children:
visit(os.path.join(path, name), name if relative == "." else relative + "/" + name)
elif stat.S_ISREG(before.st_mode):
if before.st_size > 256 * 1024 * 1024:
raise SystemExit("registry fingerprint file bound exceeded")
total_bytes += before.st_size
if total_bytes > 2 * 1024 * 1024 * 1024:
raise SystemExit("registry fingerprint total bound exceeded")
flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
descriptor = os.open(path, flags)
try:
opened = os.fstat(descriptor)
if snapshot(opened) != metadata:
raise SystemExit("registry changed while fingerprinting")
digest = hashlib.sha256()
while True:
chunk = os.read(descriptor, 1024 * 1024)
if not chunk:
break
digest.update(chunk)
finally:
os.close(descriptor)
records.append(("file", relative, metadata, digest.hexdigest()))
else:
raise SystemExit("registry fingerprint rejected a special file")
if snapshot(os.lstat(path)) != metadata:
raise SystemExit("registry changed while fingerprinting")
visit(root, ".")
encoded = repr(records).encode("utf-8")
print("sha256:" + hashlib.sha256(encoded).hexdigest())
PY
}
task13_sanitize() {
local line
while IFS= read -r line || [[ -n "$line" ]]; do
@@ -983,8 +1048,9 @@ PY
}
task13_assert_server_oidc_restore_verification() {
local archive frontend status diagnostics active_state_before restore_output restore_rc restore_cause
local archive frontend status diagnostics restore_output restore_rc restore_cause
local rollback_output rollback_rc rollback_sentinel provider_label checkpoint_leftover
local registry_before registry_after integrity_output
archive="$TASK13_TMP/server-oidc-restore-source.zip"
frontend="$(task13_frontend_address)"
task13_compose_logged "seed valid server authentication runtime excluded from restore" exec -T core node --input-type=module -e '
@@ -1004,12 +1070,6 @@ task13_assert_server_oidc_restore_verification() {
browserTransactionDigest: "d".repeat(64), browserTransactionTransport: "https",
});
'
active_state_before="$(task13_compose exec -T core node -e '
const fs = require("node:fs");
process.stdout.write(fs.existsSync("/data/workspace-registry/state/active.json") ? "present" : "absent");
')"
[[ "$active_state_before" == present || "$active_state_before" == absent ]] \
|| task13_fail "server restore fixture returned an invalid registry state diagnostic"
task13_compose_logged "stop server stack for OIDC restore" stop
rollback_sentinel="$TASK13_SERVER_DATA/task13-restore-rollback"
printf 'backup-state\n' >"$rollback_sentinel"
@@ -1059,6 +1119,22 @@ task13_assert_server_oidc_restore_verification() {
>>"$TASK13_LOG" 2>&1 || task13_log_failure "restored scoped fake OIDC provider readiness"
printf 'Task 13 server rollback injection passed: exit=%s; recovery checkpoint removed.\n' "$rollback_rc"
registry_before="$(task13_registry_filesystem_fingerprint "$TASK13_SERVER_REGISTRY")"
integrity_output="$TASK13_TMP/server-workspace-integrity.json"
if ! task13_compose run --rm --no-deps --no-TTY core \
node /app/backend/dist/operator-command.js workspace-integrity \
>"$integrity_output" 2>>"$TASK13_LOG"; then
task13_log_failure "stopped read-only workspace registry verification"
fi
node -e '
const value=JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));
const keys=Object.keys(value).sort().join(",");
if(keys!=="fingerprint,ready,state,workspaces"||value.ready!==true||value.state!=="uninitialized"||value.workspaces!==0||!/^sha256:[0-9a-f]{64}$/.test(value.fingerprint)) process.exit(1);
' "$integrity_output" || task13_fail "stopped registry inspector returned malformed or additional output"
registry_after="$(task13_registry_filesystem_fingerprint "$TASK13_SERVER_REGISTRY")"
[[ "$registry_after" == "$registry_before" ]] \
|| task13_fail "stopped restore verification mutated the workspace registry filesystem"
restore_output="$TASK13_TMP/server-oidc-restore.out"
set +e
"$TASK13_THT" --installation "$TASK13_INSTALLATION" restore "$archive" --yes \
@@ -1070,8 +1146,8 @@ task13_assert_server_oidc_restore_verification() {
if grep -Fq 'restore verification workspace: validate restored workspace registry' "$restore_output"; then
restore_cause=workspace-validator-rejected
fi
printf 'Task 13 stopped restore diagnostic: exit=%s active-state-before=%s cause=%s\n' \
"$restore_rc" "$active_state_before" "$restore_cause" >&2
printf 'Task 13 stopped restore diagnostic: exit=%s cause=%s\n' \
"$restore_rc" "$restore_cause" >&2
task13_sanitize <"$restore_output" | tail -n 8 >&2
return "$restore_rc"
fi
@@ -1086,7 +1162,8 @@ task13_assert_server_oidc_restore_verification() {
[[ "$status" == 401 ]] && break
sleep 1
done
[[ "$status" == 401 ]] || task13_fail "OIDC restore did not require browser reauthentication"
[[ "$status" == 401 ]] \
|| task13_fail "OIDC restore did not require browser reauthentication (HTTP ${status:-unavailable})"
task13_compose_logged "verify private empty server authentication state" exec -T core sh -ceu '
test "$(stat -c %a /data/auth)" = 700
test "$(stat -c %a /data/auth/sessions)" = 700
@@ -2030,6 +2107,26 @@ task13_self_test_server_release_contract() {
|| task13_fail "workflow lacks an outer timeout for the Linux server smoke"
}
task13_self_test_registry_fingerprint() {
local fixture fixture_escaped before after linked
fixture="$(mktemp -d "${TMPDIR:-/tmp}/thothii-task13-registry-fingerprint.XXXXXX")"
printf -v fixture_escaped '%q' "$fixture"
trap "rm -rf -- $fixture_escaped; trap - RETURN" RETURN
before="$(task13_registry_filesystem_fingerprint "$fixture")"
[[ "$before" =~ ^sha256:[0-9a-f]{64}$ ]] \
|| task13_fail "registry fingerprint did not return a bounded digest"
[[ "$(task13_registry_filesystem_fingerprint "$fixture")" == "$before" ]] \
|| task13_fail "registry fingerprint changed without a filesystem mutation"
mkdir "$fixture/locks"
after="$(task13_registry_filesystem_fingerprint "$fixture")"
[[ "$after" != "$before" ]] || task13_fail "registry fingerprint ignored a new directory"
linked="$fixture/linked"
ln -s "$fixture/locks" "$linked" 2>/dev/null || return 0
if task13_registry_filesystem_fingerprint "$fixture" >/dev/null 2>&1; then
task13_fail "registry fingerprint accepted a symbolic link"
fi
}
task13_self_test_source_contract() {
local root host_network push_command registry_function workflow uses_count pinned_uses_count
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
@@ -2094,6 +2191,7 @@ task13_self_test() {
task13_self_test_internal_semantic_offline_contract
task13_self_test_windows_release_contract
task13_self_test_server_release_contract
task13_self_test_registry_fingerprint
task13_self_test_source_contract
printf 'Task 13 smoke safety contracts passed.\n'
}
+28
View File
@@ -104,20 +104,43 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
return result, err
}
mutated := false
maintenanceAttempted := false
stopAttempted := false
defer func() {
if resultErr != nil && mutated {
if recoveryErr := deps.recover(context.Background(), installation, recovery, wasRunning); recoveryErr != nil {
resultErr = errors.Join(resultErr, fmt.Errorf("restore recovery checkpoint: %w", recoveryErr))
} else {
stopAttempted = false
}
}
if maintenanceAttempted {
cleanupContext := context.Background()
if wasRunning && stopAttempted {
if startErr := composeStartAndVerify(cleanupContext, installation, deps.runner); startErr != nil {
resultErr = errors.Join(resultErr, fmt.Errorf("restore maintenance cleanup restart: %w", startErr))
} else {
stopAttempted = false
}
}
if deactivateErr := maintenance(cleanupContext, installation, deps.runner, false); deactivateErr != nil {
resultErr = errors.Join(resultErr, fmt.Errorf("restore maintenance cleanup: %w", deactivateErr))
}
}
}()
if wasRunning {
// The activation command may take effect even when its response is lost. Track the attempt,
// not merely a successful return, so every subsequent path removes the admissions barrier.
maintenanceAttempted = true
if err := maintenance(ctx, installation, deps.runner, true); err != nil {
return result, err
}
if err := waitForNoActiveSessions(ctx, installation, deps.runner, request.Drain, deps.sleep); err != nil {
return result, err
}
// Compose may stop the core and then lose its response. Cleanup must therefore restart after
// any stop attempt, including a command that returns an error.
stopAttempted = true
if err := runCompose(ctx, installation, deps.runner, "stop"); err != nil {
return result, err
}
@@ -133,7 +156,12 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
if err := composeStartAndVerify(ctx, installation, deps.runner); err != nil {
return result, err
}
stopAttempted = false
result.Restarted = true
if err := maintenance(ctx, installation, deps.runner, false); err != nil {
return result, err
}
maintenanceAttempted = false
}
for _, name := range []string{"health", "doctor", "pi", "workspace"} {
check := deps.verify[name]
@@ -3,57 +3,45 @@
package backup
import (
"errors"
"os"
"path/filepath"
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
"golang.org/x/sys/windows"
)
func replaceRestoreFile(target string, contents []byte, mode os.FileMode) error {
if safeio.ValidateCanonicalPath(target) != nil || mode&os.ModeType != 0 || mode.Perm() == 0 {
// replaceRestoreFile deliberately supports only owner-private Windows parents. The retained
// native directory handle pins every ancestor, rejects reparse components, creates an owner-only
// temporary file, and publishes it by an NT RootDirectory-relative atomic rename. Installations
// whose restore targets do not satisfy that custody contract fail closed instead of falling back
// to a path-based replacement.
func replaceRestoreFile(target string, contents []byte, mode os.FileMode) (resultErr error) {
if safeio.ValidateCanonicalPath(target) != nil || mode&os.ModeType != 0 || mode.Perm() == 0 || len(contents) == 0 {
return safeio.ErrUnsafeFile
}
parent := filepath.Dir(target)
resolved, err := filepath.EvalSymlinks(parent)
if err != nil || resolved != parent || !safeWindowsRestoreTarget(target) {
parent, found, err := safeio.OpenPrivateDirectory(filepath.Dir(target), false)
if err != nil || !found || parent == nil {
return safeio.ErrUnsafeFile
}
temporary, err := os.CreateTemp(parent, ".tht-restore-*.tmp")
defer func() {
if closeErr := parent.Close(); closeErr != nil {
resultErr = safeio.ErrUnsafeFile
}
}()
safeio.NotifyPrivateDirectoryTestHookForTest("after-restore-parent-open")
if parent.Validate() != nil {
return safeio.ErrUnsafeFile
}
created, err := parent.CreateRegular(filepath.Base(target), contents)
if err != nil {
return safeio.ErrUnsafeFile
}
temporaryPath := temporary.Name()
defer os.Remove(temporaryPath)
if err := temporary.Chmod(mode.Perm()); err == nil {
_, err = temporary.Write(contents)
if !created {
if err := parent.ReplaceRegular(filepath.Base(target), contents); err != nil {
return safeio.ErrUnsafeFile
}
}
if err == nil {
err = temporary.Sync()
}
closeErr := temporary.Close()
if err == nil {
err = closeErr
}
if err != nil || !safeWindowsRestoreTarget(target) {
return safeio.ErrUnsafeFile
}
from, fromErr := windows.UTF16PtrFromString(temporaryPath)
to, toErr := windows.UTF16PtrFromString(target)
if fromErr != nil || toErr != nil {
return safeio.ErrUnsafeFile
}
if err := windows.MoveFileEx(from, to, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH); err != nil {
if parent.Validate() != nil {
return safeio.ErrUnsafeFile
}
return nil
}
func safeWindowsRestoreTarget(target string) bool {
info, err := os.Lstat(target)
if errors.Is(err, os.ErrNotExist) {
return true
}
return err == nil && info.Mode().IsRegular() && info.Mode()&os.ModeSymlink == 0
}
@@ -0,0 +1,31 @@
package backup
import (
"os"
"path/filepath"
"runtime"
"strings"
"testing"
)
func TestWindowsRestoreReplacementUsesPinnedHandleRelativeSafeIO(t *testing.T) {
_, current, _, ok := runtime.Caller(0)
if !ok {
t.Fatal("cannot locate Windows restore implementation")
}
contents, err := os.ReadFile(filepath.Join(filepath.Dir(current), "restore_file_windows.go"))
if err != nil {
t.Fatal(err)
}
source := string(contents)
for _, forbidden := range []string{"EvalSymlinks", "MoveFileEx", "CreateTemp"} {
if strings.Contains(source, forbidden) {
t.Fatalf("Windows restore replacement retains forbidden path-based primitive %q", forbidden)
}
}
for _, required := range []string{"OpenPrivateDirectory", "CreateRegular", "ReplaceRegular", "after-restore-parent-open"} {
if !strings.Contains(source, required) {
t.Fatalf("Windows restore replacement omits pinned safe-I/O primitive %q", required)
}
}
}
@@ -0,0 +1,83 @@
//go:build windows
package backup
import (
"errors"
"os"
"path/filepath"
"testing"
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
)
func TestReplaceRestoreFileWindowsCreatesAndReplacesOwnerOnlyRegular(t *testing.T) {
parent := filepath.Join(t.TempDir(), "private")
if err := os.Mkdir(parent, 0o700); err != nil {
t.Fatal(err)
}
if err := safeio.ProtectPrivateDirectory(parent); err != nil {
t.Fatal(err)
}
target := filepath.Join(parent, "auth.yaml")
if err := replaceRestoreFile(target, []byte("created"), 0o600); err != nil {
t.Fatal(err)
}
if err := safeio.ValidatePrivateRegular(target); err != nil {
t.Fatalf("created restore file is not owner-only: %v", err)
}
if err := replaceRestoreFile(target, []byte("replacement"), 0o600); err != nil {
t.Fatal(err)
}
if err := safeio.ValidatePrivateRegular(target); err != nil {
t.Fatalf("replacement restore file is not owner-only: %v", err)
}
contents, err := os.ReadFile(target)
if err != nil || string(contents) != "replacement" {
t.Fatalf("replacement contents = %q, error = %v", contents, err)
}
}
func TestReplaceRestoreFileWindowsPinsParentAndRejectsReparsePath(t *testing.T) {
root := t.TempDir()
parent := filepath.Join(root, "private")
if err := os.Mkdir(parent, 0o700); err != nil {
t.Fatal(err)
}
if err := safeio.ProtectPrivateDirectory(parent); err != nil {
t.Fatal(err)
}
target := filepath.Join(parent, "users.yaml")
if err := os.WriteFile(target, []byte("old"), 0o600); err != nil {
t.Fatal(err)
}
if err := safeio.ProtectPrivateRegular(target); err != nil {
t.Fatal(err)
}
renamed := parent + "-renamed"
var hookCalled bool
restoreHook := safeio.SetPrivateDirectoryTestHookForTest(func(stage string) {
if stage != "after-restore-parent-open" {
return
}
hookCalled = true
if err := os.Rename(parent, renamed); err == nil {
t.Fatal("parent rename succeeded while restore retained its directory handles")
}
})
defer restoreHook()
if err := replaceRestoreFile(target, []byte("new"), 0o600); err != nil {
t.Fatal(err)
}
if !hookCalled {
t.Fatal("parent-pinning race hook was not reached")
}
link := filepath.Join(root, "reparse-parent")
if err := os.Symlink(parent, link); err != nil {
t.Skipf("Windows host does not permit symlink creation: %v", err)
}
if err := replaceRestoreFile(filepath.Join(link, "users.yaml"), []byte("unsafe"), 0o600); !errors.Is(err, safeio.ErrUnsafeFile) {
t.Fatalf("reparse-parent restore error = %v, want ErrUnsafeFile", err)
}
}
+6 -3
View File
@@ -11,6 +11,7 @@ import (
"io"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
@@ -420,9 +421,10 @@ func verifyRestoreWorkspace(ctx context.Context, installation config.Installatio
return errors.New("restored workspace registry returned an invalid result")
}
var payload struct {
Ready bool `json:"ready"`
State string `json:"state"`
Workspaces int `json:"workspaces"`
Ready bool `json:"ready"`
State string `json:"state"`
Workspaces int `json:"workspaces"`
Fingerprint string `json:"fingerprint"`
}
decoder := json.NewDecoder(strings.NewReader(result.Stdout))
decoder.DisallowUnknownFields()
@@ -434,6 +436,7 @@ func verifyRestoreWorkspace(ctx context.Context, installation config.Installatio
return errors.New("restored workspace registry returned an invalid result")
}
if !payload.Ready || payload.Workspaces < 0 ||
!regexp.MustCompile(`^sha256:[0-9a-f]{64}$`).MatchString(payload.Fingerprint) ||
(payload.State != "active" && payload.State != "uninitialized") ||
(payload.State == "uninitialized" && payload.Workspaces != 0) {
return errors.New("restored workspace registry did not pass integrity validation")
+145 -9
View File
@@ -466,8 +466,109 @@ func TestRestoreRefusesActiveSessionsWithoutDrain(t *testing.T) {
if !errors.Is(err, ErrActiveSessions) {
t.Fatalf("restore error = %v, want active-session refusal", err)
}
if restoredFiles != 0 || runner.stopCount != 0 || runner.startCount != 0 || !runner.running {
t.Fatalf("active-session refusal mutated target: files=%d stops=%d starts=%d running=%t", restoredFiles, runner.stopCount, runner.startCount, runner.running)
if restoredFiles != 0 || runner.stopCount != 0 || runner.startCount != 0 || !runner.running || runner.maintenance {
t.Fatalf("active-session refusal changed lifecycle state: files=%d stops=%d starts=%d running=%t maintenance=%t", restoredFiles, runner.stopCount, runner.startCount, runner.running, runner.maintenance)
}
}
func TestRestoreCleansMaintenanceAfterActivationFailure(t *testing.T) {
installation := preflightTestInstallation(t)
backing := newBackupRunner(installation, true)
activationErr := errors.New("activation response lost")
runner := &restoreFailureRunner{
fakeBackupRunner: backing,
failContains: "operator-command.js maintenance-activate",
err: activationErr,
beforeFailure: func() { backing.maintenance = true },
}
deps := restoreTestDependencies(t, runner)
_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: restoreArchive(t), Confirm: true}, deps)
if !errors.Is(err, activationErr) {
t.Fatalf("restore error = %v, want activation failure", err)
}
if backing.maintenance || !backing.running || runner.matchCount != 1 {
t.Fatalf("activation cleanup state: maintenance=%t running=%t matches=%d", backing.maintenance, backing.running, runner.matchCount)
}
}
func TestRestoreRestartsAndCleansMaintenanceAfterStopFailure(t *testing.T) {
installation := preflightTestInstallation(t)
backing := newBackupRunner(installation, true)
stopErr := errors.New("stop response lost")
runner := &restoreFailureRunner{
fakeBackupRunner: backing,
failSuffix: " stop",
err: stopErr,
beforeFailure: func() {
backing.stopCount++
backing.running, backing.coreRunning = false, false
},
}
deps := restoreTestDependencies(t, runner)
_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: restoreArchive(t), Confirm: true}, deps)
if !errors.Is(err, stopErr) {
t.Fatalf("restore error = %v, want stop failure", err)
}
if backing.maintenance || !backing.running || backing.startCount != 1 {
t.Fatalf("stop cleanup state: maintenance=%t running=%t starts=%d", backing.maintenance, backing.running, backing.startCount)
}
}
func TestRestoreCleansMaintenanceAfterMutationAndRollbackFailures(t *testing.T) {
installation := preflightTestInstallation(t)
for _, test := range []struct {
name string
recoveryErr error
}{
{name: "mutation"},
{name: "rollback", recoveryErr: errors.New("rollback failed")},
} {
t.Run(test.name, func(t *testing.T) {
backing := newBackupRunner(installation, true)
deps := restoreTestDependencies(t, backing)
mutationErr := errors.New("mutation failed")
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
return mutationErr
}
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
if test.recoveryErr == nil {
backing.running, backing.coreRunning = true, true
}
return test.recoveryErr
}
_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: restoreArchive(t), Confirm: true}, deps)
if !errors.Is(err, mutationErr) || (test.recoveryErr != nil && !errors.Is(err, test.recoveryErr)) {
t.Fatalf("restore error = %v, want mutation and rollback failures", err)
}
if backing.maintenance || !backing.running {
t.Fatalf("failure cleanup state: maintenance=%t running=%t", backing.maintenance, backing.running)
}
})
}
}
func TestRestorePreservesDrainAndMaintenanceCleanupFailures(t *testing.T) {
installation := preflightTestInstallation(t)
backing := newBackupRunner(installation, true)
backing.sessionResponses = []string{`[{"status":"running","archived":false}]`}
cleanupErr := errors.New("maintenance cleanup failed")
runner := &restoreFailureRunner{
fakeBackupRunner: backing,
failContains: "operator-command.js maintenance-deactivate",
err: cleanupErr,
beforeFailure: func() { backing.maintenance = false },
}
deps := restoreTestDependencies(t, runner)
_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: restoreArchive(t), Confirm: true}, deps)
if !errors.Is(err, ErrActiveSessions) || !errors.Is(err, cleanupErr) {
t.Fatalf("restore error = %v, want drain and cleanup failures", err)
}
if backing.maintenance || !backing.running {
t.Fatalf("cleanup failure state: maintenance=%t running=%t", backing.maintenance, backing.running)
}
}
@@ -491,6 +592,37 @@ func (runner failStartRestoreRunner) Run(ctx context.Context, args []string, std
return runner.fakeBackupRunner.Run(ctx, args, stdin)
}
type restoreFailureRunner struct {
*fakeBackupRunner
failContains string
failSuffix string
err error
beforeFailure func()
matchCount int
}
func (runner *restoreFailureRunner) Run(ctx context.Context, args []string, stdin io.Reader) (compose.Result, error) {
command := strings.Join(args, " ")
matches := runner.failContains != "" && strings.Contains(command, runner.failContains)
matches = matches || runner.failSuffix != "" && strings.HasSuffix(command, runner.failSuffix)
if matches {
runner.matchCount++
if runner.beforeFailure != nil {
runner.beforeFailure()
}
return compose.Result{}, runner.err
}
return runner.fakeBackupRunner.Run(ctx, args, stdin)
}
func (runner *restoreFailureRunner) Stream(ctx context.Context, args []string, stdin io.Reader, stdout io.Writer) (compose.Result, error) {
return runner.fakeBackupRunner.Stream(ctx, args, stdin, stdout)
}
func (runner *restoreFailureRunner) SessionInventoryScope() string {
return runner.fakeBackupRunner.SessionInventoryScope()
}
type fakeRestoreLock struct {
release func()
}
@@ -542,14 +674,15 @@ func (*workspaceVerificationRunner) SessionInventoryScope() string { return "min
func TestVerifyRestoreWorkspaceUsesFixedNonNetworkOperatorPath(t *testing.T) {
installation := preflightTestInstallation(t)
fingerprint := "sha256:" + strings.Repeat("a", 64)
for _, test := range []struct {
name string
running bool
payload string
prefix string
}{
{name: "stopped uninitialized", payload: `{"ready":true,"state":"uninitialized","workspaces":0}`, prefix: "run --rm --no-deps --no-TTY core"},
{name: "running active", running: true, payload: `{"ready":true,"state":"active","workspaces":1}`, prefix: "exec -T core"},
{name: "stopped uninitialized", payload: fmt.Sprintf(`{"ready":true,"state":"uninitialized","workspaces":0,"fingerprint":%q}`, fingerprint), prefix: "run --rm --no-deps --no-TTY core"},
{name: "running active", running: true, payload: fmt.Sprintf(`{"ready":true,"state":"active","workspaces":1,"fingerprint":%q}`, fingerprint), prefix: "exec -T core"},
} {
t.Run(test.name, func(t *testing.T) {
runner := &workspaceVerificationRunner{
@@ -573,6 +706,7 @@ func TestVerifyRestoreWorkspaceUsesFixedNonNetworkOperatorPath(t *testing.T) {
func TestVerifyRestoreWorkspaceRejectsInvalidOperatorResults(t *testing.T) {
installation := preflightTestInstallation(t)
valid := `{"ready":true,"state":"active","workspaces":1,"fingerprint":"sha256:` + strings.Repeat("a", 64) + `"}`
for _, test := range []struct {
name string
result compose.Result
@@ -580,11 +714,13 @@ func TestVerifyRestoreWorkspaceRejectsInvalidOperatorResults(t *testing.T) {
}{
{name: "empty"},
{name: "malformed", result: compose.Result{Stdout: `{malformed`}},
{name: "trailing document", result: compose.Result{Stdout: `{"ready":true,"state":"active","workspaces":1}{}`}},
{name: "unknown field", result: compose.Result{Stdout: `{"ready":true,"state":"active","workspaces":1,"detail":"unsafe"}`}},
{name: "not ready", result: compose.Result{Stdout: `{"ready":false,"state":"uninitialized","workspaces":0}`}},
{name: "unknown state", result: compose.Result{Stdout: `{"ready":true,"state":"unknown","workspaces":0}`}},
{name: "inconsistent count", result: compose.Result{Stdout: `{"ready":true,"state":"uninitialized","workspaces":1}`}},
{name: "trailing document", result: compose.Result{Stdout: valid + `{}`}},
{name: "unknown field", result: compose.Result{Stdout: strings.TrimSuffix(valid, "}") + `,"detail":"unsafe"}`}},
{name: "not ready", result: compose.Result{Stdout: strings.Replace(valid, `"ready":true`, `"ready":false`, 1)}},
{name: "unknown state", result: compose.Result{Stdout: strings.Replace(valid, `"state":"active"`, `"state":"unknown"`, 1)}},
{name: "inconsistent count", result: compose.Result{Stdout: strings.Replace(valid, `"state":"active"`, `"state":"uninitialized"`, 1)}},
{name: "missing fingerprint", result: compose.Result{Stdout: `{"ready":true,"state":"active","workspaces":1}`}},
{name: "malformed fingerprint", result: compose.Result{Stdout: `{"ready":true,"state":"active","workspaces":1,"fingerprint":"sha256:not-a-digest"}`}},
{name: "nonzero", result: compose.Result{ExitCode: 2}, err: errors.New("exit status 2")},
} {
t.Run(test.name, func(t *testing.T) {