fix: harden runtime config helper protocol and lifecycle
This commit is contained in:
@@ -54,6 +54,10 @@ interface SnapshotIdentity {
|
||||
descriptorIno: string;
|
||||
}
|
||||
interface PublishedResponse {
|
||||
protocol_version: 1;
|
||||
kind: "publication";
|
||||
workspace_id: string;
|
||||
workspace_revision: string;
|
||||
path: string;
|
||||
manifestPath: string;
|
||||
manifest: string;
|
||||
@@ -129,6 +133,120 @@ function parseInstallationOverlay(path: string): RuntimeInstallationOverlay {
|
||||
|
||||
function digest(data: string | Buffer): string { return createHash("sha256").update(data).digest("hex"); }
|
||||
|
||||
|
||||
export interface BoundedHelperOptions {
|
||||
cwd: string;
|
||||
env: NodeJS.ProcessEnv;
|
||||
action: string;
|
||||
payload: string;
|
||||
timeoutMs?: number;
|
||||
outputLimit?: number;
|
||||
/** Package-internal seam used by lifecycle tests; production uses node spawn. */
|
||||
spawnProcess?: typeof spawn;
|
||||
}
|
||||
|
||||
/** Run one JSON helper with bounded output and deterministic child cleanup. */
|
||||
export async function runBoundedHelper(
|
||||
executable: string,
|
||||
args: readonly string[],
|
||||
options: BoundedHelperOptions,
|
||||
): Promise<unknown> {
|
||||
const timeoutMs = options.timeoutMs ?? 10_000;
|
||||
const outputLimit = options.outputLimit ?? 16 * 1024 * 1024;
|
||||
const spawnProcess = options.spawnProcess ?? spawn;
|
||||
const child = spawnProcess(executable, [...args], {
|
||||
cwd: options.cwd,
|
||||
env: options.env,
|
||||
detached: true,
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
let closed = false;
|
||||
let finishing = false;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
let closeResolve: () => void = () => undefined;
|
||||
const closePromise = new Promise<void>((resolveClose) => { closeResolve = resolveClose; });
|
||||
|
||||
const killGroup = () => {
|
||||
try {
|
||||
if (child.pid !== undefined && child.pid !== null) process.kill(-child.pid, "SIGKILL");
|
||||
} catch {
|
||||
try { child.kill("SIGKILL"); } catch { /* process already gone */ }
|
||||
}
|
||||
};
|
||||
const destroyStreams = () => {
|
||||
try { child.stdin?.destroy(); } catch { /* already closed */ }
|
||||
try { child.stdout?.destroy(); } catch { /* already closed */ }
|
||||
try { child.stderr?.destroy(); } catch { /* already closed */ }
|
||||
};
|
||||
const awaitClose = async () => {
|
||||
if (closed) return;
|
||||
// A descendant can keep stdio open even after the direct child exits. Streams
|
||||
// are destroyed before this bounded reap wait so the backend cannot hang.
|
||||
await Promise.race([closePromise, new Promise<void>((resolveWait) => setTimeout(resolveWait, 1_000))]);
|
||||
};
|
||||
|
||||
return new Promise<unknown>((resolveResult, rejectResult) => {
|
||||
const finish = async (error?: Error, value?: unknown, terminate = false) => {
|
||||
if (finishing) return;
|
||||
finishing = true;
|
||||
if (timer !== undefined) clearTimeout(timer);
|
||||
if (terminate) killGroup();
|
||||
destroyStreams();
|
||||
await awaitClose();
|
||||
destroyStreams();
|
||||
if (error) rejectResult(error); else resolveResult(value);
|
||||
};
|
||||
const append = (target: "stdout" | "stderr", data: Buffer) => {
|
||||
const next = target === "stdout" ? stdout + data.toString() : stderr + data.toString();
|
||||
if (Buffer.byteLength(next) > outputLimit) {
|
||||
void finish(new Error(`runtime config ${options.action} output exceeded limit`), undefined, true);
|
||||
return;
|
||||
}
|
||||
if (target === "stdout") stdout = next; else stderr = next;
|
||||
};
|
||||
|
||||
// Every stream gets an error listener before any data is written. In
|
||||
// particular, EPIPE from end() must become the same bounded failure path.
|
||||
child.stdin?.on("error", (error) => void finish(error instanceof Error ? error : new Error(String(error)), undefined, true));
|
||||
child.stdout?.on("error", (error) => void finish(error instanceof Error ? error : new Error(String(error)), undefined, true));
|
||||
child.stderr?.on("error", (error) => void finish(error instanceof Error ? error : new Error(String(error)), undefined, true));
|
||||
child.stdout?.on("data", (data: Buffer) => append("stdout", data));
|
||||
child.stderr?.on("data", (data: Buffer) => append("stderr", data));
|
||||
child.on("error", (error) => void finish(error instanceof Error ? error : new Error(String(error)), undefined, true));
|
||||
child.on("close", (code) => {
|
||||
closed = true;
|
||||
closeResolve();
|
||||
if (finishing) return;
|
||||
if (code !== 0) {
|
||||
let detail = stderr.trim() || stdout.trim() || `runtime config ${options.action} failed`;
|
||||
try { detail = (JSON.parse(stdout) as { error?: string }).error ?? detail; } catch { /* preserve detail */ }
|
||||
void finish(new Error(detail));
|
||||
return;
|
||||
}
|
||||
try { void finish(undefined, JSON.parse(stdout)); }
|
||||
catch { void finish(new Error(`runtime config ${options.action} returned invalid JSON`)); }
|
||||
});
|
||||
timer = setTimeout(() => {
|
||||
void finish(new Error(`runtime config ${options.action} timed out`), undefined, true);
|
||||
}, timeoutMs);
|
||||
try {
|
||||
// The listener above is intentionally installed before end(), since a
|
||||
// helper may close its input immediately and emit EPIPE synchronously.
|
||||
child.stdin?.end(options.payload);
|
||||
} catch (error) {
|
||||
void finish(error instanceof Error ? error : new Error(String(error)), undefined, true);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function sameBinding(left: unknown, right: unknown): boolean {
|
||||
if (!left || typeof left !== "object" || Array.isArray(left) || !right || typeof right !== "object" || Array.isArray(right)) return false;
|
||||
const a = left as Record<string, unknown>; const b = right as Record<string, unknown>;
|
||||
return a.workspace_id === b.workspace_id && a.config_fingerprint === b.config_fingerprint && a.input_fingerprint === b.input_fingerprint;
|
||||
}
|
||||
|
||||
export class WorkspaceRuntimeConfigLeaseFactory {
|
||||
private readonly env: NodeJS.ProcessEnv;
|
||||
private readonly secretRoots: readonly string[];
|
||||
@@ -186,40 +304,32 @@ export class WorkspaceRuntimeConfigLeaseFactory {
|
||||
const executable = existsSync(python) ? python : existsSync(projectPython) ? projectPython : (process.env.PYTHON ?? "python3");
|
||||
const helperArgs = existsSync(modulePath) ? [modulePath] : ["-m", "tht.runtime_config_lease_io"];
|
||||
const env = { ...this.env };
|
||||
// Never pass ambient capability variables to binding/snapshot/publication.
|
||||
// Capability variables are never inherited. Fault seams are only available in
|
||||
// tests, and are copied explicitly rather than forwarding ambient state.
|
||||
for (const key of Object.keys(env)) {
|
||||
if ((key.startsWith("THT_RUNTIME_CONFIG_") && !["THT_RUNTIME_CONFIG_FSYNC_FAIL", "THT_RUNTIME_CONFIG_RENAME_FAIL"].includes(key)) || key.startsWith("THT_CONFIG_")) delete env[key];
|
||||
if (key.startsWith("THT_RUNTIME_CONFIG_") || key.startsWith("THT_CONFIG_")) delete env[key];
|
||||
}
|
||||
if (env.NODE_ENV === "test") {
|
||||
for (const key of ["THT_RUNTIME_CONFIG_FSYNC_FAIL", "THT_RUNTIME_CONFIG_RENAME_FAIL"]) {
|
||||
const value = this.env[key];
|
||||
if (value !== undefined) env[key] = value;
|
||||
}
|
||||
}
|
||||
env.PYTHONPATH = [this.input.harnessDir, dirname(dirname(modulePath)), env.PYTHONPATH].filter(Boolean).join(":");
|
||||
const payload = JSON.stringify({ protocol_version: 1, action, ...extra });
|
||||
const timeoutMs = 10_000;
|
||||
return await new Promise((resolveResult, reject) => {
|
||||
const child = spawn(executable, helperArgs, { cwd: this.input.harnessDir, env, detached: true, stdio: ["pipe", "pipe", "pipe"] });
|
||||
let stdout = ""; let stderr = ""; let settled = false;
|
||||
const finish = (error?: Error, value?: unknown) => { if (settled) return; settled = true; clearTimeout(timer); error ? reject(error) : resolveResult(value); };
|
||||
const kill = () => { try { process.kill(-child.pid!, "SIGKILL"); } catch { try { child.kill("SIGKILL"); } catch { /* gone */ } } };
|
||||
const timer = setTimeout(() => { kill(); finish(new Error(`runtime config ${action} timed out`)); }, timeoutMs);
|
||||
const append = (target: "stdout" | "stderr", data: Buffer) => {
|
||||
const next = target === "stdout" ? stdout + data.toString() : stderr + data.toString();
|
||||
if (next.length > 16 * 1024 * 1024) { kill(); finish(new Error(`runtime config ${action} output exceeded limit`)); return; }
|
||||
if (target === "stdout") stdout = next; else stderr = next;
|
||||
};
|
||||
child.stdout.on("data", (d: Buffer) => append("stdout", d)); child.stderr.on("data", (d: Buffer) => append("stderr", d));
|
||||
child.on("error", (error) => finish(error));
|
||||
child.on("close", (code) => {
|
||||
if (code !== 0) { let detail = stderr.trim() || stdout.trim() || `runtime config ${action} failed`; try { detail = (JSON.parse(stdout) as {error?: string}).error ?? detail; } catch { /* preserve detail */ } finish(new Error(detail)); return; }
|
||||
try { finish(undefined, JSON.parse(stdout)); } catch { finish(new Error(`runtime config ${action} returned invalid JSON`)); }
|
||||
});
|
||||
child.stdin.end(payload);
|
||||
return runBoundedHelper(executable, helperArgs, {
|
||||
cwd: this.input.harnessDir, env, action, payload,
|
||||
});
|
||||
}
|
||||
|
||||
private async computeBinding(content: string): Promise<Record<string, string>> {
|
||||
const value = await this.helper("binding", { config_hex: Buffer.from(content).toString("hex") });
|
||||
if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).sort().join(",") !== "config_fingerprint,input_fingerprint,workspace_id") throw new Error("runtime config binding helper returned malformed output");
|
||||
if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).sort().join(",") !== "config_fingerprint,input_fingerprint,kind,protocol_version,workspace_id") throw new Error("runtime config binding helper returned malformed output");
|
||||
const record = value as Record<string, unknown>;
|
||||
if (Object.values(record).some((v) => typeof v !== "string")) throw new Error("runtime config binding helper returned malformed output");
|
||||
return record as Record<string, string>;
|
||||
if (record.protocol_version !== 1 || record.kind !== "binding"
|
||||
|| Object.values(record).some((v) => typeof v !== "string" && typeof v !== "number")) throw new Error("runtime config binding helper returned malformed output");
|
||||
if (typeof record.workspace_id !== "string" || typeof record.config_fingerprint !== "string" || typeof record.input_fingerprint !== "string") throw new Error("runtime config binding helper returned malformed output");
|
||||
return { workspace_id: record.workspace_id, config_fingerprint: record.config_fingerprint, input_fingerprint: record.input_fingerprint };
|
||||
}
|
||||
|
||||
private async publishSecure(workspaceId: string, revision: string, content: string, manifestBase: Record<string, unknown>): Promise<PublishedResponse> {
|
||||
@@ -227,17 +337,33 @@ export class WorkspaceRuntimeConfigLeaseFactory {
|
||||
workspace_revision: revision, config_hex: Buffer.from(content).toString("hex"), manifest_base: manifestBase });
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("runtime config publish helper returned malformed output");
|
||||
const result = value as Record<string, unknown>;
|
||||
if (Object.keys(result).sort().join(",") !== "dev,ino,manifest,manifestPath,manifest_sha256,path") throw new Error("runtime config publish helper returned malformed output");
|
||||
if (Object.keys(result).sort().join(",") !== "dev,ino,kind,manifest,manifestPath,manifest_sha256,path,protocol_version,workspace_id,workspace_revision") throw new Error("runtime config publish helper returned malformed output");
|
||||
let expectedRoot = resolve(this.input.dataRoot);
|
||||
if (process.platform === "darwin" && (expectedRoot === "/var" || expectedRoot.startsWith("/var/") || expectedRoot === "/tmp" || expectedRoot.startsWith("/tmp/"))) expectedRoot = `/private${expectedRoot}`;
|
||||
const expectedPath = join(expectedRoot, "sessions", workspaceId, "preprocessing", "runtime-config", `${revision}.yaml`);
|
||||
const expectedManifestPath = join(expectedRoot, "sessions", workspaceId, "preprocessing", "runtime-config-manifests", `${revision}.json`);
|
||||
if (result.path !== expectedPath || result.manifestPath !== expectedManifestPath
|
||||
|| typeof result.path !== "string" || !isAbsolute(result.path) || normalize(result.path) !== result.path
|
||||
|| typeof result.manifestPath !== "string" || !isAbsolute(result.manifestPath) || normalize(result.manifestPath) !== result.manifestPath
|
||||
|| result.workspace_id !== undefined || typeof result.manifest !== "string"
|
||||
|| typeof result.manifest_sha256 !== "string" || !/^[0-9a-f]{64}$/.test(result.manifest_sha256)
|
||||
if (result.protocol_version !== 1 || result.kind !== "publication"
|
||||
|| result.workspace_id !== workspaceId || result.workspace_revision !== revision
|
||||
|| typeof result.path !== "string" || result.path !== expectedPath || !isAbsolute(result.path) || normalize(result.path) !== result.path
|
||||
|| typeof result.manifestPath !== "string" || result.manifestPath !== expectedManifestPath || !isAbsolute(result.manifestPath) || normalize(result.manifestPath) !== result.manifestPath
|
||||
|| typeof result.manifest !== "string" || typeof result.manifest_sha256 !== "string" || !/^[0-9a-f]{64}$/.test(result.manifest_sha256)
|
||||
|| digest(result.manifest) !== result.manifest_sha256
|
||||
|| typeof result.dev !== "number" || !Number.isSafeInteger(result.dev) || typeof result.ino !== "number" || !Number.isSafeInteger(result.ino)) throw new Error("runtime config publish helper returned malformed output");
|
||||
let manifest: unknown;
|
||||
try { manifest = JSON.parse(result.manifest); } catch { throw new Error("runtime config publish helper returned malformed output"); }
|
||||
if (!manifest || typeof manifest !== "object" || Array.isArray(manifest)) throw new Error("runtime config publish helper returned malformed output");
|
||||
const manifestRecord = manifest as Record<string, unknown>;
|
||||
if (manifestRecord.version !== 1
|
||||
|| manifestRecord.workspace_id !== workspaceId || manifestRecord.workspace_revision !== revision
|
||||
|| manifestRecord.descriptor_git_blob !== manifestBase.descriptor_git_blob
|
||||
|| manifestRecord.descriptor_sha256 !== manifestBase.descriptor_sha256
|
||||
|| manifestRecord.descriptor_dev !== manifestBase.descriptor_dev
|
||||
|| manifestRecord.descriptor_ino !== manifestBase.descriptor_ino
|
||||
|| manifestRecord.config_sha256 !== digest(content)
|
||||
|| !sameBinding(manifestRecord.config_dwh_binding, manifestBase.config_dwh_binding)
|
||||
|| manifestRecord.config_dev !== String(result.dev) || manifestRecord.config_ino !== String(result.ino)) {
|
||||
throw new Error("runtime config publish helper returned malformed output");
|
||||
}
|
||||
return result as unknown as PublishedResponse;
|
||||
}
|
||||
|
||||
@@ -271,8 +397,9 @@ export class WorkspaceRuntimeConfigLeaseFactory {
|
||||
snapshots_root: root, repository_root: repositoryRoot,
|
||||
workspace_revision: match[1], workspace_id: match[2],
|
||||
}) as Record<string, unknown>;
|
||||
const verifiedKeys = ["descriptor_dev", "descriptor_git_blob", "descriptor_ino", "git_source", "sha256", "snapshot_path", "source", "workspace_id", "workspace_revision"];
|
||||
const verifiedKeys = ["descriptor_dev", "descriptor_git_blob", "descriptor_ino", "git_source", "kind", "protocol_version", "sha256", "snapshot_path", "source", "workspace_id", "workspace_revision"];
|
||||
if (Object.keys(verified).sort().join(",") !== verifiedKeys.join(",")
|
||||
|| verified.protocol_version !== 1 || verified.kind !== "verified_snapshot"
|
||||
|| verified.workspace_id !== match[2] || verified.workspace_revision !== match[1]
|
||||
|| typeof verified.source !== "string" || typeof verified.git_source !== "string"
|
||||
|| verified.sha256 !== digest(verified.source) || verified.snapshot_path !== path
|
||||
|
||||
Reference in New Issue
Block a user