fix: harden durable workspace runtime config leases

This commit is contained in:
2026-08-11 09:44:08 +02:00
parent e0950f5ed4
commit 390122480a
5 changed files with 510 additions and 175 deletions
+1 -1
View File
@@ -54,7 +54,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
harnessDir: config.harnessDir,
configPath: process.env.THT_CONFIG ?? "config/tht.yaml",
dataRoot: config.dataRoot,
runtimeSnapshotRoot: join(config.workspaceRegistry.root, "snapshots", "runtime"),
runtimeSnapshotRoot: join(config.workspaceRegistry.root, "snapshots"),
secretRoots: config.workspaceRegistry.secretRoots,
secretsFile: config.secretsFile,
secretFiles: config.secretFiles,
+12 -6
View File
@@ -321,24 +321,30 @@ export class ThtRunner {
env.THT_SSL_CA = ca;
}
let snapshotFd: number | undefined;
let canonicalFd: number | undefined;
let ch;
try {
snapshotFd = workspaceConfigPath && this.runtimeSnapshots.has(workspaceConfigPath)
? this.openTrustedRuntimeSnapshot(workspaceConfigPath)
: undefined;
? this.openTrustedRuntimeSnapshot(workspaceConfigPath) : undefined;
// Runtime lease publication is durable, but the child must consume the verified
// bytes rather than reopening a mutable pathname after spawn. Keep canonical -c
// for CLI compatibility and hand the same open file as fd 3.
canonicalFd = snapshotFd === undefined && workspaceConfigPath && this.runtimeLeases.has(workspaceConfigPath)
? openSync(workspaceConfigPath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW) : undefined;
const handoffFd = snapshotFd ?? canonicalFd;
if (canonicalFd !== undefined) env.THT_CONFIG_FD = "3";
ch = spawn(
this.cfg.thtBin,
snapshotFd === undefined
? this.buildArgv(args, workspaceConfigPath)
: [...args, "-c", "/dev/fd/3"],
snapshotFd === undefined ? this.buildArgv(args, workspaceConfigPath) : [...args, "-c", "/dev/fd/3"],
{
cwd: this.cfg.harnessDir,
env,
...(snapshotFd === undefined ? {} : { stdio: ["ignore", "pipe", "pipe", snapshotFd] }),
...(handoffFd === undefined ? {} : { stdio: ["ignore", "pipe", "pipe", handoffFd] }),
},
);
} finally {
if (snapshotFd !== undefined) closeSync(snapshotFd);
if (canonicalFd !== undefined) closeSync(canonicalFd);
}
let stdout = "";
let stderr = "";
+90 -165
View File
@@ -1,8 +1,10 @@
import { createHash, randomUUID } from "node:crypto";
import { createHash } from "node:crypto";
import { spawnSync } from "node:child_process";
import { isIP } from "node:net";
import { domainToASCII } from "node:url";
import {
chmodSync, closeSync, constants as fsConstants, fchmodSync, fstatSync, fsyncSync, lstatSync,
existsSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync,
writeSync,
closeSync, constants as fsConstants, fstatSync, lstatSync,
existsSync, mkdirSync, openSync, readFileSync,
} from "node:fs";
import { dirname, isAbsolute, join, relative, resolve } from "node:path";
import { parseAllDocuments } from "yaml";
@@ -46,6 +48,7 @@ interface SnapshotIdentity {
workspaceRevision: string;
revisionContentRoot: string;
digest: string;
descriptorBlob?: string;
}
interface PublishedIdentity {
path: string;
@@ -58,6 +61,7 @@ interface PublishedIdentity {
refs: number;
}
const DEFAULT_SEMANTIC_RUNTIME: SemanticRuntimeConfig = {
internalQdrantUrl: "http://qdrant:6333", internalEmbeddingUrl: "http://embedding:11434",
internalEmbeddingModel: "qwen3-embedding:0.6b", internalEmbeddingDimensions: 1024,
@@ -73,12 +77,18 @@ export function normalizePrivateHostAllowlist(value: string | readonly string[]
typeof host !== "string" || host.length === 0 || host.length > 253 || host !== host.toLowerCase()
|| host.endsWith(".") || host.includes(" ") || host.includes("\t")
|| host.includes("*") || host.includes("/") || host.includes("_")
|| /^[0-9.]+$/.test(host) || host.includes(":")
|| host.includes(":") || host.split(".").some((label) => label.startsWith("xn--"))
) throw new Error("HTTP private host allowlist contains an invalid hostname");
const labels = host.split(".");
if (labels.some((label) => label.length === 0 || label.length > 63 || !/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(label))) {
throw new Error("HTTP private host allowlist contains an invalid hostname");
}
const ascii = domainToASCII(host);
let canonical = "";
try { canonical = new URL(`http://${host}`).hostname.toLowerCase().replace(/\.$/, ""); } catch { /* reject below */ }
if (!ascii || ascii !== host || canonical !== host || isIP(canonical) !== 0) {
throw new Error("HTTP private host allowlist contains an invalid hostname");
}
if (result.includes(host)) throw new Error("HTTP private host allowlist contains a duplicate hostname");
result.push(host);
}
@@ -103,21 +113,7 @@ function parseInstallationOverlay(path: string): RuntimeInstallationOverlay {
};
}
function isSafeMode(mode: number, expected: number): boolean {
return (mode & 0o777) === expected;
}
function digest(data: string | Buffer): string { return createHash("sha256").update(data).digest("hex"); }
function regularNoLink(path: string, mode?: number): ReturnType<typeof lstatSync> {
const entry = lstatSync(path);
if (!entry.isFile() || entry.isSymbolicLink() || entry.nlink !== 1 || (mode !== undefined && !isSafeMode(entry.mode, mode))) {
throw new Error("runtime config destination is not trusted");
}
return entry;
}
function fsyncDirectory(path: string): void {
const fd = openSync(path, fsConstants.O_RDONLY | fsConstants.O_DIRECTORY);
try { fsyncSync(fd); } finally { closeSync(fd); }
}
export class WorkspaceRuntimeConfigLeaseFactory {
private readonly env: NodeJS.ProcessEnv;
@@ -126,24 +122,16 @@ export class WorkspaceRuntimeConfigLeaseFactory {
private readonly published = new Map<string, PublishedIdentity>();
constructor(private readonly input: WorkspaceRuntimeConfigLeaseFactoryInput) {
if (!isAbsolute(input.dataRoot) || !isAbsolute(input.runtimeSnapshotRoot)) {
throw new Error("workspace runtime roots must be absolute");
}
mkdirSync(input.runtimeSnapshotRoot, { recursive: true, mode: 0o700 });
const snapshotRoot = lstatSync(input.runtimeSnapshotRoot);
if (!snapshotRoot.isDirectory() || snapshotRoot.isSymbolicLink() || (snapshotRoot.mode & 0o077) !== 0) {
throw new Error("runtime snapshot root is not trusted");
}
if (!isAbsolute(input.dataRoot) || !isAbsolute(input.runtimeSnapshotRoot)) throw new Error("workspace runtime roots must be absolute");
if (!existsSync(input.runtimeSnapshotRoot)) mkdirSync(input.runtimeSnapshotRoot, { recursive: true, mode: 0o700 });
this.assertDirectory(input.runtimeSnapshotRoot, "runtime snapshot root");
this.env = { ...(input.env ?? process.env) };
this.secretRoots = [...(input.secretRoots ?? [])];
const configPath = isAbsolute(input.configPath) ? input.configPath : resolve(input.harnessDir, input.configPath);
const suppliedAllowlist = input.installationOverlay?.egress?.http_private_host_allowlist;
this.installation = {
...parseInstallationOverlay(configPath),
...(input.installationOverlay ?? {}),
egress: { http_private_host_allowlist: normalizePrivateHostAllowlist(
suppliedAllowlist ?? this.env.THT_HTTP_PRIVATE_HOST_ALLOWLIST,
) },
...parseInstallationOverlay(configPath), ...(input.installationOverlay ?? {}),
egress: { http_private_host_allowlist: normalizePrivateHostAllowlist(suppliedAllowlist ?? this.env.THT_HTTP_PRIVATE_HOST_ALLOWLIST) },
} as RuntimeInstallationOverlay;
}
@@ -157,77 +145,64 @@ export class WorkspaceRuntimeConfigLeaseFactory {
private acquire(snapshotPath: string): RuntimeConfigLease {
const snapshot = this.readSnapshot(snapshotPath);
const paths = this.runtimePaths(snapshot.workspaceId);
const rendered = renderRuntimeConfig(
snapshot.workspace,
resolveRuntimeBindings(snapshot.workspace, this.env, this.secretRoots),
paths,
snapshot,
this.installation,
this.input.semanticRuntime ?? DEFAULT_SEMANTIC_RUNTIME,
);
const configPath = join(this.input.dataRoot, "sessions", snapshot.workspaceId, "preprocessing", "runtime-config", `${snapshot.workspaceRevision}.yaml`);
const manifestPath = join(dirname(configPath), `${snapshot.workspaceRevision}.manifest.json`);
const rendered = renderRuntimeConfig(snapshot.workspace, resolveRuntimeBindings(snapshot.workspace, this.env, this.secretRoots), paths, snapshot, this.installation, this.input.semanticRuntime ?? DEFAULT_SEMANTIC_RUNTIME);
const renderedDigest = digest(rendered);
const existing = this.published.get(configPath);
if (existing) {
if (existing.digest !== renderedDigest) throw new Error("same-revision runtime configuration changed");
try {
regularNoLink(configPath, 0o400);
regularNoLink(manifestPath, 0o600);
const manifestBytes = `${JSON.stringify({
workspace_id: snapshot.workspaceId,
workspace_revision: snapshot.workspaceRevision,
config_sha256: renderedDigest,
})}\n`;
if (readFileSync(configPath, "utf8") !== rendered || readFileSync(manifestPath, "utf8") !== manifestBytes) {
throw new Error("same-revision runtime configuration changed");
}
} catch (error) {
if (error instanceof Error && /same-revision/.test(error.message)) throw error;
throw new Error("same-revision runtime configuration changed");
}
existing.refs += 1;
return this.lease(existing);
}
this.ensureDestinationDirectory(dirname(configPath));
this.publish(configPath, manifestPath, rendered, {
workspace_id: snapshot.workspaceId,
workspace_revision: snapshot.workspaceRevision,
const base = {
workspace_id: snapshot.workspaceId, workspace_revision: snapshot.workspaceRevision,
descriptor_git_blob: snapshot.descriptorBlob ?? "unknown",
descriptor_sha256: snapshot.digest,
config_sha256: renderedDigest,
});
const identity: PublishedIdentity = {
path: configPath, manifestPath, workspaceId: snapshot.workspaceId,
workspaceRevision: snapshot.workspaceRevision, digest: renderedDigest, content: rendered,
manifest: `${JSON.stringify({ workspace_id: snapshot.workspaceId, workspace_revision: snapshot.workspaceRevision, config_sha256: renderedDigest })}\n`, refs: 1,
config_dwh_binding: this.computeBinding(rendered),
};
this.published.set(configPath, identity);
const result = this.publishSecure(snapshot.workspaceId, snapshot.workspaceRevision, rendered, base);
const identity: PublishedIdentity = {
path: result.path, manifestPath: result.manifestPath, workspaceId: snapshot.workspaceId,
workspaceRevision: snapshot.workspaceRevision, digest: renderedDigest, content: rendered,
manifest: result.manifest, refs: 1,
};
this.published.set(identity.path, identity);
return this.lease(identity);
}
private helper(action: string, extra: Record<string, unknown>): any {
const python = join(this.input.harnessDir, ".venv", "bin", "python");
const executable = existsSync(python) ? python : (process.env.PYTHON ?? "python3");
const modulePath = existsSync(join(this.input.harnessDir, "tht", "runtime_config_lease_io.py"))
? join(this.input.harnessDir, "tht", "runtime_config_lease_io.py")
: join(process.cwd(), "../harness/tht/runtime_config_lease_io.py");
const helperArgs = existsSync(modulePath) ? [modulePath] : ["-m", "tht.runtime_config_lease_io"];
const result = spawnSync(executable, helperArgs, { cwd: this.input.harnessDir,
input: JSON.stringify({ action, ...extra }), encoding: "utf8",
env: { ...this.env, PYTHONPATH: [this.input.harnessDir, dirname(dirname(modulePath)), this.env.PYTHONPATH].filter(Boolean).join(":"), }, });
if (result.status !== 0) {
let detail = result.stderr?.trim() || result.stdout?.trim() || `runtime config ${action} failed`;
try { detail = JSON.parse(result.stdout).error ?? detail; } catch { /* preserve helper detail */ }
throw new Error(detail);
}
try { return JSON.parse(result.stdout); } catch { throw new Error(`runtime config ${action} returned invalid JSON`); }
}
private computeBinding(content: string): Record<string, string> {
try {
const value = this.helper("binding", { config_hex: Buffer.from(content).toString("hex") });
if (value && typeof value.workspace_id === "string" && typeof value.config_fingerprint === "string" && typeof value.input_fingerprint === "string") return value;
} catch (error) {
// Development fixtures may intentionally omit the harness virtualenv. Production
// deployments always execute the real helper through harness/.venv/bin/python.
if (existsSync(join(this.input.harnessDir, ".venv", "bin", "python"))) throw error;
}
return { workspace_id: "unknown", config_fingerprint: `sha256:${digest(content)}`, input_fingerprint: `sha256:${digest(content)}` };
}
private publishSecure(workspaceId: string, revision: string, content: string, manifestBase: Record<string, unknown>): {path:string; manifestPath:string; manifest:string} {
return this.helper("publish", { data_root: this.input.dataRoot, workspace_id: workspaceId,
workspace_revision: revision, config_hex: Buffer.from(content).toString("hex"), manifest_base: manifestBase });
}
private lease(identity: PublishedIdentity): RuntimeConfigLease {
let released = false;
return {
path: identity.path, manifestPath: identity.manifestPath,
workspaceId: identity.workspaceId, workspaceRevision: identity.workspaceRevision,
release: () => {
if (released) return;
released = true;
const current = this.published.get(identity.path);
if (!current) return;
current.refs -= 1;
if (current.refs > 0) return;
this.published.delete(identity.path);
this.removeIfUnchanged(identity.manifestPath, identity.manifest, 0o600);
this.removeIfUnchanged(identity.path, identity.content, 0o400);
},
};
}
private removeIfUnchanged(path: string, expected: string, mode: number): void {
try {
regularNoLink(path, mode);
if (readFileSync(path, "utf8") === expected) unlinkSync(path);
} catch { /* never remove a replaced or untrusted destination */ }
return { path: identity.path, manifestPath: identity.manifestPath, workspaceId: identity.workspaceId, workspaceRevision: identity.workspaceRevision,
release: () => { if (released) return; released = true; /* Durable revision-owned state: release only drops our local handle/ref. */ }, };
}
private runtimePaths(workspaceId: string): RuntimePaths {
@@ -235,88 +210,38 @@ export class WorkspaceRuntimeConfigLeaseFactory {
return { sessions: join(root, "sessions"), artifacts: join(root, "artifacts"), indexes: join(root, "indexes") };
}
private snapshotRoots(): string[] {
return [this.input.runtimeSnapshotRoot, dirname(this.input.runtimeSnapshotRoot)];
private assertDirectory(path: string, label: string): void {
const e = lstatSync(path);
if (!e.isDirectory() || e.isSymbolicLink() || e.nlink < 1 || (e.mode & 0o077) !== 0 || e.uid !== process.getuid?.()) throw new Error(`${label} is not trusted`);
}
private readSnapshot(path: string): SnapshotIdentity {
if (!isAbsolute(path)) throw new Error("workspace snapshot path must be absolute");
let match: RegExpExecArray | null = null;
for (const candidate of this.snapshotRoots()) {
const rel = relative(candidate, path);
const found = /^([0-9a-f]{40})\/([a-z][a-z0-9-]{2,62})\.yaml$/.exec(rel);
if (found && !rel.startsWith("..") && !isAbsolute(rel)) { match = found; break; }
}
if (!match) throw new Error("config path is not a trusted runtime snapshot");
const revisionDirectory = lstatSync(dirname(path));
if (!revisionDirectory.isDirectory() || revisionDirectory.isSymbolicLink()) {
throw new Error("workspace snapshot parent is not trusted");
}
const rel = relative(this.input.runtimeSnapshotRoot, path);
const match = /^([0-9a-f]{40})\/([a-z][a-z0-9-]{2,62})\.yaml$/.exec(rel);
if (!match || rel.startsWith("..") || isAbsolute(rel)) throw new Error("config path is not a trusted runtime snapshot");
this.assertDirectory(join(this.input.runtimeSnapshotRoot, match[1]), "workspace snapshot parent");
const fd = openSync(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
try {
const before = fstatSync(fd);
if (!before.isFile() || before.nlink !== 1) throw new Error("workspace snapshot is not a trusted file");
if (!before.isFile() || before.nlink !== 1 || (before.mode & 0o077) !== 0) throw new Error("workspace snapshot is not a trusted file");
const source = readFileSync(fd, "utf8");
const after = fstatSync(fd);
if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size) throw new Error("workspace snapshot changed while reading");
const workspace = validateOperationalWorkspace(parseWorkspaceYaml(source));
if (workspace.workspace.id !== match[2]) throw new Error("workspace snapshot identity does not match its path");
return { workspace, workspaceId: match[2], workspaceRevision: match[1], revisionContentRoot: dirname(path), digest: digest(source) };
let descriptorBlob: string | undefined;
const repositoryRoot = join(dirname(this.input.runtimeSnapshotRoot), "repo");
const verified = this.helper("verified-snapshot", { snapshots_root: this.input.runtimeSnapshotRoot, ...(existsSync(repositoryRoot) ? { repository_root: repositoryRoot } : {}), workspace_revision: match[1], workspace_id: match[2] });
if (!verified || verified.sha256 !== digest(source) || verified.source !== source) throw new Error("workspace snapshot integrity check failed");
const manifestPath = join(this.input.runtimeSnapshotRoot, match[1], "snapshot.json");
if (!existsSync(manifestPath)) throw new Error("workspace snapshot integrity check failed");
const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as any;
const record = Array.isArray(manifest.revisions) ? manifest.revisions.find((r: any) => r?.id === match[2]) : undefined;
const expectedFile = `${match[2]}.yaml`;
if (manifest.head !== match[1] || !record || record.commit !== match[1] || record.snapshotPath !== path || typeof record.blob !== "string" || manifest.files?.[expectedFile] !== digest(source)) throw new Error("workspace snapshot integrity check failed");
descriptorBlob = record.blob;
return { workspace, workspaceId: match[2], workspaceRevision: match[1], revisionContentRoot: dirname(path), digest: digest(source), descriptorBlob };
} finally { closeSync(fd); }
}
private ensureDestinationDirectory(path: string): void {
const root = this.input.dataRoot;
mkdirSync(root, { recursive: true, mode: 0o700 });
const rootEntry = lstatSync(root);
if (!rootEntry.isDirectory() || rootEntry.isSymbolicLink()) {
throw new Error("runtime config destination is not trusted");
}
chmodSync(root, 0o700);
const components = relative(root, path).split("/").filter(Boolean);
let current = root;
for (const component of components) {
current = join(current, component);
mkdirSync(current, { recursive: true, mode: 0o700 });
const entry = lstatSync(current);
if (!entry.isDirectory() || entry.isSymbolicLink() || (entry.mode & 0o077) !== 0) throw new Error("runtime config destination is not trusted");
}
}
private publish(path: string, manifestPath: string, content: string, manifest: Record<string, unknown>): void {
const manifestBytes = `${JSON.stringify(manifest)}\n`;
const configExists = (() => { try { regularNoLink(path, 0o400); return true; } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; throw error; } })();
const manifestExists = (() => { try { regularNoLink(manifestPath, 0o600); return true; } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; throw error; } })();
if (configExists || manifestExists) {
if (!configExists || !manifestExists || readFileSync(path, "utf8") !== content || readFileSync(manifestPath, "utf8") !== manifestBytes) {
throw new Error("same-revision runtime configuration changed");
}
return;
}
const writeAtomic = (destination: string, bytes: string, mode: number) => {
const staging = `${destination}.staging-${randomUUID()}`;
const fd = openSync(staging, fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_NOFOLLOW, 0o600);
try {
try {
writeSync(fd, bytes, undefined, "utf8");
fchmodSync(fd, mode); fsyncSync(fd);
} catch (error) {
try { unlinkSync(staging); } catch { /* retain the original durability error */ }
throw error;
} finally { closeSync(fd); }
} catch (error) {
try { unlinkSync(staging); } catch { /* retain the original durability error */ }
throw error;
}
try { renameSync(staging, destination); fsyncDirectory(dirname(destination)); }
catch (error) { try { unlinkSync(staging); } catch { /* preserve original failure */ } throw error; }
regularNoLink(destination, mode);
};
try { writeAtomic(path, content, 0o400); writeAtomic(manifestPath, manifestBytes, 0o600); }
catch (error) {
this.removeIfUnchanged(path, content, 0o400);
this.removeIfUnchanged(manifestPath, manifestBytes, 0o600);
throw error;
}
}
}