Files
ThothII/backend/src/workspaces/runtime-config-lease.ts
T

434 lines
23 KiB
TypeScript

import { createHash } from "node:crypto";
import { spawn } from "node:child_process";
import { isIP } from "node:net";
import { domainToASCII } from "node:url";
import {
existsSync, lstatSync, readFileSync,
} from "node:fs";
import { dirname, isAbsolute, join, relative, resolve, normalize } from "node:path";
import { parseAllDocuments } from "yaml";
import { resolveRuntimeBindings } from "./bindings.js";
import {
renderRuntimeConfig,
type RuntimeInstallationOverlay,
type RuntimePaths,
type SemanticRuntimeConfig,
} from "./runtime-renderer.js";
import { parseWorkspaceYaml, serializeWorkspaceYaml, validateOperationalWorkspace, type WorkspaceDescriptor } from "./schema.js";
export interface RuntimeConfigLease {
path: string;
manifestPath: string;
/** SHA-256 of the exact durable manifest bytes handed to the child. */
manifestSha256: string;
workspaceId: string;
workspaceRevision: string;
release(): void;
}
export interface MaintenanceRuntimeInput {
/** The immutable registry snapshot. `workspaceConfigPath` is accepted for callers using that name. */
snapshotPath?: string;
workspaceConfigPath?: string;
}
export interface WorkspaceRuntimeConfigLeaseFactoryInput {
dataRoot: string;
runtimeSnapshotRoot: string;
harnessDir: string;
configPath: string;
secretRoots?: readonly string[];
env?: NodeJS.ProcessEnv;
installationOverlay?: RuntimeInstallationOverlay;
semanticRuntime: SemanticRuntimeConfig;
}
interface SnapshotIdentity {
workspace: WorkspaceDescriptor;
workspaceId: string;
workspaceRevision: string;
revisionContentRoot: string;
digest: string;
descriptorBlob?: string;
descriptorDev: string;
descriptorIno: string;
}
interface PublishedResponse {
protocol_version: 1;
kind: "publication";
workspace_id: string;
workspace_revision: string;
path: string;
manifestPath: string;
manifest: string;
manifest_sha256: string;
dev: number;
ino: number;
}
interface PublishedIdentity {
path: string;
manifestPath: string;
/** SHA-256 of the exact durable manifest bytes handed to the child. */
manifestSha256: string;
workspaceId: string;
workspaceRevision: string;
digest: string;
content: string;
manifest: string;
}
const DEFAULT_SEMANTIC_RUNTIME: SemanticRuntimeConfig = {
internalQdrantUrl: "http://qdrant:6333", internalEmbeddingUrl: "http://embedding:11434",
internalEmbeddingModel: "qwen3-embedding:0.6b", internalEmbeddingDimensions: 1024,
};
/** Normalize the installation HTTP private-host policy once, before rendering. */
export function normalizePrivateHostAllowlist(value: string | readonly string[] | undefined): string[] {
const values: readonly string[] = value === undefined ? [] : typeof value === "string" ? (value === "" ? [] : value.split(",")) : [...value];
if (values.length > 32) throw new Error("HTTP private host allowlist has too many entries");
const result: string[] = [];
for (const host of values) {
if (
typeof host !== "string" || host.length === 0 || host.length > 253 || host !== host.toLowerCase()
|| host.endsWith(".") || host.includes(" ") || host.includes("\t")
|| host.includes("*") || host.includes("/") || host.includes("_")
|| host.includes(":") || host.split(".").some((label) => label.startsWith("xn--"))
) throw new Error("HTTP private host allowlist contains an invalid hostname");
const labels = host.split(".");
if (labels.some((label) => label.length === 0 || label.length > 63 || !/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(label))) {
throw new Error("HTTP private host allowlist contains an invalid hostname");
}
const ascii = domainToASCII(host);
let canonical = "";
try { canonical = new URL(`http://${host}`).hostname.toLowerCase().replace(/\.$/, ""); } catch { /* reject below */ }
if (!ascii || ascii !== host || canonical !== host || isIP(canonical) !== 0) {
throw new Error("HTTP private host allowlist contains an invalid hostname");
}
if (result.includes(host)) throw new Error("HTTP private host allowlist contains a duplicate hostname");
result.push(host);
}
// The policy is a set. Canonical ordering prevents semantically identical
// installation overlays from producing different durable config bytes.
return result.sort();
}
export const normalizeHttpPrivateHostAllowlist = normalizePrivateHostAllowlist;
function parseInstallationOverlay(path: string): RuntimeInstallationOverlay {
if (!isAbsolute(path) || !existsSync(path)) return {};
const docs = parseAllDocuments(readFileSync(path, "utf8"), { uniqueKeys: true });
if (docs.length !== 1 || docs[0].errors.length || docs[0].warnings.length) {
throw new Error("installation config contains invalid YAML");
}
const value = docs[0].toJSON();
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("installation config must be a YAML mapping");
const source = value as Record<string, unknown>;
return {
...(source.session_storage === undefined ? {} : { session_storage: source.session_storage }),
...(source.profile === undefined ? {} : { profile: source.profile }),
};
}
function digest(data: string | Buffer): string { return createHash("sha256").update(data).digest("hex"); }
export interface BoundedHelperOptions {
cwd: string;
env: NodeJS.ProcessEnv;
action: string;
payload: string;
timeoutMs?: number;
outputLimit?: number;
/** Package-internal seam used by lifecycle tests; production uses node spawn. */
spawnProcess?: typeof spawn;
}
/** Run one JSON helper with bounded output and deterministic child cleanup. */
export async function runBoundedHelper(
executable: string,
args: readonly string[],
options: BoundedHelperOptions,
): Promise<unknown> {
const timeoutMs = options.timeoutMs ?? 10_000;
const outputLimit = options.outputLimit ?? 16 * 1024 * 1024;
const spawnProcess = options.spawnProcess ?? spawn;
const child = spawnProcess(executable, [...args], {
cwd: options.cwd,
env: options.env,
detached: true,
stdio: ["pipe", "pipe", "pipe"],
});
let stdout = "";
let stderr = "";
let closed = false;
let finishing = false;
let timer: ReturnType<typeof setTimeout> | undefined;
let closeResolve: () => void = () => undefined;
const closePromise = new Promise<void>((resolveClose) => { closeResolve = resolveClose; });
const killGroup = () => {
try {
if (child.pid !== undefined && child.pid !== null) process.kill(-child.pid, "SIGKILL");
} catch {
try { child.kill("SIGKILL"); } catch { /* process already gone */ }
}
};
const destroyStreams = () => {
try { child.stdin?.destroy(); } catch { /* already closed */ }
try { child.stdout?.destroy(); } catch { /* already closed */ }
try { child.stderr?.destroy(); } catch { /* already closed */ }
};
const awaitClose = async () => {
if (closed) return;
// A descendant can keep stdio open even after the direct child exits. Streams
// are destroyed before this bounded reap wait so the backend cannot hang.
await Promise.race([closePromise, new Promise<void>((resolveWait) => setTimeout(resolveWait, 1_000))]);
};
return new Promise<unknown>((resolveResult, rejectResult) => {
const finish = async (error?: Error, value?: unknown, terminate = false) => {
if (finishing) return;
finishing = true;
if (timer !== undefined) clearTimeout(timer);
if (terminate) killGroup();
destroyStreams();
await awaitClose();
destroyStreams();
if (error) rejectResult(error); else resolveResult(value);
};
const append = (target: "stdout" | "stderr", data: Buffer) => {
const next = target === "stdout" ? stdout + data.toString() : stderr + data.toString();
if (Buffer.byteLength(next) > outputLimit) {
void finish(new Error(`runtime config ${options.action} output exceeded limit`), undefined, true);
return;
}
if (target === "stdout") stdout = next; else stderr = next;
};
// Every stream gets an error listener before any data is written. In
// particular, EPIPE from end() must become the same bounded failure path.
child.stdin?.on("error", (error) => void finish(error instanceof Error ? error : new Error(String(error)), undefined, true));
child.stdout?.on("error", (error) => void finish(error instanceof Error ? error : new Error(String(error)), undefined, true));
child.stderr?.on("error", (error) => void finish(error instanceof Error ? error : new Error(String(error)), undefined, true));
child.stdout?.on("data", (data: Buffer) => append("stdout", data));
child.stderr?.on("data", (data: Buffer) => append("stderr", data));
child.on("error", (error) => void finish(error instanceof Error ? error : new Error(String(error)), undefined, true));
child.on("close", (code) => {
closed = true;
closeResolve();
if (finishing) return;
if (code !== 0) {
let detail = stderr.trim() || stdout.trim() || `runtime config ${options.action} failed`;
try { detail = (JSON.parse(stdout) as { error?: string }).error ?? detail; } catch { /* preserve detail */ }
void finish(new Error(detail));
return;
}
try { void finish(undefined, JSON.parse(stdout)); }
catch { void finish(new Error(`runtime config ${options.action} returned invalid JSON`)); }
});
timer = setTimeout(() => {
void finish(new Error(`runtime config ${options.action} timed out`), undefined, true);
}, timeoutMs);
try {
// The listener above is intentionally installed before end(), since a
// helper may close its input immediately and emit EPIPE synchronously.
child.stdin?.end(options.payload);
} catch (error) {
void finish(error instanceof Error ? error : new Error(String(error)), undefined, true);
}
});
}
function sameBinding(left: unknown, right: unknown): boolean {
if (!left || typeof left !== "object" || Array.isArray(left) || !right || typeof right !== "object" || Array.isArray(right)) return false;
const a = left as Record<string, unknown>; const b = right as Record<string, unknown>;
return a.workspace_id === b.workspace_id && a.config_fingerprint === b.config_fingerprint && a.input_fingerprint === b.input_fingerprint;
}
export class WorkspaceRuntimeConfigLeaseFactory {
private readonly env: NodeJS.ProcessEnv;
private readonly secretRoots: readonly string[];
private readonly installation: RuntimeInstallationOverlay;
constructor(private readonly input: WorkspaceRuntimeConfigLeaseFactoryInput) {
if (!isAbsolute(input.dataRoot) || !isAbsolute(input.runtimeSnapshotRoot)) throw new Error("workspace runtime roots must be absolute");
// Registry snapshots are produced by WorkspaceRegistry. Never recursively
// create this security boundary from a pathname (an ancestor could be swapped).
if (!existsSync(input.runtimeSnapshotRoot)) throw new Error("runtime snapshot root is unavailable");
this.assertDirectory(input.runtimeSnapshotRoot, "runtime snapshot root");
this.env = { ...(input.env ?? process.env) };
this.secretRoots = [...(input.secretRoots ?? [])];
const configPath = isAbsolute(input.configPath) ? input.configPath : resolve(input.harnessDir, input.configPath);
const suppliedAllowlist = input.installationOverlay?.egress?.http_private_host_allowlist;
this.installation = {
...parseInstallationOverlay(configPath), ...(input.installationOverlay ?? {}),
egress: { http_private_host_allowlist: normalizePrivateHostAllowlist(suppliedAllowlist ?? this.env.THT_HTTP_PRIVATE_HOST_ALLOWLIST) },
} as RuntimeInstallationOverlay;
}
async acquireSession(snapshotPath: string): Promise<RuntimeConfigLease> { return this.acquire(snapshotPath); }
async acquireMaintenance(input: MaintenanceRuntimeInput): Promise<RuntimeConfigLease> {
const snapshotPath = input.snapshotPath ?? input.workspaceConfigPath;
if (!snapshotPath) throw new Error("maintenance runtime snapshot is required");
return this.acquire(snapshotPath);
}
private async acquire(snapshotPath: string): Promise<RuntimeConfigLease> {
const snapshot = await this.readSnapshot(snapshotPath);
const paths = this.runtimePaths(snapshot.workspaceId);
const rendered = renderRuntimeConfig(snapshot.workspace, resolveRuntimeBindings(snapshot.workspace, this.env, this.secretRoots), paths, snapshot, this.installation, this.input.semanticRuntime ?? DEFAULT_SEMANTIC_RUNTIME);
const renderedDigest = digest(rendered);
const base = {
workspace_id: snapshot.workspaceId, workspace_revision: snapshot.workspaceRevision,
descriptor_git_blob: snapshot.descriptorBlob!, descriptor_sha256: snapshot.digest,
descriptor_dev: snapshot.descriptorDev, descriptor_ino: snapshot.descriptorIno,
config_sha256: renderedDigest, config_dwh_binding: await this.computeBinding(rendered),
};
const result = await this.publishSecure(snapshot.workspaceId, snapshot.workspaceRevision, rendered, base);
const identity: PublishedIdentity = {
path: result.path, manifestPath: result.manifestPath, workspaceId: snapshot.workspaceId,
workspaceRevision: snapshot.workspaceRevision, digest: renderedDigest, content: rendered,
manifest: result.manifest, manifestSha256: result.manifest_sha256,
};
return this.lease(identity);
}
private async helper(action: string, extra: Record<string, unknown>): Promise<unknown> {
const python = join(this.input.harnessDir, ".venv", "bin", "python");
const modulePath = existsSync(join(this.input.harnessDir, "tht", "runtime_config_lease_io.py"))
? join(this.input.harnessDir, "tht", "runtime_config_lease_io.py")
: join(process.cwd(), "../harness/tht/runtime_config_lease_io.py");
const projectPython = join(dirname(dirname(modulePath)), ".venv", "bin", "python");
const executable = existsSync(python) ? python : existsSync(projectPython) ? projectPython : (process.env.PYTHON ?? "python3");
const helperArgs = existsSync(modulePath) ? [modulePath] : ["-m", "tht.runtime_config_lease_io"];
const env = { ...this.env };
// Capability variables are never inherited. Fault seams are only available in
// tests, and are copied explicitly rather than forwarding ambient state.
for (const key of Object.keys(env)) {
if (key.startsWith("THT_RUNTIME_CONFIG_") || key.startsWith("THT_CONFIG_")) delete env[key];
}
if (env.NODE_ENV === "test") {
for (const key of ["THT_RUNTIME_CONFIG_FSYNC_FAIL", "THT_RUNTIME_CONFIG_RENAME_FAIL"]) {
const value = this.env[key];
if (value !== undefined) env[key] = value;
}
}
env.PYTHONPATH = [this.input.harnessDir, dirname(dirname(modulePath)), env.PYTHONPATH].filter(Boolean).join(":");
const payload = JSON.stringify({ protocol_version: 1, action, ...extra });
return runBoundedHelper(executable, helperArgs, {
cwd: this.input.harnessDir, env, action, payload,
});
}
private async computeBinding(content: string): Promise<Record<string, string>> {
const value = await this.helper("binding", { config_hex: Buffer.from(content).toString("hex") });
if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).sort().join(",") !== "config_fingerprint,input_fingerprint,kind,protocol_version,workspace_id") throw new Error("runtime config binding helper returned malformed output");
const record = value as Record<string, unknown>;
if (record.protocol_version !== 1 || record.kind !== "binding"
|| Object.values(record).some((v) => typeof v !== "string" && typeof v !== "number")) throw new Error("runtime config binding helper returned malformed output");
if (typeof record.workspace_id !== "string" || typeof record.config_fingerprint !== "string" || typeof record.input_fingerprint !== "string") throw new Error("runtime config binding helper returned malformed output");
return { workspace_id: record.workspace_id, config_fingerprint: record.config_fingerprint, input_fingerprint: record.input_fingerprint };
}
private async publishSecure(workspaceId: string, revision: string, content: string, manifestBase: Record<string, unknown>): Promise<PublishedResponse> {
const value = await this.helper("publish", { data_root: this.input.dataRoot, workspace_id: workspaceId,
workspace_revision: revision, config_hex: Buffer.from(content).toString("hex"), manifest_base: manifestBase });
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("runtime config publish helper returned malformed output");
const result = value as Record<string, unknown>;
if (Object.keys(result).sort().join(",") !== "dev,ino,kind,manifest,manifestPath,manifest_sha256,path,protocol_version,workspace_id,workspace_revision") throw new Error("runtime config publish helper returned malformed output");
let expectedRoot = resolve(this.input.dataRoot);
if (process.platform === "darwin" && (expectedRoot === "/var" || expectedRoot.startsWith("/var/") || expectedRoot === "/tmp" || expectedRoot.startsWith("/tmp/"))) expectedRoot = `/private${expectedRoot}`;
const expectedPath = join(expectedRoot, "sessions", workspaceId, "preprocessing", "runtime-config", `${revision}.yaml`);
const expectedManifestPath = join(expectedRoot, "sessions", workspaceId, "preprocessing", "runtime-config-manifests", `${revision}.json`);
if (result.protocol_version !== 1 || result.kind !== "publication"
|| result.workspace_id !== workspaceId || result.workspace_revision !== revision
|| typeof result.path !== "string" || result.path !== expectedPath || !isAbsolute(result.path) || normalize(result.path) !== result.path
|| typeof result.manifestPath !== "string" || result.manifestPath !== expectedManifestPath || !isAbsolute(result.manifestPath) || normalize(result.manifestPath) !== result.manifestPath
|| typeof result.manifest !== "string" || typeof result.manifest_sha256 !== "string" || !/^[0-9a-f]{64}$/.test(result.manifest_sha256)
|| digest(result.manifest) !== result.manifest_sha256
|| typeof result.dev !== "number" || !Number.isSafeInteger(result.dev) || typeof result.ino !== "number" || !Number.isSafeInteger(result.ino)) throw new Error("runtime config publish helper returned malformed output");
let manifest: unknown;
try { manifest = JSON.parse(result.manifest); } catch { throw new Error("runtime config publish helper returned malformed output"); }
if (!manifest || typeof manifest !== "object" || Array.isArray(manifest)) throw new Error("runtime config publish helper returned malformed output");
const manifestRecord = manifest as Record<string, unknown>;
if (manifestRecord.version !== 1
|| manifestRecord.workspace_id !== workspaceId || manifestRecord.workspace_revision !== revision
|| manifestRecord.descriptor_git_blob !== manifestBase.descriptor_git_blob
|| manifestRecord.descriptor_sha256 !== manifestBase.descriptor_sha256
|| manifestRecord.descriptor_dev !== manifestBase.descriptor_dev
|| manifestRecord.descriptor_ino !== manifestBase.descriptor_ino
|| manifestRecord.config_sha256 !== digest(content)
|| !sameBinding(manifestRecord.config_dwh_binding, manifestBase.config_dwh_binding)
|| manifestRecord.config_dev !== String(result.dev) || manifestRecord.config_ino !== String(result.ino)) {
throw new Error("runtime config publish helper returned malformed output");
}
return result as unknown as PublishedResponse;
}
private lease(identity: PublishedIdentity): RuntimeConfigLease {
let released = false;
return { path: identity.path, manifestPath: identity.manifestPath, manifestSha256: identity.manifestSha256, workspaceId: identity.workspaceId, workspaceRevision: identity.workspaceRevision,
release: () => { if (released) return; released = true; /* Durable revision-owned state: release only drops our local handle/ref. */ }, };
}
private runtimePaths(workspaceId: string): RuntimePaths {
const root = join(this.input.dataRoot, "sessions", workspaceId);
return { sessions: join(root, "sessions"), artifacts: join(root, "artifacts"), indexes: join(root, "indexes") };
}
private assertDirectory(path: string, label: string): void {
const e = lstatSync(path);
if (!e.isDirectory() || e.isSymbolicLink() || e.nlink < 1 || (e.mode & 0o077) !== 0 || e.uid !== process.getuid?.()) throw new Error(`${label} is not trusted`);
}
private async readSnapshot(path: string): Promise<SnapshotIdentity> {
if (!isAbsolute(path)) throw new Error("workspace snapshot path must be absolute");
const root = resolve(this.input.runtimeSnapshotRoot);
const rel = relative(root, path);
const match = /^([0-9a-f]{40})\/([a-z][a-z0-9-]{2,62})\.yaml$/.exec(rel);
if (!match || rel.startsWith("..") || isAbsolute(rel)) throw new Error("config path is not a trusted runtime snapshot");
// The helper is the canonical registry capability boundary. It opens the exact
// production snapshot.json and descriptor component-by-component, and MUST prove
// the Git commit/blob identity; a pathname-shaped file is never sufficient.
const repositoryRoot = join(dirname(root), "repo");
const verified = await this.helper("verified-snapshot", {
snapshots_root: root, repository_root: repositoryRoot,
workspace_revision: match[1], workspace_id: match[2],
}) as Record<string, unknown>;
const verifiedKeys = ["descriptor_dev", "descriptor_git_blob", "descriptor_ino", "git_source", "kind", "protocol_version", "sha256", "snapshot_path", "source", "workspace_id", "workspace_revision"];
if (Object.keys(verified).sort().join(",") !== verifiedKeys.join(",")
|| verified.protocol_version !== 1 || verified.kind !== "verified_snapshot"
|| verified.workspace_id !== match[2] || verified.workspace_revision !== match[1]
|| typeof verified.source !== "string" || typeof verified.git_source !== "string"
|| verified.sha256 !== digest(verified.source) || verified.snapshot_path !== path
|| !/^\d+$/.test(String(verified.descriptor_dev)) || !/^\d+$/.test(String(verified.descriptor_ino))) {
throw new Error("workspace snapshot integrity check failed");
}
// The registry's production canonicalizer is the sole descriptor equivalence
// rule. Raw token containment is not identity: it permits changed values.
let workspace: WorkspaceDescriptor;
let gitWorkspace: WorkspaceDescriptor;
try {
workspace = validateOperationalWorkspace(parseWorkspaceYaml(verified.source));
gitWorkspace = validateOperationalWorkspace(parseWorkspaceYaml(verified.git_source));
if (serializeWorkspaceYaml(workspace) !== serializeWorkspaceYaml(gitWorkspace)
|| serializeWorkspaceYaml(workspace) !== verified.source) {
throw new Error("canonical descriptor differs from Git");
}
} catch (error) {
throw new Error(`workspace snapshot integrity check failed: ${error instanceof Error ? error.message : "invalid descriptor"}`);
}
if (workspace.workspace.id !== match[2] || typeof verified.descriptor_git_blob !== "string") {
throw new Error("workspace snapshot integrity check failed");
}
return {
workspace, workspaceId: match[2], workspaceRevision: match[1],
revisionContentRoot: join(root, match[1]), digest: verified.sha256,
descriptorBlob: verified.descriptor_git_blob,
descriptorDev: String(verified.descriptor_dev), descriptorIno: String(verified.descriptor_ino),
};
}
}