Files
ThothII/backend/src/workspaces/runtime-config-lease.ts
T
Codex 82e2c91f42
Publish documentation / publish (push) Successful in 1m27s
feat: implement memory and evidence administration with guided repairs
Add PostgreSQL-backed memory, editable evidence with source review and activation, and human-approved archive repairs across the harness, API, and UI. Include migrations, deployment support, regression coverage, and validation documentation.

Refresh permissions from validated session roles so existing administrator logins can access newly deployed archive management features.
2026-09-10 10:31:34 +02:00

645 lines
24 KiB
TypeScript

import { createHash, randomUUID } from "node:crypto";
import {
closeSync,
constants as fsConstants,
fchmodSync,
fstatSync,
fsyncSync,
lstatSync,
mkdirSync,
openSync,
readFileSync,
readSync,
renameSync,
statSync,
unlinkSync,
writeFileSync,
} from "node:fs";
import { dirname, isAbsolute, join, relative, resolve } from "node:path";
import { readFile as readFileAsync } from "node:fs/promises";
import { parse, parseAllDocuments, stringify } from "yaml";
import {
buildCanonicalEffectiveConfig,
canonicalEffectiveConfigJson,
configFingerprint,
effectiveConfigIdentity,
preprocessingInputFingerprint,
type CanonicalEffectiveConfig,
} from "./effective-config.js";
import { resolveRuntimeBindings, type RuntimeBindings } from "./bindings.js";
import { resolveRuntimeBindingsWithWorkspaceSecrets } from "./secret-requirements.js";
import type { WorkspaceSecretStore } from "./secret-store.js";
import { GitWorkspaceRepository } from "./git-repository.js";
import { WorkspaceRegistry } from "./registry.js";
import {
renderRuntimeConfig,
type RuntimeIdentity,
type RuntimeInstallationOverlay,
type RuntimePaths,
type RuntimeRenderContext,
type SemanticRuntimeConfig,
} from "./runtime-renderer.js";
import { parseWorkspaceYaml, validateOperationalWorkspace, type WorkspaceDescriptor } from "./schema.js";
import type { WorkspaceRegistryConfig } from "./types.js";
import { resolveCatalogRuntimeBinding } from "../catalog/runtime-binding.js";
import type { WorkspaceDatabase } from "../catalog/types.js";
export interface RuntimeConfigLease {
path: string;
workspaceId: string;
workspaceRevision: string;
release(): void;
}
export interface RenderedWorkspaceRuntime {
workspace: WorkspaceDescriptor;
workspaceId: string;
workspaceRevision: string;
revisionContentRoot: string;
runtimePaths: RuntimePaths;
installationOverlay: RuntimeInstallationOverlay;
bindings: RuntimeBindings;
bindingDigest: string;
renderedConfig: string;
semanticQdrantUrl: string;
releaseSecrets(): void;
}
export interface ActiveRenderedWorkspaceRuntime extends RenderedWorkspaceRuntime {
snapshotPath: string;
descriptorBlob: string;
catalogBlob: string;
}
export interface DeterministicRuntimeConfigLease extends RuntimeConfigLease {
manifestPath: string;
descriptorBlob: string;
catalogBlob: string;
configDigest: string;
bindingDigest: string;
semanticQdrantUrl: string;
effectiveConfig: CanonicalEffectiveConfig;
effectiveConfigIdentity: string;
configFingerprint: string;
inputFingerprint: string;
}
export class RuntimeConfigLeaseError extends Error {
constructor(readonly code: "effective_config_mismatch", message: string) {
super(message);
this.name = "RuntimeConfigLeaseError";
}
}
interface SnapshotIdentity extends RuntimeIdentity {
snapshotPath: string;
}
interface PublishedRuntimeConfigManifest {
schemaVersion: 1;
workspaceId: string;
workspaceRevision: string;
descriptorBlob: string;
catalogBlob: string;
configDigest: string;
bindingDigest: string;
effectiveConfigIdentity: string;
configFingerprint: string;
inputFingerprint: string;
path: string;
file: {
dev: number;
ino: number;
size: number;
mode: number;
nlink: number;
};
}
function sha256(value: string | Buffer): string {
return `sha256:${createHash("sha256").update(value).digest("hex")}`;
}
function stableBindingDigest(bindings: RuntimeBindings): string {
const encodeRecord = (value: Record<string, string>) => Object.entries(value).sort(([left], [right]) => (
left.localeCompare(right)
));
return sha256(JSON.stringify({
dwh: {
transport: bindings.dwh.transport,
values: encodeRecord(bindings.dwh.values),
missing: [...bindings.dwh.missing].sort(),
},
evidence: {
values: encodeRecord(bindings.evidence.values),
missing: [...bindings.evidence.missing].sort(),
},
}));
}
function syncDirectory(directory: string): void {
if (process.platform === "win32") return;
const fd = openSync(directory, "r");
try { fsyncSync(fd); } finally { closeSync(fd); }
}
function ensureTrustedDirectory(directory: string): string {
mkdirSync(directory, { recursive: true, mode: 0o700 });
const entry = lstatSync(directory);
if (!entry.isDirectory() || entry.isSymbolicLink()) {
throw new Error("runtime configuration directory is unavailable");
}
return directory;
}
function writeAtomicFile(path: string, contents: string, mode: number): void {
ensureTrustedDirectory(dirname(path));
const staging = `${path}.tmp-${process.pid}-${Date.now()}-${randomUUID()}`;
const fd = openSync(
staging,
fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_NOFOLLOW,
0o600,
);
let closed = false;
try {
writeFileSync(fd, contents, "utf8");
fsyncSync(fd);
fchmodSync(fd, mode);
closeSync(fd);
closed = true;
renameSync(staging, path);
syncDirectory(dirname(path));
} catch (error) {
if (!closed) try { closeSync(fd); } catch { /* preserve original error */ }
try { unlinkSync(staging); } catch { /* best effort */ }
throw error;
}
}
function readTrustedFile(path: string): { contents: string; stat: ReturnType<typeof statSync> } {
const entry = lstatSync(path);
if (!entry.isFile() || entry.isSymbolicLink()) {
throw new Error("runtime configuration file is untrusted");
}
const fd = openSync(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
try {
const before = fstatSync(fd);
if (!before.isFile() || before.nlink !== 1) {
throw new Error("runtime configuration file is untrusted");
}
const contents = readFileSync(fd, "utf8");
const after = fstatSync(fd);
if (
before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size || before.nlink !== after.nlink
) {
throw new Error("runtime configuration file changed while reading");
}
return { contents, stat: statSync(path) };
} finally {
closeSync(fd);
}
}
function readStrictJson(path: string): unknown {
return JSON.parse(readTrustedFile(path).contents);
}
function trustedSnapshotIdentity(snapshotPath: string): SnapshotIdentity {
if (!isAbsolute(snapshotPath)) throw new Error("config path is not a trusted runtime snapshot");
const commitDirectory = dirname(snapshotPath);
const snapshotsRoot = dirname(commitDirectory);
const pathRelative = relative(snapshotsRoot, snapshotPath);
const match = /^([0-9a-f]{40})\/([a-z][a-z0-9-]{2,62})\.yaml$/.exec(pathRelative);
if (pathRelative.startsWith("..") || isAbsolute(pathRelative) || !match) {
throw new Error("config path is not a trusted runtime snapshot");
}
const entry = lstatSync(snapshotPath);
if (!entry.isFile() || entry.isSymbolicLink()) {
throw new Error("config path is not a trusted runtime snapshot");
}
return { snapshotPath, workspaceRevision: match[1], workspaceId: match[2] };
}
function readSnapshotWorkspace(snapshotPath: string): {
workspace: WorkspaceDescriptor;
identity: SnapshotIdentity;
revisionContentRoot: string;
} {
const identity = trustedSnapshotIdentity(snapshotPath);
const fd = openSync(snapshotPath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
try {
const before = fstatSync(fd);
if (!before.isFile() || before.nlink !== 1) throw new Error("workspace snapshot is not a file");
const source = readFileSync(fd, "utf8");
const after = fstatSync(fd);
if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size) {
throw new Error("workspace snapshot changed while reading");
}
const workspace = validateOperationalWorkspace(parseWorkspaceYaml(source));
if (workspace.workspace.id !== identity.workspaceId) {
throw new Error("workspace snapshot identity does not match its path");
}
return { workspace, identity, revisionContentRoot: dirname(snapshotPath) };
} finally {
closeSync(fd);
}
}
function runtimePaths(
dataRoot: string,
workspaceId: string,
): RuntimePaths {
if (!isAbsolute(dataRoot)) throw new Error("registry workspace runtime requires an absolute data root");
const root = join(dataRoot, "sessions", workspaceId);
return {
sessions: join(root, "sessions"),
artifacts: join(root, "artifacts"),
indexes: join(root, "indexes"),
memory: join(root, "memory"),
catalog_metadata_snapshot: join(root, "preprocessing", "catalog-metadata.json"),
};
}
function installationOverlay(harnessDir: string, configPath: string): RuntimeInstallationOverlay {
const path = isAbsolute(configPath) ? configPath : resolve(harnessDir, configPath);
try {
const documents = parseAllDocuments(readFileSync(path, "utf8"), { uniqueKeys: true });
if (documents.length !== 1) throw new Error("installation config must contain one YAML document");
const document = documents[0];
if (document.errors.length > 0 || document.warnings.length > 0) {
throw new Error("installation config contains invalid YAML");
}
const parsed = document.toJSON();
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
throw new Error("installation config must be a YAML mapping");
}
const source = parsed as Record<string, unknown>;
return {
...(source.session_storage === undefined ? {} : { session_storage: source.session_storage }),
...(source.profile === undefined ? {} : { profile: source.profile }),
};
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") return {};
throw error;
}
}
function applyCollectionLifecycle(config: string, lifecycle: "require_existing"): string {
const parsed = parse(config) as Record<string, any>;
if (!parsed.resources || typeof parsed.resources !== "object") {
throw new Error("runtime configuration is missing resources");
}
parsed.resources = { ...parsed.resources };
parsed.resources.vector = { ...(parsed.resources.vector ?? {}), collection_lifecycle: lifecycle };
return stringify(parsed, { lineWidth: 0, sortMapEntries: false });
}
function renderWorkspaceRuntimeFromWorkspace(options: {
workspace: WorkspaceDescriptor;
workspaceId: string;
workspaceRevision: string;
revisionContentRoot: string;
harnessDir: string;
configPath: string;
dataRoot: string;
secretRoots: readonly string[];
semanticRuntime: SemanticRuntimeConfig;
workspaceSecretStore?: WorkspaceSecretStore;
catalogDatabase?: WorkspaceDatabase;
}): RenderedWorkspaceRuntime {
if (options.catalogDatabase && !options.workspaceSecretStore) {
throw new Error("Catalog runtime binding requires the workspace secret store");
}
const catalogLease = options.catalogDatabase === undefined
? undefined
: resolveCatalogRuntimeBinding({
workspace: options.workspace,
database: options.catalogDatabase,
environment: process.env,
secretRoots: options.secretRoots,
secretStore: options.workspaceSecretStore!,
});
const runtimeWorkspace = catalogLease?.workspace ?? options.workspace;
const secretLease = catalogLease !== undefined || options.workspaceSecretStore === undefined
? undefined
: resolveRuntimeBindingsWithWorkspaceSecrets(
runtimeWorkspace,
process.env,
options.secretRoots,
options.workspaceSecretStore,
);
const bindings = catalogLease?.bindings ?? secretLease?.bindings
?? resolveRuntimeBindings(runtimeWorkspace, process.env, options.secretRoots);
const overlay = installationOverlay(options.harnessDir, options.configPath);
const context: RuntimeRenderContext = {
workspaceId: options.workspaceId,
workspaceRevision: options.workspaceRevision,
revisionContentRoot: options.revisionContentRoot,
};
try {
return {
workspace: options.workspace,
workspaceId: options.workspaceId,
workspaceRevision: options.workspaceRevision,
revisionContentRoot: options.revisionContentRoot,
runtimePaths: runtimePaths(options.dataRoot, options.workspaceId),
installationOverlay: overlay,
bindings,
bindingDigest: stableBindingDigest(bindings),
semanticQdrantUrl: options.semanticRuntime.internalQdrantUrl,
releaseSecrets: () => {
catalogLease?.release();
secretLease?.release();
},
renderedConfig: renderRuntimeConfig(
runtimeWorkspace,
bindings,
runtimePaths(options.dataRoot, options.workspaceId),
context,
overlay,
options.semanticRuntime,
),
};
} catch (error) {
catalogLease?.release();
secretLease?.release();
throw error;
}
}
export function renderWorkspaceRuntimeFromSnapshotPath(options: {
snapshotPath: string;
harnessDir: string;
configPath: string;
dataRoot: string;
secretRoots: readonly string[];
semanticRuntime: SemanticRuntimeConfig;
workspaceSecretStore?: WorkspaceSecretStore;
catalogDatabase?: WorkspaceDatabase;
}): RenderedWorkspaceRuntime {
const snapshot = readSnapshotWorkspace(options.snapshotPath);
return renderWorkspaceRuntimeFromWorkspace({
workspace: snapshot.workspace,
workspaceId: snapshot.identity.workspaceId,
workspaceRevision: snapshot.identity.workspaceRevision,
revisionContentRoot: snapshot.revisionContentRoot,
harnessDir: options.harnessDir,
configPath: options.configPath,
dataRoot: options.dataRoot,
secretRoots: options.secretRoots,
semanticRuntime: options.semanticRuntime,
workspaceSecretStore: options.workspaceSecretStore,
catalogDatabase: options.catalogDatabase,
});
}
export async function renderActiveWorkspaceRuntime(options: {
workspaceId: string;
registry: WorkspaceRegistry;
registryConfig: WorkspaceRegistryConfig;
harnessDir: string;
configPath: string;
dataRoot: string;
secretRoots: readonly string[];
semanticRuntime: SemanticRuntimeConfig;
workspaceSecretStore?: WorkspaceSecretStore;
catalogDatabase?: WorkspaceDatabase;
}): Promise<ActiveRenderedWorkspaceRuntime> {
// The persisted active state may reference host-side snapshot paths (written by another
// process or installation). Read the active state directly and resolve the immutable snapshot
// beneath this process's own configured registry root, so the path is correct inside the
// maintenance container and on the host. This deliberately bypasses WorkspaceRegistry.read,
// whose integrity check would fail on host-side paths inside the container.
const activePath = join(options.registryConfig.root, "state", "active.json");
const active = JSON.parse(await readFileAsync(activePath, "utf8")) as {
head: string;
revisions?: Array<{ id: string; commit: string; blob: string }>;
};
const revision = (active.revisions ?? []).find((entry) => entry.id === options.workspaceId);
if (!revision) throw new Error("workspace is not active");
const repository = new GitWorkspaceRepository(options.registryConfig);
await repository.ensureLayout();
const snapshotPath = options.registry.snapshotPath(revision.commit, revision.id);
const descriptorSource = await readFileAsync(snapshotPath, "utf8");
const workspace = parseWorkspaceYaml(descriptorSource);
const catalogSource = await repository.readCatalog(revision.commit);
const rendered = renderWorkspaceRuntimeFromWorkspace({
workspace,
workspaceId: revision.id,
workspaceRevision: revision.commit,
revisionContentRoot: dirname(snapshotPath),
harnessDir: options.harnessDir,
configPath: options.configPath,
dataRoot: options.dataRoot,
secretRoots: options.secretRoots,
semanticRuntime: options.semanticRuntime,
workspaceSecretStore: options.workspaceSecretStore,
catalogDatabase: options.catalogDatabase,
});
return {
...rendered,
snapshotPath,
descriptorBlob: `sha256:${createHash("sha256").update(descriptorSource).digest("hex")}`,
catalogBlob: `sha256:${createHash("sha256").update(catalogSource).digest("hex")}`,
};
}
function decodePublishedRuntimeConfigManifest(value: unknown): PublishedRuntimeConfigManifest {
if (!value || typeof value !== "object" || Array.isArray(value)) {
throw new RuntimeConfigLeaseError("effective_config_mismatch", "runtime configuration manifest is invalid");
}
const manifest = value as Record<string, unknown>;
const file = manifest.file as Record<string, unknown> | undefined;
if (
manifest.schemaVersion !== 1
|| typeof manifest.workspaceId !== "string"
|| typeof manifest.workspaceRevision !== "string"
|| typeof manifest.descriptorBlob !== "string"
|| typeof manifest.catalogBlob !== "string"
|| typeof manifest.configDigest !== "string"
|| typeof manifest.bindingDigest !== "string"
|| typeof manifest.effectiveConfigIdentity !== "string"
|| typeof manifest.configFingerprint !== "string"
|| typeof manifest.inputFingerprint !== "string"
|| typeof manifest.path !== "string"
|| !file
|| typeof file.dev !== "number"
|| typeof file.ino !== "number"
|| typeof file.size !== "number"
|| typeof file.mode !== "number"
|| typeof file.nlink !== "number"
) {
throw new RuntimeConfigLeaseError("effective_config_mismatch", "runtime configuration manifest is invalid");
}
return manifest as unknown as PublishedRuntimeConfigManifest;
}
export async function publishDeterministicRuntimeConfigLease(options: {
workspaceId: string;
registry: WorkspaceRegistry;
registryConfig: WorkspaceRegistryConfig;
harnessDir: string;
configPath: string;
dataRoot: string;
secretRoots: readonly string[];
semanticRuntime: SemanticRuntimeConfig;
workspaceSecretStore?: WorkspaceSecretStore;
catalogDatabase?: WorkspaceDatabase;
}): Promise<DeterministicRuntimeConfigLease> {
const rendered = await renderActiveWorkspaceRuntime(options);
const publishedConfig = applyCollectionLifecycle(rendered.renderedConfig, "require_existing");
const renderedConfigObject = parse(rendered.renderedConfig) as Record<string, unknown>;
const effectiveConfig = buildCanonicalEffectiveConfig(renderedConfigObject);
const effectiveConfigIdentityValue = effectiveConfigIdentity(rendered.workspaceId, renderedConfigObject);
const configFingerprintValue = configFingerprint(renderedConfigObject);
const inputFingerprintValue = preprocessingInputFingerprint(
rendered.workspaceId,
rendered.workspaceRevision,
renderedConfigObject,
);
// The Catalog input identity intentionally excludes representation-only changes.
// A lease also identifies its bytes, so a new renderer never collides with an
// immutable config produced by an earlier release for the same Catalog inputs.
const identitySuffix = sha256(inputFingerprintValue + "\n" + publishedConfig).slice(7, 23);
const preprocessingRoot = ensureTrustedDirectory(join(
options.dataRoot,
"sessions",
rendered.workspaceId,
"preprocessing",
));
const configDirectory = ensureTrustedDirectory(join(preprocessingRoot, "runtime-config"));
const manifestDirectory = ensureTrustedDirectory(join(preprocessingRoot, "runtime-config-manifests"));
const path = join(configDirectory, `${rendered.workspaceRevision}-${identitySuffix}.yaml`);
const manifestPath = join(manifestDirectory, `${rendered.workspaceRevision}-${identitySuffix}.json`);
const configDigest = sha256(publishedConfig);
const verifyPublished = (): PublishedRuntimeConfigManifest | undefined => {
let manifest: PublishedRuntimeConfigManifest | undefined;
try {
manifest = decodePublishedRuntimeConfigManifest(readStrictJson(manifestPath));
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
}
let configStat: ReturnType<typeof statSync> | undefined;
let contents: string | undefined;
try {
const file = readTrustedFile(path);
contents = file.contents;
configStat = file.stat;
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
}
if (contents === undefined) return manifest;
if ((Number(configStat!.mode) & 0o777) !== 0o400 || configStat!.nlink !== 1) {
throw new RuntimeConfigLeaseError("effective_config_mismatch", "runtime configuration lease changed");
}
if (sha256(contents) !== configDigest) {
throw new RuntimeConfigLeaseError("effective_config_mismatch", "runtime configuration lease changed");
}
if (!manifest) return undefined;
if (
manifest.workspaceId !== rendered.workspaceId
|| manifest.workspaceRevision !== rendered.workspaceRevision
|| manifest.descriptorBlob !== rendered.descriptorBlob
|| manifest.catalogBlob !== rendered.catalogBlob
|| manifest.configDigest !== configDigest
|| manifest.bindingDigest !== rendered.bindingDigest
|| manifest.effectiveConfigIdentity !== effectiveConfigIdentityValue
|| manifest.configFingerprint !== configFingerprintValue
|| manifest.inputFingerprint !== inputFingerprintValue
|| manifest.path !== path
|| manifest.file.dev !== configStat!.dev
|| manifest.file.ino !== configStat!.ino
|| manifest.file.size !== configStat!.size
|| manifest.file.mode !== (Number(configStat!.mode) & 0o777)
|| manifest.file.nlink !== configStat!.nlink
) {
throw new RuntimeConfigLeaseError("effective_config_mismatch", "runtime configuration lease changed");
}
return manifest;
};
const existing = verifyPublished();
if (existing) {
return {
path,
manifestPath,
workspaceId: rendered.workspaceId,
workspaceRevision: rendered.workspaceRevision,
descriptorBlob: rendered.descriptorBlob,
catalogBlob: rendered.catalogBlob,
configDigest,
bindingDigest: rendered.bindingDigest,
semanticQdrantUrl: rendered.semanticQdrantUrl,
effectiveConfig,
effectiveConfigIdentity: effectiveConfigIdentityValue,
configFingerprint: configFingerprintValue,
inputFingerprint: inputFingerprintValue,
release: () => rendered.releaseSecrets(),
};
}
try {
readTrustedFile(path);
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
writeAtomicFile(path, publishedConfig, 0o400);
} else {
throw error;
}
}
const published = readTrustedFile(path);
const publishedStat = published.stat!;
if (sha256(published.contents) !== configDigest) {
throw new RuntimeConfigLeaseError("effective_config_mismatch", "runtime configuration lease changed");
}
const manifest: PublishedRuntimeConfigManifest = {
schemaVersion: 1,
workspaceId: rendered.workspaceId,
workspaceRevision: rendered.workspaceRevision,
descriptorBlob: rendered.descriptorBlob,
catalogBlob: rendered.catalogBlob,
configDigest,
bindingDigest: rendered.bindingDigest,
effectiveConfigIdentity: effectiveConfigIdentityValue,
configFingerprint: configFingerprintValue,
inputFingerprint: inputFingerprintValue,
path,
file: {
dev: Number(publishedStat.dev),
ino: Number(publishedStat.ino),
size: Number(publishedStat.size),
mode: Number(publishedStat.mode) & 0o777,
nlink: Number(publishedStat.nlink),
},
};
try {
readStrictJson(manifestPath);
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
writeAtomicFile(manifestPath, `${JSON.stringify(manifest)}
`, 0o600);
} else {
throw error;
}
}
verifyPublished();
return {
path,
manifestPath,
workspaceId: rendered.workspaceId,
workspaceRevision: rendered.workspaceRevision,
descriptorBlob: rendered.descriptorBlob,
catalogBlob: rendered.catalogBlob,
configDigest,
bindingDigest: rendered.bindingDigest,
semanticQdrantUrl: rendered.semanticQdrantUrl,
effectiveConfig,
effectiveConfigIdentity: effectiveConfigIdentityValue,
configFingerprint: configFingerprintValue,
inputFingerprint: inputFingerprintValue,
release: () => rendered.releaseSecrets(),
};
}