ThothII/backend/src/workspaces/runtime-config-lease.ts

import { createHash, randomUUID } from "node:crypto";
import {
  closeSync,
  constants as fsConstants,
  fchmodSync,
  fstatSync,
  fsyncSync,
  lstatSync,
  mkdirSync,
  openSync,
  readFileSync,
  readSync,
  renameSync,
  statSync,
  unlinkSync,
  writeFileSync,
} from "node:fs";
import { dirname, isAbsolute, join, relative, resolve } from "node:path";
import { readFile as readFileAsync } from "node:fs/promises";
import { parse, parseAllDocuments, stringify } from "yaml";
import {
  buildCanonicalEffectiveConfig,
  canonicalEffectiveConfigJson,
  configFingerprint,
  effectiveConfigIdentity,
  preprocessingInputFingerprint,
  type CanonicalEffectiveConfig,
} from "./effective-config.js";
import { resolveRuntimeBindings, type RuntimeBindings } from "./bindings.js";
import { resolveRuntimeBindingsWithWorkspaceSecrets } from "./secret-requirements.js";
import type { WorkspaceSecretStore } from "./secret-store.js";
import { GitWorkspaceRepository } from "./git-repository.js";
import { WorkspaceRegistry } from "./registry.js";
import {
  renderRuntimeConfig,
  type RuntimeIdentity,
  type RuntimeInstallationOverlay,
  type RuntimePaths,
  type RuntimeRenderContext,
  type SemanticRuntimeConfig,
} from "./runtime-renderer.js";
import { parseWorkspaceYaml, validateOperationalWorkspace, type WorkspaceDescriptor } from "./schema.js";
import type { WorkspaceRegistryConfig } from "./types.js";
import { resolveCatalogRuntimeBinding } from "../catalog/runtime-binding.js";
import type { WorkspaceDatabase } from "../catalog/types.js";

export interface RuntimeConfigLease {
  path: string;
  workspaceId: string;
  workspaceRevision: string;
  release(): void;
}

export interface RenderedWorkspaceRuntime {
  workspace: WorkspaceDescriptor;
  workspaceId: string;
  workspaceRevision: string;
  revisionContentRoot: string;
  runtimePaths: RuntimePaths;
  installationOverlay: RuntimeInstallationOverlay;
  bindings: RuntimeBindings;
  bindingDigest: string;
  renderedConfig: string;
  semanticQdrantUrl: string;
  releaseSecrets(): void;
}

export interface ActiveRenderedWorkspaceRuntime extends RenderedWorkspaceRuntime {
  snapshotPath: string;
  descriptorBlob: string;
  catalogBlob: string;
}

export interface DeterministicRuntimeConfigLease extends RuntimeConfigLease {
  manifestPath: string;
  descriptorBlob: string;
  catalogBlob: string;
  configDigest: string;
  bindingDigest: string;
  semanticQdrantUrl: string;
  effectiveConfig: CanonicalEffectiveConfig;
  effectiveConfigIdentity: string;
  configFingerprint: string;
  inputFingerprint: string;
}

export class RuntimeConfigLeaseError extends Error {
  constructor(readonly code: "effective_config_mismatch", message: string) {
    super(message);
    this.name = "RuntimeConfigLeaseError";
  }
}

interface SnapshotIdentity extends RuntimeIdentity {
  snapshotPath: string;
}

interface PublishedRuntimeConfigManifest {
  schemaVersion: 1;
  workspaceId: string;
  workspaceRevision: string;
  descriptorBlob: string;
  catalogBlob: string;
  configDigest: string;
  bindingDigest: string;
  effectiveConfigIdentity: string;
  configFingerprint: string;
  inputFingerprint: string;
  path: string;
  file: {
    dev: number;
    ino: number;
    size: number;
    mode: number;
    nlink: number;
  };
}

function sha256(value: string | Buffer): string {
  return `sha256:${createHash("sha256").update(value).digest("hex")}`;
}

function stableBindingDigest(bindings: RuntimeBindings): string {
  const encodeRecord = (value: Record<string, string>) => Object.entries(value).sort(([left], [right]) => (
    left.localeCompare(right)
  ));
  return sha256(JSON.stringify({
    dwh: {
      transport: bindings.dwh.transport,
      values: encodeRecord(bindings.dwh.values),
      missing: [...bindings.dwh.missing].sort(),
    },
    evidence: {
      values: encodeRecord(bindings.evidence.values),
      missing: [...bindings.evidence.missing].sort(),
    },
  }));
}

function syncDirectory(directory: string): void {
  if (process.platform === "win32") return;
  const fd = openSync(directory, "r");
  try { fsyncSync(fd); } finally { closeSync(fd); }
}

function ensureTrustedDirectory(directory: string): string {
  mkdirSync(directory, { recursive: true, mode: 0o700 });
  const entry = lstatSync(directory);
  if (!entry.isDirectory() || entry.isSymbolicLink()) {
    throw new Error("runtime configuration directory is unavailable");
  }
  return directory;
}

function writeAtomicFile(path: string, contents: string, mode: number): void {
  ensureTrustedDirectory(dirname(path));
  const staging = `${path}.tmp-${process.pid}-${Date.now()}-${randomUUID()}`;
  const fd = openSync(
    staging,
    fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_NOFOLLOW,
    0o600,
  );
  let closed = false;
  try {
    writeFileSync(fd, contents, "utf8");
    fsyncSync(fd);
    fchmodSync(fd, mode);
    closeSync(fd);
    closed = true;
    renameSync(staging, path);
    syncDirectory(dirname(path));
  } catch (error) {
    if (!closed) try { closeSync(fd); } catch { /* preserve original error */ }
    try { unlinkSync(staging); } catch { /* best effort */ }
    throw error;
  }
}

function readTrustedFile(path: string): { contents: string; stat: ReturnType<typeof statSync> } {
  const entry = lstatSync(path);
  if (!entry.isFile() || entry.isSymbolicLink()) {
    throw new Error("runtime configuration file is untrusted");
  }
  const fd = openSync(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
  try {
    const before = fstatSync(fd);
    if (!before.isFile() || before.nlink !== 1) {
      throw new Error("runtime configuration file is untrusted");
    }
    const contents = readFileSync(fd, "utf8");
    const after = fstatSync(fd);
    if (
      before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size || before.nlink !== after.nlink
    ) {
      throw new Error("runtime configuration file changed while reading");
    }
    return { contents, stat: statSync(path) };
  } finally {
    closeSync(fd);
  }
}

function readStrictJson(path: string): unknown {
  return JSON.parse(readTrustedFile(path).contents);
}

function trustedSnapshotIdentity(snapshotPath: string): SnapshotIdentity {
  if (!isAbsolute(snapshotPath)) throw new Error("config path is not a trusted runtime snapshot");
  const commitDirectory = dirname(snapshotPath);
  const snapshotsRoot = dirname(commitDirectory);
  const pathRelative = relative(snapshotsRoot, snapshotPath);
  const match = /^([0-9a-f]{40})\/([a-z][a-z0-9-]{2,62})\.yaml$/.exec(pathRelative);
  if (pathRelative.startsWith("..") || isAbsolute(pathRelative) || !match) {
    throw new Error("config path is not a trusted runtime snapshot");
  }
  const entry = lstatSync(snapshotPath);
  if (!entry.isFile() || entry.isSymbolicLink()) {
    throw new Error("config path is not a trusted runtime snapshot");
  }
  return { snapshotPath, workspaceRevision: match[1], workspaceId: match[2] };
}

function readSnapshotWorkspace(snapshotPath: string): {
  workspace: WorkspaceDescriptor;
  identity: SnapshotIdentity;
  revisionContentRoot: string;
} {
  const identity = trustedSnapshotIdentity(snapshotPath);
  const fd = openSync(snapshotPath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
  try {
    const before = fstatSync(fd);
    if (!before.isFile() || before.nlink !== 1) throw new Error("workspace snapshot is not a file");
    const source = readFileSync(fd, "utf8");
    const after = fstatSync(fd);
    if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size) {
      throw new Error("workspace snapshot changed while reading");
    }
    const workspace = validateOperationalWorkspace(parseWorkspaceYaml(source));
    if (workspace.workspace.id !== identity.workspaceId) {
      throw new Error("workspace snapshot identity does not match its path");
    }
    return { workspace, identity, revisionContentRoot: dirname(snapshotPath) };
  } finally {
    closeSync(fd);
  }
}

function runtimePaths(
  dataRoot: string,
  workspaceId: string,
): RuntimePaths {
  if (!isAbsolute(dataRoot)) throw new Error("registry workspace runtime requires an absolute data root");
  const root = join(dataRoot, "sessions", workspaceId);
  return {
    sessions: join(root, "sessions"),
    artifacts: join(root, "artifacts"),
    indexes: join(root, "indexes"),
    memory: join(root, "memory"),
    catalog_metadata_snapshot: join(root, "preprocessing", "catalog-metadata.json"),
  };
}

function installationOverlay(harnessDir: string, configPath: string): RuntimeInstallationOverlay {
  const path = isAbsolute(configPath) ? configPath : resolve(harnessDir, configPath);
  try {
    const documents = parseAllDocuments(readFileSync(path, "utf8"), { uniqueKeys: true });
    if (documents.length !== 1) throw new Error("installation config must contain one YAML document");
    const document = documents[0];
    if (document.errors.length > 0 || document.warnings.length > 0) {
      throw new Error("installation config contains invalid YAML");
    }
    const parsed = document.toJSON();
    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
      throw new Error("installation config must be a YAML mapping");
    }
    const source = parsed as Record<string, unknown>;
    return {
      ...(source.session_storage === undefined ? {} : { session_storage: source.session_storage }),
      ...(source.profile === undefined ? {} : { profile: source.profile }),
    };
  } catch (error) {
    if ((error as NodeJS.ErrnoException).code === "ENOENT") return {};
    throw error;
  }
}

function applyCollectionLifecycle(config: string, lifecycle: "require_existing"): string {
  const parsed = parse(config) as Record<string, any>;
  if (!parsed.resources || typeof parsed.resources !== "object") {
    throw new Error("runtime configuration is missing resources");
  }
  parsed.resources = { ...parsed.resources };
  parsed.resources.vector = { ...(parsed.resources.vector ?? {}), collection_lifecycle: lifecycle };
  return stringify(parsed, { lineWidth: 0, sortMapEntries: false });
}

function renderWorkspaceRuntimeFromWorkspace(options: {
  workspace: WorkspaceDescriptor;
  workspaceId: string;
  workspaceRevision: string;
  revisionContentRoot: string;
  harnessDir: string;
  configPath: string;
  dataRoot: string;
  secretRoots: readonly string[];
  semanticRuntime: SemanticRuntimeConfig;
  workspaceSecretStore?: WorkspaceSecretStore;
  catalogDatabase?: WorkspaceDatabase;
}): RenderedWorkspaceRuntime {
  if (options.catalogDatabase && !options.workspaceSecretStore) {
    throw new Error("Catalog runtime binding requires the workspace secret store");
  }
  const catalogLease = options.catalogDatabase === undefined
    ? undefined
    : resolveCatalogRuntimeBinding({
      workspace: options.workspace,
      database: options.catalogDatabase,
      environment: process.env,
      secretRoots: options.secretRoots,
      secretStore: options.workspaceSecretStore!,
    });
  const runtimeWorkspace = catalogLease?.workspace ?? options.workspace;
  const secretLease = catalogLease !== undefined || options.workspaceSecretStore === undefined
    ? undefined
    : resolveRuntimeBindingsWithWorkspaceSecrets(
      runtimeWorkspace,
      process.env,
      options.secretRoots,
      options.workspaceSecretStore,
    );
  const bindings = catalogLease?.bindings ?? secretLease?.bindings
    ?? resolveRuntimeBindings(runtimeWorkspace, process.env, options.secretRoots);
  const overlay = installationOverlay(options.harnessDir, options.configPath);
  const context: RuntimeRenderContext = {
    workspaceId: options.workspaceId,
    workspaceRevision: options.workspaceRevision,
    revisionContentRoot: options.revisionContentRoot,
  };
  try {
    return {
      workspace: options.workspace,
      workspaceId: options.workspaceId,
      workspaceRevision: options.workspaceRevision,
      revisionContentRoot: options.revisionContentRoot,
      runtimePaths: runtimePaths(options.dataRoot, options.workspaceId),
      installationOverlay: overlay,
      bindings,
      bindingDigest: stableBindingDigest(bindings),
      semanticQdrantUrl: options.semanticRuntime.internalQdrantUrl,
      releaseSecrets: () => {
        catalogLease?.release();
        secretLease?.release();
      },
      renderedConfig: renderRuntimeConfig(
        runtimeWorkspace,
        bindings,
        runtimePaths(options.dataRoot, options.workspaceId),
        context,
        overlay,
        options.semanticRuntime,
      ),
    };
  } catch (error) {
    catalogLease?.release();
    secretLease?.release();
    throw error;
  }
}

export function renderWorkspaceRuntimeFromSnapshotPath(options: {
  snapshotPath: string;
  harnessDir: string;
  configPath: string;
  dataRoot: string;
  secretRoots: readonly string[];
  semanticRuntime: SemanticRuntimeConfig;
  workspaceSecretStore?: WorkspaceSecretStore;
  catalogDatabase?: WorkspaceDatabase;
}): RenderedWorkspaceRuntime {
  const snapshot = readSnapshotWorkspace(options.snapshotPath);
  return renderWorkspaceRuntimeFromWorkspace({
    workspace: snapshot.workspace,
    workspaceId: snapshot.identity.workspaceId,
    workspaceRevision: snapshot.identity.workspaceRevision,
    revisionContentRoot: snapshot.revisionContentRoot,
    harnessDir: options.harnessDir,
    configPath: options.configPath,
    dataRoot: options.dataRoot,
    secretRoots: options.secretRoots,
    semanticRuntime: options.semanticRuntime,
    workspaceSecretStore: options.workspaceSecretStore,
    catalogDatabase: options.catalogDatabase,
  });
}

export async function renderActiveWorkspaceRuntime(options: {
  workspaceId: string;
  registry: WorkspaceRegistry;
  registryConfig: WorkspaceRegistryConfig;
  harnessDir: string;
  configPath: string;
  dataRoot: string;
  secretRoots: readonly string[];
  semanticRuntime: SemanticRuntimeConfig;
  workspaceSecretStore?: WorkspaceSecretStore;
  catalogDatabase?: WorkspaceDatabase;
}): Promise<ActiveRenderedWorkspaceRuntime> {
  // The persisted active state may reference host-side snapshot paths (written by another
  // process or installation). Read the active state directly and resolve the immutable snapshot
  // beneath this process's own configured registry root, so the path is correct inside the
  // maintenance container and on the host. This deliberately bypasses WorkspaceRegistry.read,
  // whose integrity check would fail on host-side paths inside the container.
  const activePath = join(options.registryConfig.root, "state", "active.json");
  const active = JSON.parse(await readFileAsync(activePath, "utf8")) as {
    head: string;
    revisions?: Array<{ id: string; commit: string; blob: string }>;
  };
  const revision = (active.revisions ?? []).find((entry) => entry.id === options.workspaceId);
  if (!revision) throw new Error("workspace is not active");
  const repository = new GitWorkspaceRepository(options.registryConfig);
  await repository.ensureLayout();
  const snapshotPath = options.registry.snapshotPath(revision.commit, revision.id);
  const descriptorSource = await readFileAsync(snapshotPath, "utf8");
  const workspace = parseWorkspaceYaml(descriptorSource);
  const catalogSource = await repository.readCatalog(revision.commit);
  const rendered = renderWorkspaceRuntimeFromWorkspace({
    workspace,
    workspaceId: revision.id,
    workspaceRevision: revision.commit,
    revisionContentRoot: dirname(snapshotPath),
    harnessDir: options.harnessDir,
    configPath: options.configPath,
    dataRoot: options.dataRoot,
    secretRoots: options.secretRoots,
    semanticRuntime: options.semanticRuntime,
    workspaceSecretStore: options.workspaceSecretStore,
    catalogDatabase: options.catalogDatabase,
  });
  return {
    ...rendered,
    snapshotPath,
    descriptorBlob: `sha256:${createHash("sha256").update(descriptorSource).digest("hex")}`,
    catalogBlob: `sha256:${createHash("sha256").update(catalogSource).digest("hex")}`,
  };
}

function decodePublishedRuntimeConfigManifest(value: unknown): PublishedRuntimeConfigManifest {
  if (!value || typeof value !== "object" || Array.isArray(value)) {
    throw new RuntimeConfigLeaseError("effective_config_mismatch", "runtime configuration manifest is invalid");
  }
  const manifest = value as Record<string, unknown>;
  const file = manifest.file as Record<string, unknown> | undefined;
  if (
    manifest.schemaVersion !== 1
    || typeof manifest.workspaceId !== "string"
    || typeof manifest.workspaceRevision !== "string"
    || typeof manifest.descriptorBlob !== "string"
    || typeof manifest.catalogBlob !== "string"
    || typeof manifest.configDigest !== "string"
    || typeof manifest.bindingDigest !== "string"
    || typeof manifest.effectiveConfigIdentity !== "string"
    || typeof manifest.configFingerprint !== "string"
    || typeof manifest.inputFingerprint !== "string"
    || typeof manifest.path !== "string"
    || !file
    || typeof file.dev !== "number"
    || typeof file.ino !== "number"
    || typeof file.size !== "number"
    || typeof file.mode !== "number"
    || typeof file.nlink !== "number"
  ) {
    throw new RuntimeConfigLeaseError("effective_config_mismatch", "runtime configuration manifest is invalid");
  }
  return manifest as unknown as PublishedRuntimeConfigManifest;
}

export async function publishDeterministicRuntimeConfigLease(options: {
  workspaceId: string;
  registry: WorkspaceRegistry;
  registryConfig: WorkspaceRegistryConfig;
  harnessDir: string;
  configPath: string;
  dataRoot: string;
  secretRoots: readonly string[];
  semanticRuntime: SemanticRuntimeConfig;
  workspaceSecretStore?: WorkspaceSecretStore;
  catalogDatabase?: WorkspaceDatabase;
}): Promise<DeterministicRuntimeConfigLease> {
  const rendered = await renderActiveWorkspaceRuntime(options);
  const publishedConfig = applyCollectionLifecycle(rendered.renderedConfig, "require_existing");
  const renderedConfigObject = parse(rendered.renderedConfig) as Record<string, unknown>;
  const effectiveConfig = buildCanonicalEffectiveConfig(renderedConfigObject);
  const effectiveConfigIdentityValue = effectiveConfigIdentity(rendered.workspaceId, renderedConfigObject);
  const configFingerprintValue = configFingerprint(renderedConfigObject);
  const inputFingerprintValue = preprocessingInputFingerprint(
    rendered.workspaceId,
    rendered.workspaceRevision,
    renderedConfigObject,
  );
  const identitySuffix = inputFingerprintValue.slice(7, 23);

  const preprocessingRoot = ensureTrustedDirectory(join(
    options.dataRoot,
    "sessions",
    rendered.workspaceId,
    "preprocessing",
  ));
  const configDirectory = ensureTrustedDirectory(join(preprocessingRoot, "runtime-config"));
  const manifestDirectory = ensureTrustedDirectory(join(preprocessingRoot, "runtime-config-manifests"));
  const path = join(configDirectory, `${rendered.workspaceRevision}-${identitySuffix}.yaml`);
  const manifestPath = join(manifestDirectory, `${rendered.workspaceRevision}-${identitySuffix}.json`);
  const configDigest = sha256(publishedConfig);

  const verifyPublished = (): PublishedRuntimeConfigManifest | undefined => {
    let manifest: PublishedRuntimeConfigManifest | undefined;
    try {
      manifest = decodePublishedRuntimeConfigManifest(readStrictJson(manifestPath));
    } catch (error) {
      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
    }
    let configStat: ReturnType<typeof statSync> | undefined;
    let contents: string | undefined;
    try {
      const file = readTrustedFile(path);
      contents = file.contents;
      configStat = file.stat;
    } catch (error) {
      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
    }
    if (contents === undefined) return manifest;
    if ((Number(configStat!.mode) & 0o777) !== 0o400 || configStat!.nlink !== 1) {
      throw new RuntimeConfigLeaseError("effective_config_mismatch", "runtime configuration lease changed");
    }
    if (sha256(contents) !== configDigest) {
      throw new RuntimeConfigLeaseError("effective_config_mismatch", "runtime configuration lease changed");
    }
    if (!manifest) return undefined;
    if (
      manifest.workspaceId !== rendered.workspaceId
      || manifest.workspaceRevision !== rendered.workspaceRevision
      || manifest.descriptorBlob !== rendered.descriptorBlob
      || manifest.catalogBlob !== rendered.catalogBlob
      || manifest.configDigest !== configDigest
      || manifest.bindingDigest !== rendered.bindingDigest
      || manifest.effectiveConfigIdentity !== effectiveConfigIdentityValue
      || manifest.configFingerprint !== configFingerprintValue
      || manifest.inputFingerprint !== inputFingerprintValue
      || manifest.path !== path
      || manifest.file.dev !== configStat!.dev
      || manifest.file.ino !== configStat!.ino
      || manifest.file.size !== configStat!.size
      || manifest.file.mode !== (Number(configStat!.mode) & 0o777)
      || manifest.file.nlink !== configStat!.nlink
    ) {
      throw new RuntimeConfigLeaseError("effective_config_mismatch", "runtime configuration lease changed");
    }
    return manifest;
  };

  const existing = verifyPublished();
  if (existing) {
    return {
      path,
      manifestPath,
      workspaceId: rendered.workspaceId,
      workspaceRevision: rendered.workspaceRevision,
      descriptorBlob: rendered.descriptorBlob,
      catalogBlob: rendered.catalogBlob,
      configDigest,
      bindingDigest: rendered.bindingDigest,
      semanticQdrantUrl: rendered.semanticQdrantUrl,
      effectiveConfig,
      effectiveConfigIdentity: effectiveConfigIdentityValue,
      configFingerprint: configFingerprintValue,
      inputFingerprint: inputFingerprintValue,
      release: () => rendered.releaseSecrets(),
    };
  }

  try {
    readTrustedFile(path);
  } catch (error) {
    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
      writeAtomicFile(path, publishedConfig, 0o400);
    } else {
      throw error;
    }
  }
  const published = readTrustedFile(path);
  const publishedStat = published.stat!;
  if (sha256(published.contents) !== configDigest) {
    throw new RuntimeConfigLeaseError("effective_config_mismatch", "runtime configuration lease changed");
  }
  const manifest: PublishedRuntimeConfigManifest = {
    schemaVersion: 1,
    workspaceId: rendered.workspaceId,
    workspaceRevision: rendered.workspaceRevision,
    descriptorBlob: rendered.descriptorBlob,
    catalogBlob: rendered.catalogBlob,
    configDigest,
    bindingDigest: rendered.bindingDigest,
    effectiveConfigIdentity: effectiveConfigIdentityValue,
    configFingerprint: configFingerprintValue,
    inputFingerprint: inputFingerprintValue,
    path,
    file: {
      dev: Number(publishedStat.dev),
      ino: Number(publishedStat.ino),
      size: Number(publishedStat.size),
      mode: Number(publishedStat.mode) & 0o777,
      nlink: Number(publishedStat.nlink),
    },
  };
  try {
    readStrictJson(manifestPath);
  } catch (error) {
    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
      writeAtomicFile(manifestPath, `${JSON.stringify(manifest)}
`, 0o600);
    } else {
      throw error;
    }
  }
  verifyPublished();
  return {
    path,
    manifestPath,
    workspaceId: rendered.workspaceId,
    workspaceRevision: rendered.workspaceRevision,
    descriptorBlob: rendered.descriptorBlob,
    catalogBlob: rendered.catalogBlob,
    configDigest,
    bindingDigest: rendered.bindingDigest,
    semanticQdrantUrl: rendered.semanticQdrantUrl,
    effectiveConfig,
    effectiveConfigIdentity: effectiveConfigIdentityValue,
    configFingerprint: configFingerprintValue,
    inputFingerprint: inputFingerprintValue,
    release: () => rendered.releaseSecrets(),
  };
}