feat: P3 effective configuration, memory root, and revision-scoped records

backend/src/workspaces/effective-config.ts

@@ -0,0 +1,219 @@
+import { createHash } from "node:crypto";
+import { normalize } from "node:path";
+
+export interface CanonicalEffectiveConfig {
+  schemaVersion: 1;
+  dwh: CanonicalDwhConfig;
+  vector: CanonicalVectorConfig;
+  embedding: CanonicalEmbeddingConfig;
+  roots: CanonicalRootsConfig;
+}
+
+export interface CanonicalDwhConfig {
+  engine: "postgres";
+  database: string;
+  schema: string;
+  transport: "postgres_direct" | "rest_api" | "ssh_tunnel";
+  host?: string;
+  port?: number;
+  baseUrl?: string;
+  user?: string;
+}
+
+export interface CanonicalVectorConfig {
+  collection: string;
+  dimensions: number;
+  distance: string;
+}
+
+export interface CanonicalEmbeddingConfig {
+  model: string;
+  dimensions: number;
+}
+
+export interface CanonicalRootsConfig {
+  artifacts: string;
+  indexes: string;
+}
+
+function asRecord(value: unknown): Record<string, unknown> | undefined {
+  if (typeof value === "object" && value !== null && !Array.isArray(value)) {
+    return value as Record<string, unknown>;
+  }
+  return undefined;
+}
+
+function requireString(value: Record<string, unknown>, key: string): string {
+  const candidate = value[key];
+  if (typeof candidate !== "string" || candidate.length === 0) {
+    throw new TypeError(`effective config is missing ${key}`);
+  }
+  return candidate;
+}
+
+function optionalString(value: Record<string, unknown>, key: string): string | undefined {
+  const candidate = value[key];
+  if (candidate === undefined || candidate === null) return undefined;
+  if (typeof candidate !== "string") return undefined;
+  return candidate;
+}
+
+function optionalNumber(value: Record<string, unknown>, key: string): number | undefined {
+  const candidate = value[key];
+  if (candidate === undefined || candidate === null) return undefined;
+  if (typeof candidate !== "number" || Number.isNaN(candidate)) return undefined;
+  return candidate;
+}
+
+function requireNumber(value: Record<string, unknown>, key: string): number {
+  const candidate = value[key];
+  if (typeof candidate !== "number" || Number.isNaN(candidate)) {
+    throw new TypeError(`effective config is missing numeric ${key}`);
+  }
+  return candidate;
+}
+
+function normalizeRoot(value: string): string {
+  return normalize(value);
+}
+
+function dwhTransport(rendered: Record<string, unknown>): CanonicalDwhConfig["transport"] {
+  const dwh = asRecord(rendered.dwh);
+  const type = dwh ? requireString(dwh, "type") : undefined;
+  if (type === "postgres_direct") return "postgres_direct";
+  if (type === "thoth_rest") return "rest_api";
+  if (type === "ssh_tunnel") return "ssh_tunnel";
+  throw new TypeError(`effective config has unsupported dwh transport ${type}`);
+}
+
+function buildDwhConfig(rendered: Record<string, unknown>): CanonicalDwhConfig {
+  const transport = dwhTransport(rendered);
+  const databaseRecord = asRecord(rendered.database) ?? asRecord(asRecord(asRecord(rendered.dwh)?.connection)?.database);
+  if (!databaseRecord) {
+    throw new TypeError("effective config is missing database identity");
+  }
+  const engine = "postgres";
+  const database = requireString(databaseRecord, "database");
+  const schema = requireString(databaseRecord, "schema");
+  const dwh: Record<string, unknown> = { engine, database, schema, transport };
+
+  if (transport === "postgres_direct") {
+    const host = optionalString(databaseRecord, "host");
+    const port = optionalNumber(databaseRecord, "port");
+    const user = optionalString(databaseRecord, "user");
+    if (host !== undefined) dwh.host = host;
+    if (port !== undefined) dwh.port = port;
+    if (user !== undefined) dwh.user = user;
+  } else if (transport === "rest_api") {
+    const rest = asRecord(rendered.rest) ?? asRecord(asRecord(asRecord(rendered.dwh)?.endpoint));
+    const baseUrl = rest ? optionalString(rest, "base_url") : undefined;
+    if (baseUrl !== undefined) dwh.baseUrl = baseUrl;
+  }
+
+  return dwh as unknown as CanonicalDwhConfig;
+}
+
+function buildVectorConfig(rendered: Record<string, unknown>): CanonicalVectorConfig {
+  const resources = asRecord(rendered.resources);
+  const vector = resources ? asRecord(resources.vector) : undefined;
+  if (!vector) {
+    throw new TypeError("effective config is missing vector resources");
+  }
+  const collection = requireString(vector, "collection");
+  const semanticIndex = asRecord(rendered.semantic_index);
+  const vectorStore = semanticIndex ? asRecord(semanticIndex.vector_store) : undefined;
+  const dimensions = vectorStore
+    ? requireNumber(vectorStore, "dimensions")
+    : requireNumber(vector, "dimensions");
+  const distance = vectorStore
+    ? requireString(vectorStore, "distance")
+    : (optionalString(vector, "distance") ?? "cosine");
+  return { collection, dimensions, distance };
+}
+
+function buildEmbeddingConfig(rendered: Record<string, unknown>): CanonicalEmbeddingConfig {
+  const resources = asRecord(rendered.resources);
+  const embeddings = resources ? asRecord(resources.embeddings) : undefined;
+  if (!embeddings) {
+    throw new TypeError("effective config is missing embedding resources");
+  }
+  return {
+    model: requireString(embeddings, "model"),
+    dimensions: requireNumber(embeddings, "dimensions"),
+  };
+}
+
+function buildRootsConfig(rendered: Record<string, unknown>): CanonicalRootsConfig {
+  const roots = asRecord(rendered.roots) ?? asRecord(rendered.paths);
+  if (!roots) {
+    throw new TypeError("effective config is missing roots");
+  }
+  return {
+    artifacts: normalizeRoot(requireString(roots, "artifacts")),
+    indexes: normalizeRoot(requireString(roots, "indexes")),
+  };
+}
+
+/**
+ * Build the versioned, non-secret effective DWH/preprocessing configuration from a
+ * rendered runtime configuration object. The result contains only the fields that
+ * affect DWH generation identity; credentials, runtime identity, session storage,
+ * evidence, and service endpoints are excluded.
+ */
+export function buildCanonicalEffectiveConfig(renderedConfig: unknown): CanonicalEffectiveConfig {
+  const rendered = asRecord(renderedConfig);
+  if (!rendered) {
+    throw new TypeError("effective config requires a rendered configuration object");
+  }
+  return {
+    schemaVersion: 1,
+    dwh: buildDwhConfig(rendered),
+    vector: buildVectorConfig(rendered),
+    embedding: buildEmbeddingConfig(rendered),
+    roots: buildRootsConfig(rendered),
+  };
+}
+
+function sha256(value: string | Buffer): string {
+  return `sha256:${createHash("sha256").update(value).digest("hex")}`;
+}
+
+/**
+ * Serialize the canonical effective config to a deterministic JSON string with the
+ * fixed key order defined by the shared contract. No whitespace is included.
+ */
+export function canonicalEffectiveConfigJson(config: CanonicalEffectiveConfig): string {
+  const ordered: Record<string, unknown> = { schemaVersion: config.schemaVersion };
+  ordered.dwh = { ...config.dwh };
+  ordered.vector = { ...config.vector };
+  ordered.embedding = { ...config.embedding };
+  ordered.roots = { ...config.roots };
+  return JSON.stringify(ordered);
+}
+
+/**
+ * Return the stable logical config-source identity for a workspace revision.
+ * This is `workspace://<workspaceId>@v1:<sha256-hex-of-canonical-json>`.
+ */
+export function effectiveConfigIdentity(workspaceId: string, renderedConfig: unknown): string {
+  const canonical = buildCanonicalEffectiveConfig(renderedConfig);
+  const digest = createHash("sha256").update(canonicalEffectiveConfigJson(canonical)).digest("hex");
+  return `workspace://${workspaceId}@v1:${digest}`;
+}
+
+/**
+ * Return the config fingerprint: `sha256:` + the SHA-256 of the canonical effective
+ * config JSON bytes.
+ */
+export function configFingerprint(renderedConfig: unknown): string {
+  const canonical = buildCanonicalEffectiveConfig(renderedConfig);
+  return sha256(canonicalEffectiveConfigJson(canonical));
+}
+
+/**
+ * Return the input fingerprint: `sha256:` + the SHA-256 of the logical config-source
+ * identity string.
+ */
+export function inputFingerprint(workspaceId: string, renderedConfig: unknown): string {
+  return sha256(effectiveConfigIdentity(workspaceId, renderedConfig));
+}

backend/src/workspaces/preprocessing-service.ts

@@ -30,6 +30,9 @@ export interface WorkspaceOperationResult {
   completedStages: string[];
   counts?: Record<string, number>;
   artifactIdentities?: Array<{ kind: string; digest: string }>;
+  effectiveConfigIdentity?: string;
+  configFingerprint?: string;
+  inputFingerprint?: string;
   /** Suggested FK annotations YAML for the operator to write to --output (schema suggest-fks). */
   suggestedFksYaml?: string;
   warnings?: string[];
@@ -92,6 +95,9 @@ function baseResult(
     descriptorBlob: runtime.descriptorBlob,
     operation,
     completedStages: [],
+    effectiveConfigIdentity: runtime.configLease.effectiveConfigIdentity,
+    configFingerprint: runtime.configLease.configFingerprint,
+    inputFingerprint: runtime.configLease.inputFingerprint,
     ...extra,
   };
 }

backend/src/workspaces/runtime-config-lease.ts

@@ -18,6 +18,14 @@ import {
 import { dirname, isAbsolute, join, relative, resolve } from "node:path";
 import { readFile as readFileAsync } from "node:fs/promises";
 import { parse, parseAllDocuments, stringify } from "yaml";
+import {
+  buildCanonicalEffectiveConfig,
+  canonicalEffectiveConfigJson,
+  configFingerprint,
+  effectiveConfigIdentity,
+  inputFingerprint,
+  type CanonicalEffectiveConfig,
+} from "./effective-config.js";
 import { resolveRuntimeBindings, type RuntimeBindings } from "./bindings.js";
 import { GitWorkspaceRepository } from "./git-repository.js";
 import { WorkspaceRegistry } from "./registry.js";
@@ -63,6 +71,10 @@ export interface DeterministicRuntimeConfigLease extends RuntimeConfigLease {
   catalogBlob: string;
   configDigest: string;
   bindingDigest: string;
+  effectiveConfig: CanonicalEffectiveConfig;
+  effectiveConfigIdentity: string;
+  configFingerprint: string;
+  inputFingerprint: string;
 }
 
 export class RuntimeConfigLeaseError extends Error {
@@ -84,6 +96,9 @@ interface PublishedRuntimeConfigManifest {
   catalogBlob: string;
   configDigest: string;
   bindingDigest: string;
+  effectiveConfigIdentity: string;
+  configFingerprint: string;
+  inputFingerprint: string;
   path: string;
   file: {
     dev: number;
@@ -230,6 +245,7 @@ function runtimePaths(dataRoot: string, workspaceId: string): RuntimePaths {
     sessions: join(root, "sessions"),
     artifacts: join(root, "artifacts"),
     indexes: join(root, "indexes"),
+    memory: join(root, "memory"),
   };
 }
 
@@ -388,6 +404,9 @@ function decodePublishedRuntimeConfigManifest(value: unknown): PublishedRuntimeC
     || typeof manifest.catalogBlob !== "string"
     || typeof manifest.configDigest !== "string"
     || typeof manifest.bindingDigest !== "string"
+    || typeof manifest.effectiveConfigIdentity !== "string"
+    || typeof manifest.configFingerprint !== "string"
+    || typeof manifest.inputFingerprint !== "string"
     || typeof manifest.path !== "string"
     || !file
     || typeof file.dev !== "number"
@@ -413,6 +432,13 @@ export async function publishDeterministicRuntimeConfigLease(options: {
 }): Promise<DeterministicRuntimeConfigLease> {
   const rendered = await renderActiveWorkspaceRuntime(options);
   const publishedConfig = applyCollectionLifecycle(rendered.renderedConfig, "require_existing");
+  const renderedConfigObject = parse(rendered.renderedConfig) as Record<string, unknown>;
+  const effectiveConfig = buildCanonicalEffectiveConfig(renderedConfigObject);
+  const effectiveConfigIdentityValue = effectiveConfigIdentity(rendered.workspaceId, renderedConfigObject);
+  const configFingerprintValue = configFingerprint(renderedConfigObject);
+  const inputFingerprintValue = inputFingerprint(rendered.workspaceId, renderedConfigObject);
+  const identitySuffix = inputFingerprintValue.slice(7, 23);
+
   const preprocessingRoot = ensureTrustedDirectory(join(
     options.dataRoot,
     "sessions",
@@ -421,8 +447,8 @@ export async function publishDeterministicRuntimeConfigLease(options: {
   ));
   const configDirectory = ensureTrustedDirectory(join(preprocessingRoot, "runtime-config"));
   const manifestDirectory = ensureTrustedDirectory(join(preprocessingRoot, "runtime-config-manifests"));
-  const path = join(configDirectory, `${rendered.workspaceRevision}.yaml`);
-  const manifestPath = join(manifestDirectory, `${rendered.workspaceRevision}.json`);
+  const path = join(configDirectory, `${rendered.workspaceRevision}-${identitySuffix}.yaml`);
+  const manifestPath = join(manifestDirectory, `${rendered.workspaceRevision}-${identitySuffix}.json`);
   const configDigest = sha256(publishedConfig);
 
   const verifyPublished = (): PublishedRuntimeConfigManifest | undefined => {
@@ -456,6 +482,9 @@ export async function publishDeterministicRuntimeConfigLease(options: {
       || manifest.catalogBlob !== rendered.catalogBlob
       || manifest.configDigest !== configDigest
       || manifest.bindingDigest !== rendered.bindingDigest
+      || manifest.effectiveConfigIdentity !== effectiveConfigIdentityValue
+      || manifest.configFingerprint !== configFingerprintValue
+      || manifest.inputFingerprint !== inputFingerprintValue
       || manifest.path !== path
       || manifest.file.dev !== configStat!.dev
       || manifest.file.ino !== configStat!.ino
@@ -479,6 +508,10 @@ export async function publishDeterministicRuntimeConfigLease(options: {
       catalogBlob: rendered.catalogBlob,
       configDigest,
       bindingDigest: rendered.bindingDigest,
+      effectiveConfig,
+      effectiveConfigIdentity: effectiveConfigIdentityValue,
+      configFingerprint: configFingerprintValue,
+      inputFingerprint: inputFingerprintValue,
       release: () => undefined,
     };
   }
@@ -505,6 +538,9 @@ export async function publishDeterministicRuntimeConfigLease(options: {
     catalogBlob: rendered.catalogBlob,
     configDigest,
     bindingDigest: rendered.bindingDigest,
+    effectiveConfigIdentity: effectiveConfigIdentityValue,
+    configFingerprint: configFingerprintValue,
+    inputFingerprint: inputFingerprintValue,
     path,
     file: {
       dev: Number(publishedStat.dev),
@@ -518,7 +554,8 @@ export async function publishDeterministicRuntimeConfigLease(options: {
     readStrictJson(manifestPath);
   } catch (error) {
     if ((error as NodeJS.ErrnoException).code === "ENOENT") {
-      writeAtomicFile(manifestPath, `${JSON.stringify(manifest)}\n`, 0o600);
+      writeAtomicFile(manifestPath, `${JSON.stringify(manifest)}
+`, 0o600);
     } else {
       throw error;
     }
@@ -533,6 +570,10 @@ export async function publishDeterministicRuntimeConfigLease(options: {
     catalogBlob: rendered.catalogBlob,
     configDigest,
     bindingDigest: rendered.bindingDigest,
+    effectiveConfig,
+    effectiveConfigIdentity: effectiveConfigIdentityValue,
+    configFingerprint: configFingerprintValue,
+    inputFingerprint: inputFingerprintValue,
     release: () => undefined,
   };
-}
+}

backend/src/workspaces/runtime-renderer.ts

@@ -9,6 +9,7 @@ export interface RuntimePaths {
   sessions: string;
   artifacts: string;
   indexes: string;
+  memory: string;
 }
 
 export interface RuntimeIdentity {
@@ -255,6 +256,7 @@ export function renderRuntimeConfig(
     ...(installation.profile === undefined ? {} : { profile: installation.profile }),
     language: descriptor.workspace.language,
     database,
+    semantic_index: descriptor.semantic_index,
     resources: {
       vector: {
         engine: "qdrant",