feat: P3 effective configuration, memory root, and revision-scoped records
This commit is contained in:
@@ -180,6 +180,7 @@ export class ThtRunner {
|
||||
sessions: join(root, "sessions"),
|
||||
artifacts: join(root, "artifacts"),
|
||||
indexes: join(root, "indexes"),
|
||||
memory: join(root, "memory"),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ function failureResult(
|
||||
const STATE_ERROR_CODES: Record<string, WorkspaceOperationResult["code"]> = {
|
||||
preprocessing_resume_mismatch: "preprocessing_resume_mismatch",
|
||||
preprocessing_conflict: "preprocessing_conflict",
|
||||
effective_config_mismatch: "effective_config_mismatch",
|
||||
};
|
||||
|
||||
function boundedJson(result: WorkspaceOperationResult): string {
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { normalize } from "node:path";
|
||||
|
||||
export interface CanonicalEffectiveConfig {
|
||||
schemaVersion: 1;
|
||||
dwh: CanonicalDwhConfig;
|
||||
vector: CanonicalVectorConfig;
|
||||
embedding: CanonicalEmbeddingConfig;
|
||||
roots: CanonicalRootsConfig;
|
||||
}
|
||||
|
||||
export interface CanonicalDwhConfig {
|
||||
engine: "postgres";
|
||||
database: string;
|
||||
schema: string;
|
||||
transport: "postgres_direct" | "rest_api" | "ssh_tunnel";
|
||||
host?: string;
|
||||
port?: number;
|
||||
baseUrl?: string;
|
||||
user?: string;
|
||||
}
|
||||
|
||||
export interface CanonicalVectorConfig {
|
||||
collection: string;
|
||||
dimensions: number;
|
||||
distance: string;
|
||||
}
|
||||
|
||||
export interface CanonicalEmbeddingConfig {
|
||||
model: string;
|
||||
dimensions: number;
|
||||
}
|
||||
|
||||
export interface CanonicalRootsConfig {
|
||||
artifacts: string;
|
||||
indexes: string;
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | undefined {
|
||||
if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function requireString(value: Record<string, unknown>, key: string): string {
|
||||
const candidate = value[key];
|
||||
if (typeof candidate !== "string" || candidate.length === 0) {
|
||||
throw new TypeError(`effective config is missing ${key}`);
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
|
||||
function optionalString(value: Record<string, unknown>, key: string): string | undefined {
|
||||
const candidate = value[key];
|
||||
if (candidate === undefined || candidate === null) return undefined;
|
||||
if (typeof candidate !== "string") return undefined;
|
||||
return candidate;
|
||||
}
|
||||
|
||||
function optionalNumber(value: Record<string, unknown>, key: string): number | undefined {
|
||||
const candidate = value[key];
|
||||
if (candidate === undefined || candidate === null) return undefined;
|
||||
if (typeof candidate !== "number" || Number.isNaN(candidate)) return undefined;
|
||||
return candidate;
|
||||
}
|
||||
|
||||
function requireNumber(value: Record<string, unknown>, key: string): number {
|
||||
const candidate = value[key];
|
||||
if (typeof candidate !== "number" || Number.isNaN(candidate)) {
|
||||
throw new TypeError(`effective config is missing numeric ${key}`);
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
|
||||
function normalizeRoot(value: string): string {
|
||||
return normalize(value);
|
||||
}
|
||||
|
||||
function dwhTransport(rendered: Record<string, unknown>): CanonicalDwhConfig["transport"] {
|
||||
const dwh = asRecord(rendered.dwh);
|
||||
const type = dwh ? requireString(dwh, "type") : undefined;
|
||||
if (type === "postgres_direct") return "postgres_direct";
|
||||
if (type === "thoth_rest") return "rest_api";
|
||||
if (type === "ssh_tunnel") return "ssh_tunnel";
|
||||
throw new TypeError(`effective config has unsupported dwh transport ${type}`);
|
||||
}
|
||||
|
||||
function buildDwhConfig(rendered: Record<string, unknown>): CanonicalDwhConfig {
|
||||
const transport = dwhTransport(rendered);
|
||||
const databaseRecord = asRecord(rendered.database) ?? asRecord(asRecord(asRecord(rendered.dwh)?.connection)?.database);
|
||||
if (!databaseRecord) {
|
||||
throw new TypeError("effective config is missing database identity");
|
||||
}
|
||||
const engine = "postgres";
|
||||
const database = requireString(databaseRecord, "database");
|
||||
const schema = requireString(databaseRecord, "schema");
|
||||
const dwh: Record<string, unknown> = { engine, database, schema, transport };
|
||||
|
||||
if (transport === "postgres_direct") {
|
||||
const host = optionalString(databaseRecord, "host");
|
||||
const port = optionalNumber(databaseRecord, "port");
|
||||
const user = optionalString(databaseRecord, "user");
|
||||
if (host !== undefined) dwh.host = host;
|
||||
if (port !== undefined) dwh.port = port;
|
||||
if (user !== undefined) dwh.user = user;
|
||||
} else if (transport === "rest_api") {
|
||||
const rest = asRecord(rendered.rest) ?? asRecord(asRecord(asRecord(rendered.dwh)?.endpoint));
|
||||
const baseUrl = rest ? optionalString(rest, "base_url") : undefined;
|
||||
if (baseUrl !== undefined) dwh.baseUrl = baseUrl;
|
||||
}
|
||||
|
||||
return dwh as unknown as CanonicalDwhConfig;
|
||||
}
|
||||
|
||||
function buildVectorConfig(rendered: Record<string, unknown>): CanonicalVectorConfig {
|
||||
const resources = asRecord(rendered.resources);
|
||||
const vector = resources ? asRecord(resources.vector) : undefined;
|
||||
if (!vector) {
|
||||
throw new TypeError("effective config is missing vector resources");
|
||||
}
|
||||
const collection = requireString(vector, "collection");
|
||||
const semanticIndex = asRecord(rendered.semantic_index);
|
||||
const vectorStore = semanticIndex ? asRecord(semanticIndex.vector_store) : undefined;
|
||||
const dimensions = vectorStore
|
||||
? requireNumber(vectorStore, "dimensions")
|
||||
: requireNumber(vector, "dimensions");
|
||||
const distance = vectorStore
|
||||
? requireString(vectorStore, "distance")
|
||||
: (optionalString(vector, "distance") ?? "cosine");
|
||||
return { collection, dimensions, distance };
|
||||
}
|
||||
|
||||
function buildEmbeddingConfig(rendered: Record<string, unknown>): CanonicalEmbeddingConfig {
|
||||
const resources = asRecord(rendered.resources);
|
||||
const embeddings = resources ? asRecord(resources.embeddings) : undefined;
|
||||
if (!embeddings) {
|
||||
throw new TypeError("effective config is missing embedding resources");
|
||||
}
|
||||
return {
|
||||
model: requireString(embeddings, "model"),
|
||||
dimensions: requireNumber(embeddings, "dimensions"),
|
||||
};
|
||||
}
|
||||
|
||||
function buildRootsConfig(rendered: Record<string, unknown>): CanonicalRootsConfig {
|
||||
const roots = asRecord(rendered.roots) ?? asRecord(rendered.paths);
|
||||
if (!roots) {
|
||||
throw new TypeError("effective config is missing roots");
|
||||
}
|
||||
return {
|
||||
artifacts: normalizeRoot(requireString(roots, "artifacts")),
|
||||
indexes: normalizeRoot(requireString(roots, "indexes")),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the versioned, non-secret effective DWH/preprocessing configuration from a
|
||||
* rendered runtime configuration object. The result contains only the fields that
|
||||
* affect DWH generation identity; credentials, runtime identity, session storage,
|
||||
* evidence, and service endpoints are excluded.
|
||||
*/
|
||||
export function buildCanonicalEffectiveConfig(renderedConfig: unknown): CanonicalEffectiveConfig {
|
||||
const rendered = asRecord(renderedConfig);
|
||||
if (!rendered) {
|
||||
throw new TypeError("effective config requires a rendered configuration object");
|
||||
}
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
dwh: buildDwhConfig(rendered),
|
||||
vector: buildVectorConfig(rendered),
|
||||
embedding: buildEmbeddingConfig(rendered),
|
||||
roots: buildRootsConfig(rendered),
|
||||
};
|
||||
}
|
||||
|
||||
function sha256(value: string | Buffer): string {
|
||||
return `sha256:${createHash("sha256").update(value).digest("hex")}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize the canonical effective config to a deterministic JSON string with the
|
||||
* fixed key order defined by the shared contract. No whitespace is included.
|
||||
*/
|
||||
export function canonicalEffectiveConfigJson(config: CanonicalEffectiveConfig): string {
|
||||
const ordered: Record<string, unknown> = { schemaVersion: config.schemaVersion };
|
||||
ordered.dwh = { ...config.dwh };
|
||||
ordered.vector = { ...config.vector };
|
||||
ordered.embedding = { ...config.embedding };
|
||||
ordered.roots = { ...config.roots };
|
||||
return JSON.stringify(ordered);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the stable logical config-source identity for a workspace revision.
|
||||
* This is `workspace://<workspaceId>@v1:<sha256-hex-of-canonical-json>`.
|
||||
*/
|
||||
export function effectiveConfigIdentity(workspaceId: string, renderedConfig: unknown): string {
|
||||
const canonical = buildCanonicalEffectiveConfig(renderedConfig);
|
||||
const digest = createHash("sha256").update(canonicalEffectiveConfigJson(canonical)).digest("hex");
|
||||
return `workspace://${workspaceId}@v1:${digest}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the config fingerprint: `sha256:` + the SHA-256 of the canonical effective
|
||||
* config JSON bytes.
|
||||
*/
|
||||
export function configFingerprint(renderedConfig: unknown): string {
|
||||
const canonical = buildCanonicalEffectiveConfig(renderedConfig);
|
||||
return sha256(canonicalEffectiveConfigJson(canonical));
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the input fingerprint: `sha256:` + the SHA-256 of the logical config-source
|
||||
* identity string.
|
||||
*/
|
||||
export function inputFingerprint(workspaceId: string, renderedConfig: unknown): string {
|
||||
return sha256(effectiveConfigIdentity(workspaceId, renderedConfig));
|
||||
}
|
||||
@@ -30,6 +30,9 @@ export interface WorkspaceOperationResult {
|
||||
completedStages: string[];
|
||||
counts?: Record<string, number>;
|
||||
artifactIdentities?: Array<{ kind: string; digest: string }>;
|
||||
effectiveConfigIdentity?: string;
|
||||
configFingerprint?: string;
|
||||
inputFingerprint?: string;
|
||||
/** Suggested FK annotations YAML for the operator to write to --output (schema suggest-fks). */
|
||||
suggestedFksYaml?: string;
|
||||
warnings?: string[];
|
||||
@@ -92,6 +95,9 @@ function baseResult(
|
||||
descriptorBlob: runtime.descriptorBlob,
|
||||
operation,
|
||||
completedStages: [],
|
||||
effectiveConfigIdentity: runtime.configLease.effectiveConfigIdentity,
|
||||
configFingerprint: runtime.configLease.configFingerprint,
|
||||
inputFingerprint: runtime.configLease.inputFingerprint,
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -18,6 +18,14 @@ import {
|
||||
import { dirname, isAbsolute, join, relative, resolve } from "node:path";
|
||||
import { readFile as readFileAsync } from "node:fs/promises";
|
||||
import { parse, parseAllDocuments, stringify } from "yaml";
|
||||
import {
|
||||
buildCanonicalEffectiveConfig,
|
||||
canonicalEffectiveConfigJson,
|
||||
configFingerprint,
|
||||
effectiveConfigIdentity,
|
||||
inputFingerprint,
|
||||
type CanonicalEffectiveConfig,
|
||||
} from "./effective-config.js";
|
||||
import { resolveRuntimeBindings, type RuntimeBindings } from "./bindings.js";
|
||||
import { GitWorkspaceRepository } from "./git-repository.js";
|
||||
import { WorkspaceRegistry } from "./registry.js";
|
||||
@@ -63,6 +71,10 @@ export interface DeterministicRuntimeConfigLease extends RuntimeConfigLease {
|
||||
catalogBlob: string;
|
||||
configDigest: string;
|
||||
bindingDigest: string;
|
||||
effectiveConfig: CanonicalEffectiveConfig;
|
||||
effectiveConfigIdentity: string;
|
||||
configFingerprint: string;
|
||||
inputFingerprint: string;
|
||||
}
|
||||
|
||||
export class RuntimeConfigLeaseError extends Error {
|
||||
@@ -84,6 +96,9 @@ interface PublishedRuntimeConfigManifest {
|
||||
catalogBlob: string;
|
||||
configDigest: string;
|
||||
bindingDigest: string;
|
||||
effectiveConfigIdentity: string;
|
||||
configFingerprint: string;
|
||||
inputFingerprint: string;
|
||||
path: string;
|
||||
file: {
|
||||
dev: number;
|
||||
@@ -230,6 +245,7 @@ function runtimePaths(dataRoot: string, workspaceId: string): RuntimePaths {
|
||||
sessions: join(root, "sessions"),
|
||||
artifacts: join(root, "artifacts"),
|
||||
indexes: join(root, "indexes"),
|
||||
memory: join(root, "memory"),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -388,6 +404,9 @@ function decodePublishedRuntimeConfigManifest(value: unknown): PublishedRuntimeC
|
||||
|| typeof manifest.catalogBlob !== "string"
|
||||
|| typeof manifest.configDigest !== "string"
|
||||
|| typeof manifest.bindingDigest !== "string"
|
||||
|| typeof manifest.effectiveConfigIdentity !== "string"
|
||||
|| typeof manifest.configFingerprint !== "string"
|
||||
|| typeof manifest.inputFingerprint !== "string"
|
||||
|| typeof manifest.path !== "string"
|
||||
|| !file
|
||||
|| typeof file.dev !== "number"
|
||||
@@ -413,6 +432,13 @@ export async function publishDeterministicRuntimeConfigLease(options: {
|
||||
}): Promise<DeterministicRuntimeConfigLease> {
|
||||
const rendered = await renderActiveWorkspaceRuntime(options);
|
||||
const publishedConfig = applyCollectionLifecycle(rendered.renderedConfig, "require_existing");
|
||||
const renderedConfigObject = parse(rendered.renderedConfig) as Record<string, unknown>;
|
||||
const effectiveConfig = buildCanonicalEffectiveConfig(renderedConfigObject);
|
||||
const effectiveConfigIdentityValue = effectiveConfigIdentity(rendered.workspaceId, renderedConfigObject);
|
||||
const configFingerprintValue = configFingerprint(renderedConfigObject);
|
||||
const inputFingerprintValue = inputFingerprint(rendered.workspaceId, renderedConfigObject);
|
||||
const identitySuffix = inputFingerprintValue.slice(7, 23);
|
||||
|
||||
const preprocessingRoot = ensureTrustedDirectory(join(
|
||||
options.dataRoot,
|
||||
"sessions",
|
||||
@@ -421,8 +447,8 @@ export async function publishDeterministicRuntimeConfigLease(options: {
|
||||
));
|
||||
const configDirectory = ensureTrustedDirectory(join(preprocessingRoot, "runtime-config"));
|
||||
const manifestDirectory = ensureTrustedDirectory(join(preprocessingRoot, "runtime-config-manifests"));
|
||||
const path = join(configDirectory, `${rendered.workspaceRevision}.yaml`);
|
||||
const manifestPath = join(manifestDirectory, `${rendered.workspaceRevision}.json`);
|
||||
const path = join(configDirectory, `${rendered.workspaceRevision}-${identitySuffix}.yaml`);
|
||||
const manifestPath = join(manifestDirectory, `${rendered.workspaceRevision}-${identitySuffix}.json`);
|
||||
const configDigest = sha256(publishedConfig);
|
||||
|
||||
const verifyPublished = (): PublishedRuntimeConfigManifest | undefined => {
|
||||
@@ -456,6 +482,9 @@ export async function publishDeterministicRuntimeConfigLease(options: {
|
||||
|| manifest.catalogBlob !== rendered.catalogBlob
|
||||
|| manifest.configDigest !== configDigest
|
||||
|| manifest.bindingDigest !== rendered.bindingDigest
|
||||
|| manifest.effectiveConfigIdentity !== effectiveConfigIdentityValue
|
||||
|| manifest.configFingerprint !== configFingerprintValue
|
||||
|| manifest.inputFingerprint !== inputFingerprintValue
|
||||
|| manifest.path !== path
|
||||
|| manifest.file.dev !== configStat!.dev
|
||||
|| manifest.file.ino !== configStat!.ino
|
||||
@@ -479,6 +508,10 @@ export async function publishDeterministicRuntimeConfigLease(options: {
|
||||
catalogBlob: rendered.catalogBlob,
|
||||
configDigest,
|
||||
bindingDigest: rendered.bindingDigest,
|
||||
effectiveConfig,
|
||||
effectiveConfigIdentity: effectiveConfigIdentityValue,
|
||||
configFingerprint: configFingerprintValue,
|
||||
inputFingerprint: inputFingerprintValue,
|
||||
release: () => undefined,
|
||||
};
|
||||
}
|
||||
@@ -505,6 +538,9 @@ export async function publishDeterministicRuntimeConfigLease(options: {
|
||||
catalogBlob: rendered.catalogBlob,
|
||||
configDigest,
|
||||
bindingDigest: rendered.bindingDigest,
|
||||
effectiveConfigIdentity: effectiveConfigIdentityValue,
|
||||
configFingerprint: configFingerprintValue,
|
||||
inputFingerprint: inputFingerprintValue,
|
||||
path,
|
||||
file: {
|
||||
dev: Number(publishedStat.dev),
|
||||
@@ -518,7 +554,8 @@ export async function publishDeterministicRuntimeConfigLease(options: {
|
||||
readStrictJson(manifestPath);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
writeAtomicFile(manifestPath, `${JSON.stringify(manifest)}\n`, 0o600);
|
||||
writeAtomicFile(manifestPath, `${JSON.stringify(manifest)}
|
||||
`, 0o600);
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
@@ -533,6 +570,10 @@ export async function publishDeterministicRuntimeConfigLease(options: {
|
||||
catalogBlob: rendered.catalogBlob,
|
||||
configDigest,
|
||||
bindingDigest: rendered.bindingDigest,
|
||||
effectiveConfig,
|
||||
effectiveConfigIdentity: effectiveConfigIdentityValue,
|
||||
configFingerprint: configFingerprintValue,
|
||||
inputFingerprint: inputFingerprintValue,
|
||||
release: () => undefined,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ export interface RuntimePaths {
|
||||
sessions: string;
|
||||
artifacts: string;
|
||||
indexes: string;
|
||||
memory: string;
|
||||
}
|
||||
|
||||
export interface RuntimeIdentity {
|
||||
@@ -255,6 +256,7 @@ export function renderRuntimeConfig(
|
||||
...(installation.profile === undefined ? {} : { profile: installation.profile }),
|
||||
language: descriptor.workspace.language,
|
||||
database,
|
||||
semantic_index: descriptor.semantic_index,
|
||||
resources: {
|
||||
vector: {
|
||||
engine: "qdrant",
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { expect, test } from "vitest";
|
||||
import {
|
||||
buildCanonicalEffectiveConfig,
|
||||
canonicalEffectiveConfigJson,
|
||||
configFingerprint,
|
||||
effectiveConfigIdentity,
|
||||
inputFingerprint,
|
||||
} from "../src/workspaces/effective-config.js";
|
||||
|
||||
const semanticRuntime = {
|
||||
internalQdrantUrl: "http://qdrant:6333",
|
||||
internalEmbeddingUrl: "http://embedding:11434",
|
||||
internalEmbeddingModel: "qwen3-embedding:0.6b",
|
||||
internalEmbeddingDimensions: 1024,
|
||||
};
|
||||
|
||||
function directRendered(): Record<string, unknown> {
|
||||
return {
|
||||
runtime_identity: {
|
||||
workspace_id: "psd-clinical",
|
||||
workspace_revision: "a".repeat(40),
|
||||
source_identity: "workspace://psd-clinical",
|
||||
},
|
||||
session_storage: { mode: "local" },
|
||||
profile: "server",
|
||||
language: "it",
|
||||
database: {
|
||||
host: "dwh.internal",
|
||||
port: 5432,
|
||||
database: "postgres",
|
||||
schema: "datawarehouse",
|
||||
user: "thoth_reader",
|
||||
password_file: "/run/secrets/dwh-password",
|
||||
ssl_ca_file: "/run/secrets/dwh-ca.pem",
|
||||
transport: "direct",
|
||||
},
|
||||
dwh: { type: "postgres_direct" },
|
||||
resources: {
|
||||
vector: {
|
||||
engine: "qdrant",
|
||||
base_url: "http://qdrant:6333",
|
||||
collection: "psd-clinical",
|
||||
dimensions: 1024,
|
||||
distance: "cosine",
|
||||
collection_lifecycle: "require_existing",
|
||||
},
|
||||
embeddings: {
|
||||
provider: "ollama_internal",
|
||||
base_url: "http://embedding:11434",
|
||||
model: "qwen3-embedding:0.6b",
|
||||
dimensions: 1024,
|
||||
},
|
||||
},
|
||||
roots: {
|
||||
sessions: "/data/sessions/psd-clinical/sessions",
|
||||
artifacts: "/data/sessions/psd-clinical/artifacts",
|
||||
indexes: "/data/sessions/psd-clinical/indexes",
|
||||
},
|
||||
paths: {
|
||||
sessions: "/data/sessions/psd-clinical/sessions",
|
||||
artifacts: "/data/sessions/psd-clinical/artifacts",
|
||||
indexes: "/data/sessions/psd-clinical/indexes",
|
||||
memory: "/data/sessions/psd-clinical/memory",
|
||||
},
|
||||
evidence: {
|
||||
sources: [{
|
||||
type: "filesystem",
|
||||
root: "/srv/registry/snapshots/rev/psd-clinical/evidence",
|
||||
patterns: ["**/*.md"],
|
||||
max_bytes: 10_485_760,
|
||||
}],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function restRendered(): Record<string, unknown> {
|
||||
return {
|
||||
...directRendered(),
|
||||
database: {
|
||||
host: "localhost",
|
||||
port: 5432,
|
||||
database: "postgres",
|
||||
schema: "datawarehouse",
|
||||
user: "rest",
|
||||
password: "",
|
||||
transport: "rest",
|
||||
},
|
||||
dwh: {
|
||||
type: "thoth_rest",
|
||||
database: { database: "postgres", schema: "datawarehouse" },
|
||||
endpoint: {
|
||||
base_url: "https://dwh.example.test",
|
||||
api_key_file: "/run/secrets/dwh-api-key",
|
||||
},
|
||||
},
|
||||
rest: {
|
||||
base_url: "https://dwh.example.test",
|
||||
api_key_file: "/run/secrets/dwh-api-key",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const directCanonical =
|
||||
`{"schemaVersion":1,"dwh":{` +
|
||||
`"engine":"postgres","database":"postgres","schema":"datawarehouse",` +
|
||||
`"transport":"postgres_direct","host":"dwh.internal","port":5432,"user":"thoth_reader"},` +
|
||||
`"vector":{"collection":"psd-clinical","dimensions":1024,"distance":"cosine"},` +
|
||||
`"embedding":{"model":"qwen3-embedding:0.6b","dimensions":1024},` +
|
||||
`"roots":{"artifacts":"/data/sessions/psd-clinical/artifacts",` +
|
||||
`"indexes":"/data/sessions/psd-clinical/indexes"}}`;
|
||||
|
||||
test("canonical effective config is deterministic and contains the expected key order", () => {
|
||||
const rendered = directRendered();
|
||||
const canonical = buildCanonicalEffectiveConfig(rendered);
|
||||
expect(canonicalEffectiveConfigJson(canonical)).toBe(directCanonical);
|
||||
expect(buildCanonicalEffectiveConfig(rendered)).toEqual(canonical);
|
||||
});
|
||||
|
||||
test("canonical effective config excludes secrets, evidence, session storage, and runtime identity", () => {
|
||||
const json = canonicalEffectiveConfigJson(buildCanonicalEffectiveConfig(directRendered()));
|
||||
expect(json).not.toContain("password_file");
|
||||
expect(json).not.toContain("ssl_ca_file");
|
||||
expect(json).not.toContain("session_storage");
|
||||
expect(json).not.toContain("runtime_identity");
|
||||
expect(json).not.toContain("evidence");
|
||||
expect(json).not.toContain("sources");
|
||||
expect(json).not.toContain("collection_lifecycle");
|
||||
expect(json).not.toContain("base_url"); // vector/embedding service URLs are not identity
|
||||
expect(json).not.toContain("memory");
|
||||
expect(json).not.toContain('"sessions"');
|
||||
});
|
||||
|
||||
test("REST transport canonicalizes to transport rest_api with baseUrl and no host/port", () => {
|
||||
const canonical = buildCanonicalEffectiveConfig(restRendered());
|
||||
const json = canonicalEffectiveConfigJson(canonical);
|
||||
expect(json).toContain(`"transport":"rest_api"`);
|
||||
expect(json).toContain(`"baseUrl":"https://dwh.example.test"`);
|
||||
expect(json).not.toContain(`"host":`);
|
||||
expect(json).not.toContain(`"port":`);
|
||||
expect(json).not.toContain("api_key_file");
|
||||
});
|
||||
|
||||
test("identity and fingerprint helpers produce stable prefixed hex values", () => {
|
||||
const rendered = directRendered();
|
||||
const identity = effectiveConfigIdentity("psd-clinical", rendered);
|
||||
const cfg = configFingerprint(rendered);
|
||||
const input = inputFingerprint("psd-clinical", rendered);
|
||||
|
||||
expect(identity).toMatch(/^workspace:\/\/psd-clinical@v1:[0-9a-f]{64}$/);
|
||||
expect(cfg).toBe("sha256:" + createHash("sha256").update(directCanonical).digest("hex"));
|
||||
expect(input).toBe("sha256:" + createHash("sha256").update(identity).digest("hex"));
|
||||
expect(input).not.toBe(cfg);
|
||||
});
|
||||
|
||||
test("content-only or session_storage changes keep the same effective config identity", () => {
|
||||
const base = directRendered();
|
||||
const identityBefore = effectiveConfigIdentity("psd-clinical", base);
|
||||
const fingerprintBefore = configFingerprint(base);
|
||||
|
||||
const contentOnly = {
|
||||
...base,
|
||||
runtime_identity: {
|
||||
...base.runtime_identity,
|
||||
workspace_revision: "b".repeat(40),
|
||||
},
|
||||
session_storage: { mode: "remote", url: "http://example.test" },
|
||||
evidence: {
|
||||
sources: [{
|
||||
type: "filesystem",
|
||||
root: "/srv/registry/snapshots/other/psd-clinical/evidence",
|
||||
patterns: ["**/*.txt"],
|
||||
max_bytes: 999,
|
||||
}],
|
||||
},
|
||||
};
|
||||
|
||||
expect(effectiveConfigIdentity("psd-clinical", contentOnly)).toBe(identityBefore);
|
||||
expect(configFingerprint(contentOnly)).toBe(fingerprintBefore);
|
||||
});
|
||||
|
||||
test("DWH-affecting changes alter the effective config identity", () => {
|
||||
const base = directRendered();
|
||||
const identityBefore = effectiveConfigIdentity("psd-clinical", base);
|
||||
|
||||
const changedHost = { ...base, database: { ...(base.database as object), host: "dwh-two.internal" } };
|
||||
expect(effectiveConfigIdentity("psd-clinical", changedHost)).not.toBe(identityBefore);
|
||||
|
||||
const changedDatabase = { ...base, database: { ...(base.database as object), database: "analytics" } };
|
||||
expect(effectiveConfigIdentity("psd-clinical", changedDatabase)).not.toBe(identityBefore);
|
||||
|
||||
const changedCollection = { ...base, resources: { ...base.resources, vector: { ...(base.resources as Record<string, any>).vector, collection: "other" } } };
|
||||
expect(effectiveConfigIdentity("psd-clinical", changedCollection)).not.toBe(identityBefore);
|
||||
|
||||
const changedTransport = restRendered();
|
||||
expect(effectiveConfigIdentity("psd-clinical", changedTransport)).not.toBe(identityBefore);
|
||||
});
|
||||
@@ -115,13 +115,31 @@ function runtime(workspace = baseWorkspace, workspaceId = workspace.workspace.id
|
||||
descriptorBlob: "b".repeat(40),
|
||||
catalogBlob: "c".repeat(40),
|
||||
configLease: {
|
||||
path: `/data/sessions/${workspaceId}/preprocessing/runtime-config/${"a".repeat(40)}.yaml`,
|
||||
path: `/data/sessions/${workspaceId}/preprocessing/runtime-config/${"a".repeat(40)}-identitysuffix.yaml`,
|
||||
workspaceId,
|
||||
workspaceRevision: "a".repeat(40),
|
||||
descriptorBlob: "b".repeat(40),
|
||||
catalogBlob: "c".repeat(40),
|
||||
configDigest: "sha256:config",
|
||||
bindingDigest: "sha256:bindings",
|
||||
effectiveConfig: {
|
||||
schemaVersion: 1,
|
||||
dwh: {
|
||||
engine: "postgres",
|
||||
database: "analytics",
|
||||
schema: "mart",
|
||||
transport: "postgres_direct",
|
||||
host: "dwh.internal",
|
||||
port: 5432,
|
||||
user: "reader",
|
||||
},
|
||||
vector: { collection: workspaceId, dimensions: 1024, distance: "cosine" },
|
||||
embedding: { model: "qwen3-embedding:0.6b", dimensions: 1024 },
|
||||
roots: { artifacts: "/data/artifacts", indexes: "/data/indexes" },
|
||||
},
|
||||
effectiveConfigIdentity: "workspace://psd-clinical@v1:" + "d".repeat(64),
|
||||
configFingerprint: "sha256:" + "e".repeat(64),
|
||||
inputFingerprint: "sha256:" + "f".repeat(64),
|
||||
release: () => undefined,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -14,7 +14,13 @@ import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import { parse } from "yaml";
|
||||
import { WorkspaceRegistry } from "../src/workspaces/registry.js";
|
||||
import {
|
||||
buildCanonicalEffectiveConfig,
|
||||
canonicalEffectiveConfigJson,
|
||||
effectiveConfigIdentity,
|
||||
} from "../src/workspaces/effective-config.js";
|
||||
import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js";
|
||||
import {
|
||||
publishDeterministicRuntimeConfigLease,
|
||||
@@ -123,6 +129,7 @@ evidence:
|
||||
return {
|
||||
dataRoot,
|
||||
harnessDir,
|
||||
source,
|
||||
registry,
|
||||
registryConfig,
|
||||
revision,
|
||||
@@ -163,7 +170,7 @@ test("active workspace rendering is byte-identical to direct snapshot rendering"
|
||||
expect(active.catalogBlob).toMatch(/^sha256:[0-9a-f]{64}$/);
|
||||
});
|
||||
|
||||
test("deterministic operator leases publish one revision-bound protected config and refuse changed same-revision bytes", async () => {
|
||||
test("deterministic operator leases are keyed by logical identity and stable across calls", async () => {
|
||||
const f = await fixture();
|
||||
const first = await publishDeterministicRuntimeConfigLease({
|
||||
workspaceId: "psd-clinical",
|
||||
@@ -186,6 +193,7 @@ test("deterministic operator leases publish one revision-bound protected config
|
||||
semanticRuntime,
|
||||
});
|
||||
|
||||
const suffix = first.inputFingerprint.slice(7, 23);
|
||||
expect(second.path).toBe(first.path);
|
||||
expect(first.path).toBe(join(
|
||||
f.dataRoot,
|
||||
@@ -193,15 +201,34 @@ test("deterministic operator leases publish one revision-bound protected config
|
||||
"psd-clinical",
|
||||
"preprocessing",
|
||||
"runtime-config",
|
||||
`${f.revision.commit}.yaml`,
|
||||
`${f.revision.commit}-${suffix}.yaml`,
|
||||
));
|
||||
expect(statSync(first.path).mode & 0o777).toBe(0o400);
|
||||
expect(statSync(first.manifestPath).mode & 0o777).toBe(0o600);
|
||||
expect(readFileSync(first.path, "utf8")).toContain("collection_lifecycle: require_existing");
|
||||
expect(readFileSync(first.path, "utf8")).toContain("memory:");
|
||||
expect(existsSync(first.manifestPath)).toBe(true);
|
||||
expect(first.effectiveConfigIdentity).toMatch(/^workspace:\/\/psd-clinical@v1:[0-9a-f]{64}$/);
|
||||
expect(first.configFingerprint).toMatch(/^sha256:[0-9a-f]{64}$/);
|
||||
expect(first.inputFingerprint).toMatch(/^sha256:[0-9a-f]{64}$/);
|
||||
expect(first.inputFingerprint).not.toBe(first.configFingerprint);
|
||||
const manifest = JSON.parse(readFileSync(first.manifestPath, "utf8"));
|
||||
expect(manifest).toMatchObject({
|
||||
schemaVersion: 1,
|
||||
workspaceId: "psd-clinical",
|
||||
workspaceRevision: f.revision.commit,
|
||||
descriptorBlob: first.descriptorBlob,
|
||||
catalogBlob: first.catalogBlob,
|
||||
configDigest: first.configDigest,
|
||||
bindingDigest: first.bindingDigest,
|
||||
effectiveConfigIdentity: first.effectiveConfigIdentity,
|
||||
configFingerprint: first.configFingerprint,
|
||||
inputFingerprint: first.inputFingerprint,
|
||||
path: first.path,
|
||||
});
|
||||
|
||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_HOST", "warehouse-two.internal");
|
||||
await expect(publishDeterministicRuntimeConfigLease({
|
||||
const changed = await publishDeterministicRuntimeConfigLease({
|
||||
workspaceId: "psd-clinical",
|
||||
registry: f.registry,
|
||||
registryConfig: f.registryConfig,
|
||||
@@ -210,7 +237,11 @@ test("deterministic operator leases publish one revision-bound protected config
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
})).rejects.toMatchObject({ code: "effective_config_mismatch" });
|
||||
});
|
||||
expect(changed.path).not.toBe(first.path);
|
||||
expect(changed.inputFingerprint).not.toBe(first.inputFingerprint);
|
||||
expect(changed.configFingerprint).not.toBe(first.configFingerprint);
|
||||
expect(readFileSync(changed.path, "utf8")).toContain("warehouse-two.internal");
|
||||
});
|
||||
|
||||
test("runtime rendering rejects untrusted snapshot paths and symlinks", async () => {
|
||||
@@ -237,3 +268,74 @@ test("runtime rendering rejects untrusted snapshot paths and symlinks", async ()
|
||||
semanticRuntime,
|
||||
})).toThrow(/trusted runtime snapshot/i);
|
||||
});
|
||||
|
||||
|
||||
test("operator lease and session snapshot produce byte-identical effective DWH bindings", async () => {
|
||||
const f = await fixture();
|
||||
const session = renderWorkspaceRuntimeFromSnapshotPath({
|
||||
snapshotPath: f.revision.snapshotPath,
|
||||
harnessDir: f.harnessDir,
|
||||
configPath: "config/tht.yaml",
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
});
|
||||
const lease = await publishDeterministicRuntimeConfigLease({
|
||||
workspaceId: "psd-clinical",
|
||||
registry: f.registry,
|
||||
registryConfig: f.registryConfig,
|
||||
harnessDir: f.harnessDir,
|
||||
configPath: "config/tht.yaml",
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
});
|
||||
|
||||
const sessionCanonical = canonicalEffectiveConfigJson(buildCanonicalEffectiveConfig(parse(session.renderedConfig)));
|
||||
const operatorCanonical = canonicalEffectiveConfigJson(lease.effectiveConfig);
|
||||
expect(operatorCanonical).toBe(sessionCanonical);
|
||||
expect(lease.effectiveConfigIdentity).toBe(
|
||||
effectiveConfigIdentity("psd-clinical", parse(session.renderedConfig)),
|
||||
);
|
||||
});
|
||||
|
||||
test("a content-only Evidence commit keeps the same effective config identity with a new revision lease", async () => {
|
||||
const f = await fixture();
|
||||
const firstLease = await publishDeterministicRuntimeConfigLease({
|
||||
workspaceId: "psd-clinical",
|
||||
registry: f.registry,
|
||||
registryConfig: f.registryConfig,
|
||||
harnessDir: f.harnessDir,
|
||||
configPath: "config/tht.yaml",
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
});
|
||||
const firstIdentity = firstLease.effectiveConfigIdentity;
|
||||
|
||||
writeFileSync(
|
||||
join(f.source, "psd-clinical", "evidence", "guide.md"),
|
||||
"# updated content only\n",
|
||||
);
|
||||
await git(f.source, ["add", "psd-clinical/evidence/guide.md"]);
|
||||
await git(f.source, ["commit", "-m", "Evidence content only"]);
|
||||
await git(f.source, ["push", "origin", "main"]);
|
||||
await f.registry.pull();
|
||||
const current = (await f.registry.list())[0];
|
||||
|
||||
const secondLease = await publishDeterministicRuntimeConfigLease({
|
||||
workspaceId: "psd-clinical",
|
||||
registry: f.registry,
|
||||
registryConfig: f.registryConfig,
|
||||
harnessDir: f.harnessDir,
|
||||
configPath: "config/tht.yaml",
|
||||
dataRoot: f.dataRoot,
|
||||
secretRoots: f.registryConfig.secretRoots,
|
||||
semanticRuntime,
|
||||
});
|
||||
|
||||
expect(secondLease.workspaceRevision).toBe(current.commit);
|
||||
expect(secondLease.workspaceRevision).not.toBe(firstLease.workspaceRevision);
|
||||
expect(secondLease.effectiveConfigIdentity).toBe(firstIdentity);
|
||||
expect(secondLease.path).not.toBe(firstLease.path);
|
||||
});
|
||||
|
||||
@@ -40,6 +40,7 @@ const paths: RuntimePaths = {
|
||||
sessions: "/data/workspaces/psd-clinical/sessions",
|
||||
artifacts: "/data/workspaces/psd-clinical/artifacts",
|
||||
indexes: "/data/workspaces/psd-clinical/indexes",
|
||||
memory: "/data/workspaces/psd-clinical/memory",
|
||||
};
|
||||
const semanticRuntime: SemanticRuntimeConfig = {
|
||||
internalQdrantUrl: "http://qdrant:6333",
|
||||
|
||||
Reference in New Issue
Block a user