feat: P3 effective configuration, memory root, and revision-scoped records
This commit is contained in:
@@ -180,6 +180,7 @@ export class ThtRunner {
|
|||||||
sessions: join(root, "sessions"),
|
sessions: join(root, "sessions"),
|
||||||
artifacts: join(root, "artifacts"),
|
artifacts: join(root, "artifacts"),
|
||||||
indexes: join(root, "indexes"),
|
indexes: join(root, "indexes"),
|
||||||
|
memory: join(root, "memory"),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ function failureResult(
|
|||||||
const STATE_ERROR_CODES: Record<string, WorkspaceOperationResult["code"]> = {
|
const STATE_ERROR_CODES: Record<string, WorkspaceOperationResult["code"]> = {
|
||||||
preprocessing_resume_mismatch: "preprocessing_resume_mismatch",
|
preprocessing_resume_mismatch: "preprocessing_resume_mismatch",
|
||||||
preprocessing_conflict: "preprocessing_conflict",
|
preprocessing_conflict: "preprocessing_conflict",
|
||||||
|
effective_config_mismatch: "effective_config_mismatch",
|
||||||
};
|
};
|
||||||
|
|
||||||
function boundedJson(result: WorkspaceOperationResult): string {
|
function boundedJson(result: WorkspaceOperationResult): string {
|
||||||
|
|||||||
@@ -0,0 +1,219 @@
|
|||||||
|
import { createHash } from "node:crypto";
|
||||||
|
import { normalize } from "node:path";
|
||||||
|
|
||||||
|
export interface CanonicalEffectiveConfig {
|
||||||
|
schemaVersion: 1;
|
||||||
|
dwh: CanonicalDwhConfig;
|
||||||
|
vector: CanonicalVectorConfig;
|
||||||
|
embedding: CanonicalEmbeddingConfig;
|
||||||
|
roots: CanonicalRootsConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CanonicalDwhConfig {
|
||||||
|
engine: "postgres";
|
||||||
|
database: string;
|
||||||
|
schema: string;
|
||||||
|
transport: "postgres_direct" | "rest_api" | "ssh_tunnel";
|
||||||
|
host?: string;
|
||||||
|
port?: number;
|
||||||
|
baseUrl?: string;
|
||||||
|
user?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CanonicalVectorConfig {
|
||||||
|
collection: string;
|
||||||
|
dimensions: number;
|
||||||
|
distance: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CanonicalEmbeddingConfig {
|
||||||
|
model: string;
|
||||||
|
dimensions: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CanonicalRootsConfig {
|
||||||
|
artifacts: string;
|
||||||
|
indexes: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function asRecord(value: unknown): Record<string, unknown> | undefined {
|
||||||
|
if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
||||||
|
return value as Record<string, unknown>;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function requireString(value: Record<string, unknown>, key: string): string {
|
||||||
|
const candidate = value[key];
|
||||||
|
if (typeof candidate !== "string" || candidate.length === 0) {
|
||||||
|
throw new TypeError(`effective config is missing ${key}`);
|
||||||
|
}
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
|
||||||
|
function optionalString(value: Record<string, unknown>, key: string): string | undefined {
|
||||||
|
const candidate = value[key];
|
||||||
|
if (candidate === undefined || candidate === null) return undefined;
|
||||||
|
if (typeof candidate !== "string") return undefined;
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
|
||||||
|
function optionalNumber(value: Record<string, unknown>, key: string): number | undefined {
|
||||||
|
const candidate = value[key];
|
||||||
|
if (candidate === undefined || candidate === null) return undefined;
|
||||||
|
if (typeof candidate !== "number" || Number.isNaN(candidate)) return undefined;
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
|
||||||
|
function requireNumber(value: Record<string, unknown>, key: string): number {
|
||||||
|
const candidate = value[key];
|
||||||
|
if (typeof candidate !== "number" || Number.isNaN(candidate)) {
|
||||||
|
throw new TypeError(`effective config is missing numeric ${key}`);
|
||||||
|
}
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeRoot(value: string): string {
|
||||||
|
return normalize(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function dwhTransport(rendered: Record<string, unknown>): CanonicalDwhConfig["transport"] {
|
||||||
|
const dwh = asRecord(rendered.dwh);
|
||||||
|
const type = dwh ? requireString(dwh, "type") : undefined;
|
||||||
|
if (type === "postgres_direct") return "postgres_direct";
|
||||||
|
if (type === "thoth_rest") return "rest_api";
|
||||||
|
if (type === "ssh_tunnel") return "ssh_tunnel";
|
||||||
|
throw new TypeError(`effective config has unsupported dwh transport ${type}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildDwhConfig(rendered: Record<string, unknown>): CanonicalDwhConfig {
|
||||||
|
const transport = dwhTransport(rendered);
|
||||||
|
const databaseRecord = asRecord(rendered.database) ?? asRecord(asRecord(asRecord(rendered.dwh)?.connection)?.database);
|
||||||
|
if (!databaseRecord) {
|
||||||
|
throw new TypeError("effective config is missing database identity");
|
||||||
|
}
|
||||||
|
const engine = "postgres";
|
||||||
|
const database = requireString(databaseRecord, "database");
|
||||||
|
const schema = requireString(databaseRecord, "schema");
|
||||||
|
const dwh: Record<string, unknown> = { engine, database, schema, transport };
|
||||||
|
|
||||||
|
if (transport === "postgres_direct") {
|
||||||
|
const host = optionalString(databaseRecord, "host");
|
||||||
|
const port = optionalNumber(databaseRecord, "port");
|
||||||
|
const user = optionalString(databaseRecord, "user");
|
||||||
|
if (host !== undefined) dwh.host = host;
|
||||||
|
if (port !== undefined) dwh.port = port;
|
||||||
|
if (user !== undefined) dwh.user = user;
|
||||||
|
} else if (transport === "rest_api") {
|
||||||
|
const rest = asRecord(rendered.rest) ?? asRecord(asRecord(asRecord(rendered.dwh)?.endpoint));
|
||||||
|
const baseUrl = rest ? optionalString(rest, "base_url") : undefined;
|
||||||
|
if (baseUrl !== undefined) dwh.baseUrl = baseUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
return dwh as unknown as CanonicalDwhConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildVectorConfig(rendered: Record<string, unknown>): CanonicalVectorConfig {
|
||||||
|
const resources = asRecord(rendered.resources);
|
||||||
|
const vector = resources ? asRecord(resources.vector) : undefined;
|
||||||
|
if (!vector) {
|
||||||
|
throw new TypeError("effective config is missing vector resources");
|
||||||
|
}
|
||||||
|
const collection = requireString(vector, "collection");
|
||||||
|
const semanticIndex = asRecord(rendered.semantic_index);
|
||||||
|
const vectorStore = semanticIndex ? asRecord(semanticIndex.vector_store) : undefined;
|
||||||
|
const dimensions = vectorStore
|
||||||
|
? requireNumber(vectorStore, "dimensions")
|
||||||
|
: requireNumber(vector, "dimensions");
|
||||||
|
const distance = vectorStore
|
||||||
|
? requireString(vectorStore, "distance")
|
||||||
|
: (optionalString(vector, "distance") ?? "cosine");
|
||||||
|
return { collection, dimensions, distance };
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildEmbeddingConfig(rendered: Record<string, unknown>): CanonicalEmbeddingConfig {
|
||||||
|
const resources = asRecord(rendered.resources);
|
||||||
|
const embeddings = resources ? asRecord(resources.embeddings) : undefined;
|
||||||
|
if (!embeddings) {
|
||||||
|
throw new TypeError("effective config is missing embedding resources");
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
model: requireString(embeddings, "model"),
|
||||||
|
dimensions: requireNumber(embeddings, "dimensions"),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildRootsConfig(rendered: Record<string, unknown>): CanonicalRootsConfig {
|
||||||
|
const roots = asRecord(rendered.roots) ?? asRecord(rendered.paths);
|
||||||
|
if (!roots) {
|
||||||
|
throw new TypeError("effective config is missing roots");
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
artifacts: normalizeRoot(requireString(roots, "artifacts")),
|
||||||
|
indexes: normalizeRoot(requireString(roots, "indexes")),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build the versioned, non-secret effective DWH/preprocessing configuration from a
|
||||||
|
* rendered runtime configuration object. The result contains only the fields that
|
||||||
|
* affect DWH generation identity; credentials, runtime identity, session storage,
|
||||||
|
* evidence, and service endpoints are excluded.
|
||||||
|
*/
|
||||||
|
export function buildCanonicalEffectiveConfig(renderedConfig: unknown): CanonicalEffectiveConfig {
|
||||||
|
const rendered = asRecord(renderedConfig);
|
||||||
|
if (!rendered) {
|
||||||
|
throw new TypeError("effective config requires a rendered configuration object");
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
schemaVersion: 1,
|
||||||
|
dwh: buildDwhConfig(rendered),
|
||||||
|
vector: buildVectorConfig(rendered),
|
||||||
|
embedding: buildEmbeddingConfig(rendered),
|
||||||
|
roots: buildRootsConfig(rendered),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function sha256(value: string | Buffer): string {
|
||||||
|
return `sha256:${createHash("sha256").update(value).digest("hex")}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Serialize the canonical effective config to a deterministic JSON string with the
|
||||||
|
* fixed key order defined by the shared contract. No whitespace is included.
|
||||||
|
*/
|
||||||
|
export function canonicalEffectiveConfigJson(config: CanonicalEffectiveConfig): string {
|
||||||
|
const ordered: Record<string, unknown> = { schemaVersion: config.schemaVersion };
|
||||||
|
ordered.dwh = { ...config.dwh };
|
||||||
|
ordered.vector = { ...config.vector };
|
||||||
|
ordered.embedding = { ...config.embedding };
|
||||||
|
ordered.roots = { ...config.roots };
|
||||||
|
return JSON.stringify(ordered);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the stable logical config-source identity for a workspace revision.
|
||||||
|
* This is `workspace://<workspaceId>@v1:<sha256-hex-of-canonical-json>`.
|
||||||
|
*/
|
||||||
|
export function effectiveConfigIdentity(workspaceId: string, renderedConfig: unknown): string {
|
||||||
|
const canonical = buildCanonicalEffectiveConfig(renderedConfig);
|
||||||
|
const digest = createHash("sha256").update(canonicalEffectiveConfigJson(canonical)).digest("hex");
|
||||||
|
return `workspace://${workspaceId}@v1:${digest}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the config fingerprint: `sha256:` + the SHA-256 of the canonical effective
|
||||||
|
* config JSON bytes.
|
||||||
|
*/
|
||||||
|
export function configFingerprint(renderedConfig: unknown): string {
|
||||||
|
const canonical = buildCanonicalEffectiveConfig(renderedConfig);
|
||||||
|
return sha256(canonicalEffectiveConfigJson(canonical));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the input fingerprint: `sha256:` + the SHA-256 of the logical config-source
|
||||||
|
* identity string.
|
||||||
|
*/
|
||||||
|
export function inputFingerprint(workspaceId: string, renderedConfig: unknown): string {
|
||||||
|
return sha256(effectiveConfigIdentity(workspaceId, renderedConfig));
|
||||||
|
}
|
||||||
@@ -30,6 +30,9 @@ export interface WorkspaceOperationResult {
|
|||||||
completedStages: string[];
|
completedStages: string[];
|
||||||
counts?: Record<string, number>;
|
counts?: Record<string, number>;
|
||||||
artifactIdentities?: Array<{ kind: string; digest: string }>;
|
artifactIdentities?: Array<{ kind: string; digest: string }>;
|
||||||
|
effectiveConfigIdentity?: string;
|
||||||
|
configFingerprint?: string;
|
||||||
|
inputFingerprint?: string;
|
||||||
/** Suggested FK annotations YAML for the operator to write to --output (schema suggest-fks). */
|
/** Suggested FK annotations YAML for the operator to write to --output (schema suggest-fks). */
|
||||||
suggestedFksYaml?: string;
|
suggestedFksYaml?: string;
|
||||||
warnings?: string[];
|
warnings?: string[];
|
||||||
@@ -92,6 +95,9 @@ function baseResult(
|
|||||||
descriptorBlob: runtime.descriptorBlob,
|
descriptorBlob: runtime.descriptorBlob,
|
||||||
operation,
|
operation,
|
||||||
completedStages: [],
|
completedStages: [],
|
||||||
|
effectiveConfigIdentity: runtime.configLease.effectiveConfigIdentity,
|
||||||
|
configFingerprint: runtime.configLease.configFingerprint,
|
||||||
|
inputFingerprint: runtime.configLease.inputFingerprint,
|
||||||
...extra,
|
...extra,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,6 +18,14 @@ import {
|
|||||||
import { dirname, isAbsolute, join, relative, resolve } from "node:path";
|
import { dirname, isAbsolute, join, relative, resolve } from "node:path";
|
||||||
import { readFile as readFileAsync } from "node:fs/promises";
|
import { readFile as readFileAsync } from "node:fs/promises";
|
||||||
import { parse, parseAllDocuments, stringify } from "yaml";
|
import { parse, parseAllDocuments, stringify } from "yaml";
|
||||||
|
import {
|
||||||
|
buildCanonicalEffectiveConfig,
|
||||||
|
canonicalEffectiveConfigJson,
|
||||||
|
configFingerprint,
|
||||||
|
effectiveConfigIdentity,
|
||||||
|
inputFingerprint,
|
||||||
|
type CanonicalEffectiveConfig,
|
||||||
|
} from "./effective-config.js";
|
||||||
import { resolveRuntimeBindings, type RuntimeBindings } from "./bindings.js";
|
import { resolveRuntimeBindings, type RuntimeBindings } from "./bindings.js";
|
||||||
import { GitWorkspaceRepository } from "./git-repository.js";
|
import { GitWorkspaceRepository } from "./git-repository.js";
|
||||||
import { WorkspaceRegistry } from "./registry.js";
|
import { WorkspaceRegistry } from "./registry.js";
|
||||||
@@ -63,6 +71,10 @@ export interface DeterministicRuntimeConfigLease extends RuntimeConfigLease {
|
|||||||
catalogBlob: string;
|
catalogBlob: string;
|
||||||
configDigest: string;
|
configDigest: string;
|
||||||
bindingDigest: string;
|
bindingDigest: string;
|
||||||
|
effectiveConfig: CanonicalEffectiveConfig;
|
||||||
|
effectiveConfigIdentity: string;
|
||||||
|
configFingerprint: string;
|
||||||
|
inputFingerprint: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class RuntimeConfigLeaseError extends Error {
|
export class RuntimeConfigLeaseError extends Error {
|
||||||
@@ -84,6 +96,9 @@ interface PublishedRuntimeConfigManifest {
|
|||||||
catalogBlob: string;
|
catalogBlob: string;
|
||||||
configDigest: string;
|
configDigest: string;
|
||||||
bindingDigest: string;
|
bindingDigest: string;
|
||||||
|
effectiveConfigIdentity: string;
|
||||||
|
configFingerprint: string;
|
||||||
|
inputFingerprint: string;
|
||||||
path: string;
|
path: string;
|
||||||
file: {
|
file: {
|
||||||
dev: number;
|
dev: number;
|
||||||
@@ -230,6 +245,7 @@ function runtimePaths(dataRoot: string, workspaceId: string): RuntimePaths {
|
|||||||
sessions: join(root, "sessions"),
|
sessions: join(root, "sessions"),
|
||||||
artifacts: join(root, "artifacts"),
|
artifacts: join(root, "artifacts"),
|
||||||
indexes: join(root, "indexes"),
|
indexes: join(root, "indexes"),
|
||||||
|
memory: join(root, "memory"),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -388,6 +404,9 @@ function decodePublishedRuntimeConfigManifest(value: unknown): PublishedRuntimeC
|
|||||||
|| typeof manifest.catalogBlob !== "string"
|
|| typeof manifest.catalogBlob !== "string"
|
||||||
|| typeof manifest.configDigest !== "string"
|
|| typeof manifest.configDigest !== "string"
|
||||||
|| typeof manifest.bindingDigest !== "string"
|
|| typeof manifest.bindingDigest !== "string"
|
||||||
|
|| typeof manifest.effectiveConfigIdentity !== "string"
|
||||||
|
|| typeof manifest.configFingerprint !== "string"
|
||||||
|
|| typeof manifest.inputFingerprint !== "string"
|
||||||
|| typeof manifest.path !== "string"
|
|| typeof manifest.path !== "string"
|
||||||
|| !file
|
|| !file
|
||||||
|| typeof file.dev !== "number"
|
|| typeof file.dev !== "number"
|
||||||
@@ -413,6 +432,13 @@ export async function publishDeterministicRuntimeConfigLease(options: {
|
|||||||
}): Promise<DeterministicRuntimeConfigLease> {
|
}): Promise<DeterministicRuntimeConfigLease> {
|
||||||
const rendered = await renderActiveWorkspaceRuntime(options);
|
const rendered = await renderActiveWorkspaceRuntime(options);
|
||||||
const publishedConfig = applyCollectionLifecycle(rendered.renderedConfig, "require_existing");
|
const publishedConfig = applyCollectionLifecycle(rendered.renderedConfig, "require_existing");
|
||||||
|
const renderedConfigObject = parse(rendered.renderedConfig) as Record<string, unknown>;
|
||||||
|
const effectiveConfig = buildCanonicalEffectiveConfig(renderedConfigObject);
|
||||||
|
const effectiveConfigIdentityValue = effectiveConfigIdentity(rendered.workspaceId, renderedConfigObject);
|
||||||
|
const configFingerprintValue = configFingerprint(renderedConfigObject);
|
||||||
|
const inputFingerprintValue = inputFingerprint(rendered.workspaceId, renderedConfigObject);
|
||||||
|
const identitySuffix = inputFingerprintValue.slice(7, 23);
|
||||||
|
|
||||||
const preprocessingRoot = ensureTrustedDirectory(join(
|
const preprocessingRoot = ensureTrustedDirectory(join(
|
||||||
options.dataRoot,
|
options.dataRoot,
|
||||||
"sessions",
|
"sessions",
|
||||||
@@ -421,8 +447,8 @@ export async function publishDeterministicRuntimeConfigLease(options: {
|
|||||||
));
|
));
|
||||||
const configDirectory = ensureTrustedDirectory(join(preprocessingRoot, "runtime-config"));
|
const configDirectory = ensureTrustedDirectory(join(preprocessingRoot, "runtime-config"));
|
||||||
const manifestDirectory = ensureTrustedDirectory(join(preprocessingRoot, "runtime-config-manifests"));
|
const manifestDirectory = ensureTrustedDirectory(join(preprocessingRoot, "runtime-config-manifests"));
|
||||||
const path = join(configDirectory, `${rendered.workspaceRevision}.yaml`);
|
const path = join(configDirectory, `${rendered.workspaceRevision}-${identitySuffix}.yaml`);
|
||||||
const manifestPath = join(manifestDirectory, `${rendered.workspaceRevision}.json`);
|
const manifestPath = join(manifestDirectory, `${rendered.workspaceRevision}-${identitySuffix}.json`);
|
||||||
const configDigest = sha256(publishedConfig);
|
const configDigest = sha256(publishedConfig);
|
||||||
|
|
||||||
const verifyPublished = (): PublishedRuntimeConfigManifest | undefined => {
|
const verifyPublished = (): PublishedRuntimeConfigManifest | undefined => {
|
||||||
@@ -456,6 +482,9 @@ export async function publishDeterministicRuntimeConfigLease(options: {
|
|||||||
|| manifest.catalogBlob !== rendered.catalogBlob
|
|| manifest.catalogBlob !== rendered.catalogBlob
|
||||||
|| manifest.configDigest !== configDigest
|
|| manifest.configDigest !== configDigest
|
||||||
|| manifest.bindingDigest !== rendered.bindingDigest
|
|| manifest.bindingDigest !== rendered.bindingDigest
|
||||||
|
|| manifest.effectiveConfigIdentity !== effectiveConfigIdentityValue
|
||||||
|
|| manifest.configFingerprint !== configFingerprintValue
|
||||||
|
|| manifest.inputFingerprint !== inputFingerprintValue
|
||||||
|| manifest.path !== path
|
|| manifest.path !== path
|
||||||
|| manifest.file.dev !== configStat!.dev
|
|| manifest.file.dev !== configStat!.dev
|
||||||
|| manifest.file.ino !== configStat!.ino
|
|| manifest.file.ino !== configStat!.ino
|
||||||
@@ -479,6 +508,10 @@ export async function publishDeterministicRuntimeConfigLease(options: {
|
|||||||
catalogBlob: rendered.catalogBlob,
|
catalogBlob: rendered.catalogBlob,
|
||||||
configDigest,
|
configDigest,
|
||||||
bindingDigest: rendered.bindingDigest,
|
bindingDigest: rendered.bindingDigest,
|
||||||
|
effectiveConfig,
|
||||||
|
effectiveConfigIdentity: effectiveConfigIdentityValue,
|
||||||
|
configFingerprint: configFingerprintValue,
|
||||||
|
inputFingerprint: inputFingerprintValue,
|
||||||
release: () => undefined,
|
release: () => undefined,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -505,6 +538,9 @@ export async function publishDeterministicRuntimeConfigLease(options: {
|
|||||||
catalogBlob: rendered.catalogBlob,
|
catalogBlob: rendered.catalogBlob,
|
||||||
configDigest,
|
configDigest,
|
||||||
bindingDigest: rendered.bindingDigest,
|
bindingDigest: rendered.bindingDigest,
|
||||||
|
effectiveConfigIdentity: effectiveConfigIdentityValue,
|
||||||
|
configFingerprint: configFingerprintValue,
|
||||||
|
inputFingerprint: inputFingerprintValue,
|
||||||
path,
|
path,
|
||||||
file: {
|
file: {
|
||||||
dev: Number(publishedStat.dev),
|
dev: Number(publishedStat.dev),
|
||||||
@@ -518,7 +554,8 @@ export async function publishDeterministicRuntimeConfigLease(options: {
|
|||||||
readStrictJson(manifestPath);
|
readStrictJson(manifestPath);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||||
writeAtomicFile(manifestPath, `${JSON.stringify(manifest)}\n`, 0o600);
|
writeAtomicFile(manifestPath, `${JSON.stringify(manifest)}
|
||||||
|
`, 0o600);
|
||||||
} else {
|
} else {
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
@@ -533,6 +570,10 @@ export async function publishDeterministicRuntimeConfigLease(options: {
|
|||||||
catalogBlob: rendered.catalogBlob,
|
catalogBlob: rendered.catalogBlob,
|
||||||
configDigest,
|
configDigest,
|
||||||
bindingDigest: rendered.bindingDigest,
|
bindingDigest: rendered.bindingDigest,
|
||||||
|
effectiveConfig,
|
||||||
|
effectiveConfigIdentity: effectiveConfigIdentityValue,
|
||||||
|
configFingerprint: configFingerprintValue,
|
||||||
|
inputFingerprint: inputFingerprintValue,
|
||||||
release: () => undefined,
|
release: () => undefined,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -9,6 +9,7 @@ export interface RuntimePaths {
|
|||||||
sessions: string;
|
sessions: string;
|
||||||
artifacts: string;
|
artifacts: string;
|
||||||
indexes: string;
|
indexes: string;
|
||||||
|
memory: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RuntimeIdentity {
|
export interface RuntimeIdentity {
|
||||||
@@ -255,6 +256,7 @@ export function renderRuntimeConfig(
|
|||||||
...(installation.profile === undefined ? {} : { profile: installation.profile }),
|
...(installation.profile === undefined ? {} : { profile: installation.profile }),
|
||||||
language: descriptor.workspace.language,
|
language: descriptor.workspace.language,
|
||||||
database,
|
database,
|
||||||
|
semantic_index: descriptor.semantic_index,
|
||||||
resources: {
|
resources: {
|
||||||
vector: {
|
vector: {
|
||||||
engine: "qdrant",
|
engine: "qdrant",
|
||||||
|
|||||||
@@ -0,0 +1,197 @@
|
|||||||
|
import { createHash } from "node:crypto";
|
||||||
|
import { expect, test } from "vitest";
|
||||||
|
import {
|
||||||
|
buildCanonicalEffectiveConfig,
|
||||||
|
canonicalEffectiveConfigJson,
|
||||||
|
configFingerprint,
|
||||||
|
effectiveConfigIdentity,
|
||||||
|
inputFingerprint,
|
||||||
|
} from "../src/workspaces/effective-config.js";
|
||||||
|
|
||||||
|
const semanticRuntime = {
|
||||||
|
internalQdrantUrl: "http://qdrant:6333",
|
||||||
|
internalEmbeddingUrl: "http://embedding:11434",
|
||||||
|
internalEmbeddingModel: "qwen3-embedding:0.6b",
|
||||||
|
internalEmbeddingDimensions: 1024,
|
||||||
|
};
|
||||||
|
|
||||||
|
function directRendered(): Record<string, unknown> {
|
||||||
|
return {
|
||||||
|
runtime_identity: {
|
||||||
|
workspace_id: "psd-clinical",
|
||||||
|
workspace_revision: "a".repeat(40),
|
||||||
|
source_identity: "workspace://psd-clinical",
|
||||||
|
},
|
||||||
|
session_storage: { mode: "local" },
|
||||||
|
profile: "server",
|
||||||
|
language: "it",
|
||||||
|
database: {
|
||||||
|
host: "dwh.internal",
|
||||||
|
port: 5432,
|
||||||
|
database: "postgres",
|
||||||
|
schema: "datawarehouse",
|
||||||
|
user: "thoth_reader",
|
||||||
|
password_file: "/run/secrets/dwh-password",
|
||||||
|
ssl_ca_file: "/run/secrets/dwh-ca.pem",
|
||||||
|
transport: "direct",
|
||||||
|
},
|
||||||
|
dwh: { type: "postgres_direct" },
|
||||||
|
resources: {
|
||||||
|
vector: {
|
||||||
|
engine: "qdrant",
|
||||||
|
base_url: "http://qdrant:6333",
|
||||||
|
collection: "psd-clinical",
|
||||||
|
dimensions: 1024,
|
||||||
|
distance: "cosine",
|
||||||
|
collection_lifecycle: "require_existing",
|
||||||
|
},
|
||||||
|
embeddings: {
|
||||||
|
provider: "ollama_internal",
|
||||||
|
base_url: "http://embedding:11434",
|
||||||
|
model: "qwen3-embedding:0.6b",
|
||||||
|
dimensions: 1024,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
roots: {
|
||||||
|
sessions: "/data/sessions/psd-clinical/sessions",
|
||||||
|
artifacts: "/data/sessions/psd-clinical/artifacts",
|
||||||
|
indexes: "/data/sessions/psd-clinical/indexes",
|
||||||
|
},
|
||||||
|
paths: {
|
||||||
|
sessions: "/data/sessions/psd-clinical/sessions",
|
||||||
|
artifacts: "/data/sessions/psd-clinical/artifacts",
|
||||||
|
indexes: "/data/sessions/psd-clinical/indexes",
|
||||||
|
memory: "/data/sessions/psd-clinical/memory",
|
||||||
|
},
|
||||||
|
evidence: {
|
||||||
|
sources: [{
|
||||||
|
type: "filesystem",
|
||||||
|
root: "/srv/registry/snapshots/rev/psd-clinical/evidence",
|
||||||
|
patterns: ["**/*.md"],
|
||||||
|
max_bytes: 10_485_760,
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function restRendered(): Record<string, unknown> {
|
||||||
|
return {
|
||||||
|
...directRendered(),
|
||||||
|
database: {
|
||||||
|
host: "localhost",
|
||||||
|
port: 5432,
|
||||||
|
database: "postgres",
|
||||||
|
schema: "datawarehouse",
|
||||||
|
user: "rest",
|
||||||
|
password: "",
|
||||||
|
transport: "rest",
|
||||||
|
},
|
||||||
|
dwh: {
|
||||||
|
type: "thoth_rest",
|
||||||
|
database: { database: "postgres", schema: "datawarehouse" },
|
||||||
|
endpoint: {
|
||||||
|
base_url: "https://dwh.example.test",
|
||||||
|
api_key_file: "/run/secrets/dwh-api-key",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
rest: {
|
||||||
|
base_url: "https://dwh.example.test",
|
||||||
|
api_key_file: "/run/secrets/dwh-api-key",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const directCanonical =
|
||||||
|
`{"schemaVersion":1,"dwh":{` +
|
||||||
|
`"engine":"postgres","database":"postgres","schema":"datawarehouse",` +
|
||||||
|
`"transport":"postgres_direct","host":"dwh.internal","port":5432,"user":"thoth_reader"},` +
|
||||||
|
`"vector":{"collection":"psd-clinical","dimensions":1024,"distance":"cosine"},` +
|
||||||
|
`"embedding":{"model":"qwen3-embedding:0.6b","dimensions":1024},` +
|
||||||
|
`"roots":{"artifacts":"/data/sessions/psd-clinical/artifacts",` +
|
||||||
|
`"indexes":"/data/sessions/psd-clinical/indexes"}}`;
|
||||||
|
|
||||||
|
test("canonical effective config is deterministic and contains the expected key order", () => {
|
||||||
|
const rendered = directRendered();
|
||||||
|
const canonical = buildCanonicalEffectiveConfig(rendered);
|
||||||
|
expect(canonicalEffectiveConfigJson(canonical)).toBe(directCanonical);
|
||||||
|
expect(buildCanonicalEffectiveConfig(rendered)).toEqual(canonical);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("canonical effective config excludes secrets, evidence, session storage, and runtime identity", () => {
|
||||||
|
const json = canonicalEffectiveConfigJson(buildCanonicalEffectiveConfig(directRendered()));
|
||||||
|
expect(json).not.toContain("password_file");
|
||||||
|
expect(json).not.toContain("ssl_ca_file");
|
||||||
|
expect(json).not.toContain("session_storage");
|
||||||
|
expect(json).not.toContain("runtime_identity");
|
||||||
|
expect(json).not.toContain("evidence");
|
||||||
|
expect(json).not.toContain("sources");
|
||||||
|
expect(json).not.toContain("collection_lifecycle");
|
||||||
|
expect(json).not.toContain("base_url"); // vector/embedding service URLs are not identity
|
||||||
|
expect(json).not.toContain("memory");
|
||||||
|
expect(json).not.toContain('"sessions"');
|
||||||
|
});
|
||||||
|
|
||||||
|
test("REST transport canonicalizes to transport rest_api with baseUrl and no host/port", () => {
|
||||||
|
const canonical = buildCanonicalEffectiveConfig(restRendered());
|
||||||
|
const json = canonicalEffectiveConfigJson(canonical);
|
||||||
|
expect(json).toContain(`"transport":"rest_api"`);
|
||||||
|
expect(json).toContain(`"baseUrl":"https://dwh.example.test"`);
|
||||||
|
expect(json).not.toContain(`"host":`);
|
||||||
|
expect(json).not.toContain(`"port":`);
|
||||||
|
expect(json).not.toContain("api_key_file");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("identity and fingerprint helpers produce stable prefixed hex values", () => {
|
||||||
|
const rendered = directRendered();
|
||||||
|
const identity = effectiveConfigIdentity("psd-clinical", rendered);
|
||||||
|
const cfg = configFingerprint(rendered);
|
||||||
|
const input = inputFingerprint("psd-clinical", rendered);
|
||||||
|
|
||||||
|
expect(identity).toMatch(/^workspace:\/\/psd-clinical@v1:[0-9a-f]{64}$/);
|
||||||
|
expect(cfg).toBe("sha256:" + createHash("sha256").update(directCanonical).digest("hex"));
|
||||||
|
expect(input).toBe("sha256:" + createHash("sha256").update(identity).digest("hex"));
|
||||||
|
expect(input).not.toBe(cfg);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("content-only or session_storage changes keep the same effective config identity", () => {
|
||||||
|
const base = directRendered();
|
||||||
|
const identityBefore = effectiveConfigIdentity("psd-clinical", base);
|
||||||
|
const fingerprintBefore = configFingerprint(base);
|
||||||
|
|
||||||
|
const contentOnly = {
|
||||||
|
...base,
|
||||||
|
runtime_identity: {
|
||||||
|
...base.runtime_identity,
|
||||||
|
workspace_revision: "b".repeat(40),
|
||||||
|
},
|
||||||
|
session_storage: { mode: "remote", url: "http://example.test" },
|
||||||
|
evidence: {
|
||||||
|
sources: [{
|
||||||
|
type: "filesystem",
|
||||||
|
root: "/srv/registry/snapshots/other/psd-clinical/evidence",
|
||||||
|
patterns: ["**/*.txt"],
|
||||||
|
max_bytes: 999,
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
expect(effectiveConfigIdentity("psd-clinical", contentOnly)).toBe(identityBefore);
|
||||||
|
expect(configFingerprint(contentOnly)).toBe(fingerprintBefore);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("DWH-affecting changes alter the effective config identity", () => {
|
||||||
|
const base = directRendered();
|
||||||
|
const identityBefore = effectiveConfigIdentity("psd-clinical", base);
|
||||||
|
|
||||||
|
const changedHost = { ...base, database: { ...(base.database as object), host: "dwh-two.internal" } };
|
||||||
|
expect(effectiveConfigIdentity("psd-clinical", changedHost)).not.toBe(identityBefore);
|
||||||
|
|
||||||
|
const changedDatabase = { ...base, database: { ...(base.database as object), database: "analytics" } };
|
||||||
|
expect(effectiveConfigIdentity("psd-clinical", changedDatabase)).not.toBe(identityBefore);
|
||||||
|
|
||||||
|
const changedCollection = { ...base, resources: { ...base.resources, vector: { ...(base.resources as Record<string, any>).vector, collection: "other" } } };
|
||||||
|
expect(effectiveConfigIdentity("psd-clinical", changedCollection)).not.toBe(identityBefore);
|
||||||
|
|
||||||
|
const changedTransport = restRendered();
|
||||||
|
expect(effectiveConfigIdentity("psd-clinical", changedTransport)).not.toBe(identityBefore);
|
||||||
|
});
|
||||||
@@ -115,13 +115,31 @@ function runtime(workspace = baseWorkspace, workspaceId = workspace.workspace.id
|
|||||||
descriptorBlob: "b".repeat(40),
|
descriptorBlob: "b".repeat(40),
|
||||||
catalogBlob: "c".repeat(40),
|
catalogBlob: "c".repeat(40),
|
||||||
configLease: {
|
configLease: {
|
||||||
path: `/data/sessions/${workspaceId}/preprocessing/runtime-config/${"a".repeat(40)}.yaml`,
|
path: `/data/sessions/${workspaceId}/preprocessing/runtime-config/${"a".repeat(40)}-identitysuffix.yaml`,
|
||||||
workspaceId,
|
workspaceId,
|
||||||
workspaceRevision: "a".repeat(40),
|
workspaceRevision: "a".repeat(40),
|
||||||
descriptorBlob: "b".repeat(40),
|
descriptorBlob: "b".repeat(40),
|
||||||
catalogBlob: "c".repeat(40),
|
catalogBlob: "c".repeat(40),
|
||||||
configDigest: "sha256:config",
|
configDigest: "sha256:config",
|
||||||
bindingDigest: "sha256:bindings",
|
bindingDigest: "sha256:bindings",
|
||||||
|
effectiveConfig: {
|
||||||
|
schemaVersion: 1,
|
||||||
|
dwh: {
|
||||||
|
engine: "postgres",
|
||||||
|
database: "analytics",
|
||||||
|
schema: "mart",
|
||||||
|
transport: "postgres_direct",
|
||||||
|
host: "dwh.internal",
|
||||||
|
port: 5432,
|
||||||
|
user: "reader",
|
||||||
|
},
|
||||||
|
vector: { collection: workspaceId, dimensions: 1024, distance: "cosine" },
|
||||||
|
embedding: { model: "qwen3-embedding:0.6b", dimensions: 1024 },
|
||||||
|
roots: { artifacts: "/data/artifacts", indexes: "/data/indexes" },
|
||||||
|
},
|
||||||
|
effectiveConfigIdentity: "workspace://psd-clinical@v1:" + "d".repeat(64),
|
||||||
|
configFingerprint: "sha256:" + "e".repeat(64),
|
||||||
|
inputFingerprint: "sha256:" + "f".repeat(64),
|
||||||
release: () => undefined,
|
release: () => undefined,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -14,7 +14,13 @@ import { tmpdir } from "node:os";
|
|||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import { promisify } from "node:util";
|
import { promisify } from "node:util";
|
||||||
import { afterEach, expect, test, vi } from "vitest";
|
import { afterEach, expect, test, vi } from "vitest";
|
||||||
|
import { parse } from "yaml";
|
||||||
import { WorkspaceRegistry } from "../src/workspaces/registry.js";
|
import { WorkspaceRegistry } from "../src/workspaces/registry.js";
|
||||||
|
import {
|
||||||
|
buildCanonicalEffectiveConfig,
|
||||||
|
canonicalEffectiveConfigJson,
|
||||||
|
effectiveConfigIdentity,
|
||||||
|
} from "../src/workspaces/effective-config.js";
|
||||||
import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js";
|
import type { WorkspaceRegistryConfig } from "../src/workspaces/types.js";
|
||||||
import {
|
import {
|
||||||
publishDeterministicRuntimeConfigLease,
|
publishDeterministicRuntimeConfigLease,
|
||||||
@@ -123,6 +129,7 @@ evidence:
|
|||||||
return {
|
return {
|
||||||
dataRoot,
|
dataRoot,
|
||||||
harnessDir,
|
harnessDir,
|
||||||
|
source,
|
||||||
registry,
|
registry,
|
||||||
registryConfig,
|
registryConfig,
|
||||||
revision,
|
revision,
|
||||||
@@ -163,7 +170,7 @@ test("active workspace rendering is byte-identical to direct snapshot rendering"
|
|||||||
expect(active.catalogBlob).toMatch(/^sha256:[0-9a-f]{64}$/);
|
expect(active.catalogBlob).toMatch(/^sha256:[0-9a-f]{64}$/);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("deterministic operator leases publish one revision-bound protected config and refuse changed same-revision bytes", async () => {
|
test("deterministic operator leases are keyed by logical identity and stable across calls", async () => {
|
||||||
const f = await fixture();
|
const f = await fixture();
|
||||||
const first = await publishDeterministicRuntimeConfigLease({
|
const first = await publishDeterministicRuntimeConfigLease({
|
||||||
workspaceId: "psd-clinical",
|
workspaceId: "psd-clinical",
|
||||||
@@ -186,6 +193,7 @@ test("deterministic operator leases publish one revision-bound protected config
|
|||||||
semanticRuntime,
|
semanticRuntime,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const suffix = first.inputFingerprint.slice(7, 23);
|
||||||
expect(second.path).toBe(first.path);
|
expect(second.path).toBe(first.path);
|
||||||
expect(first.path).toBe(join(
|
expect(first.path).toBe(join(
|
||||||
f.dataRoot,
|
f.dataRoot,
|
||||||
@@ -193,15 +201,34 @@ test("deterministic operator leases publish one revision-bound protected config
|
|||||||
"psd-clinical",
|
"psd-clinical",
|
||||||
"preprocessing",
|
"preprocessing",
|
||||||
"runtime-config",
|
"runtime-config",
|
||||||
`${f.revision.commit}.yaml`,
|
`${f.revision.commit}-${suffix}.yaml`,
|
||||||
));
|
));
|
||||||
expect(statSync(first.path).mode & 0o777).toBe(0o400);
|
expect(statSync(first.path).mode & 0o777).toBe(0o400);
|
||||||
expect(statSync(first.manifestPath).mode & 0o777).toBe(0o600);
|
expect(statSync(first.manifestPath).mode & 0o777).toBe(0o600);
|
||||||
expect(readFileSync(first.path, "utf8")).toContain("collection_lifecycle: require_existing");
|
expect(readFileSync(first.path, "utf8")).toContain("collection_lifecycle: require_existing");
|
||||||
|
expect(readFileSync(first.path, "utf8")).toContain("memory:");
|
||||||
expect(existsSync(first.manifestPath)).toBe(true);
|
expect(existsSync(first.manifestPath)).toBe(true);
|
||||||
|
expect(first.effectiveConfigIdentity).toMatch(/^workspace:\/\/psd-clinical@v1:[0-9a-f]{64}$/);
|
||||||
|
expect(first.configFingerprint).toMatch(/^sha256:[0-9a-f]{64}$/);
|
||||||
|
expect(first.inputFingerprint).toMatch(/^sha256:[0-9a-f]{64}$/);
|
||||||
|
expect(first.inputFingerprint).not.toBe(first.configFingerprint);
|
||||||
|
const manifest = JSON.parse(readFileSync(first.manifestPath, "utf8"));
|
||||||
|
expect(manifest).toMatchObject({
|
||||||
|
schemaVersion: 1,
|
||||||
|
workspaceId: "psd-clinical",
|
||||||
|
workspaceRevision: f.revision.commit,
|
||||||
|
descriptorBlob: first.descriptorBlob,
|
||||||
|
catalogBlob: first.catalogBlob,
|
||||||
|
configDigest: first.configDigest,
|
||||||
|
bindingDigest: first.bindingDigest,
|
||||||
|
effectiveConfigIdentity: first.effectiveConfigIdentity,
|
||||||
|
configFingerprint: first.configFingerprint,
|
||||||
|
inputFingerprint: first.inputFingerprint,
|
||||||
|
path: first.path,
|
||||||
|
});
|
||||||
|
|
||||||
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_HOST", "warehouse-two.internal");
|
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_HOST", "warehouse-two.internal");
|
||||||
await expect(publishDeterministicRuntimeConfigLease({
|
const changed = await publishDeterministicRuntimeConfigLease({
|
||||||
workspaceId: "psd-clinical",
|
workspaceId: "psd-clinical",
|
||||||
registry: f.registry,
|
registry: f.registry,
|
||||||
registryConfig: f.registryConfig,
|
registryConfig: f.registryConfig,
|
||||||
@@ -210,7 +237,11 @@ test("deterministic operator leases publish one revision-bound protected config
|
|||||||
dataRoot: f.dataRoot,
|
dataRoot: f.dataRoot,
|
||||||
secretRoots: f.registryConfig.secretRoots,
|
secretRoots: f.registryConfig.secretRoots,
|
||||||
semanticRuntime,
|
semanticRuntime,
|
||||||
})).rejects.toMatchObject({ code: "effective_config_mismatch" });
|
});
|
||||||
|
expect(changed.path).not.toBe(first.path);
|
||||||
|
expect(changed.inputFingerprint).not.toBe(first.inputFingerprint);
|
||||||
|
expect(changed.configFingerprint).not.toBe(first.configFingerprint);
|
||||||
|
expect(readFileSync(changed.path, "utf8")).toContain("warehouse-two.internal");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("runtime rendering rejects untrusted snapshot paths and symlinks", async () => {
|
test("runtime rendering rejects untrusted snapshot paths and symlinks", async () => {
|
||||||
@@ -237,3 +268,74 @@ test("runtime rendering rejects untrusted snapshot paths and symlinks", async ()
|
|||||||
semanticRuntime,
|
semanticRuntime,
|
||||||
})).toThrow(/trusted runtime snapshot/i);
|
})).toThrow(/trusted runtime snapshot/i);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
|
test("operator lease and session snapshot produce byte-identical effective DWH bindings", async () => {
|
||||||
|
const f = await fixture();
|
||||||
|
const session = renderWorkspaceRuntimeFromSnapshotPath({
|
||||||
|
snapshotPath: f.revision.snapshotPath,
|
||||||
|
harnessDir: f.harnessDir,
|
||||||
|
configPath: "config/tht.yaml",
|
||||||
|
dataRoot: f.dataRoot,
|
||||||
|
secretRoots: f.registryConfig.secretRoots,
|
||||||
|
semanticRuntime,
|
||||||
|
});
|
||||||
|
const lease = await publishDeterministicRuntimeConfigLease({
|
||||||
|
workspaceId: "psd-clinical",
|
||||||
|
registry: f.registry,
|
||||||
|
registryConfig: f.registryConfig,
|
||||||
|
harnessDir: f.harnessDir,
|
||||||
|
configPath: "config/tht.yaml",
|
||||||
|
dataRoot: f.dataRoot,
|
||||||
|
secretRoots: f.registryConfig.secretRoots,
|
||||||
|
semanticRuntime,
|
||||||
|
});
|
||||||
|
|
||||||
|
const sessionCanonical = canonicalEffectiveConfigJson(buildCanonicalEffectiveConfig(parse(session.renderedConfig)));
|
||||||
|
const operatorCanonical = canonicalEffectiveConfigJson(lease.effectiveConfig);
|
||||||
|
expect(operatorCanonical).toBe(sessionCanonical);
|
||||||
|
expect(lease.effectiveConfigIdentity).toBe(
|
||||||
|
effectiveConfigIdentity("psd-clinical", parse(session.renderedConfig)),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a content-only Evidence commit keeps the same effective config identity with a new revision lease", async () => {
|
||||||
|
const f = await fixture();
|
||||||
|
const firstLease = await publishDeterministicRuntimeConfigLease({
|
||||||
|
workspaceId: "psd-clinical",
|
||||||
|
registry: f.registry,
|
||||||
|
registryConfig: f.registryConfig,
|
||||||
|
harnessDir: f.harnessDir,
|
||||||
|
configPath: "config/tht.yaml",
|
||||||
|
dataRoot: f.dataRoot,
|
||||||
|
secretRoots: f.registryConfig.secretRoots,
|
||||||
|
semanticRuntime,
|
||||||
|
});
|
||||||
|
const firstIdentity = firstLease.effectiveConfigIdentity;
|
||||||
|
|
||||||
|
writeFileSync(
|
||||||
|
join(f.source, "psd-clinical", "evidence", "guide.md"),
|
||||||
|
"# updated content only\n",
|
||||||
|
);
|
||||||
|
await git(f.source, ["add", "psd-clinical/evidence/guide.md"]);
|
||||||
|
await git(f.source, ["commit", "-m", "Evidence content only"]);
|
||||||
|
await git(f.source, ["push", "origin", "main"]);
|
||||||
|
await f.registry.pull();
|
||||||
|
const current = (await f.registry.list())[0];
|
||||||
|
|
||||||
|
const secondLease = await publishDeterministicRuntimeConfigLease({
|
||||||
|
workspaceId: "psd-clinical",
|
||||||
|
registry: f.registry,
|
||||||
|
registryConfig: f.registryConfig,
|
||||||
|
harnessDir: f.harnessDir,
|
||||||
|
configPath: "config/tht.yaml",
|
||||||
|
dataRoot: f.dataRoot,
|
||||||
|
secretRoots: f.registryConfig.secretRoots,
|
||||||
|
semanticRuntime,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(secondLease.workspaceRevision).toBe(current.commit);
|
||||||
|
expect(secondLease.workspaceRevision).not.toBe(firstLease.workspaceRevision);
|
||||||
|
expect(secondLease.effectiveConfigIdentity).toBe(firstIdentity);
|
||||||
|
expect(secondLease.path).not.toBe(firstLease.path);
|
||||||
|
});
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ const paths: RuntimePaths = {
|
|||||||
sessions: "/data/workspaces/psd-clinical/sessions",
|
sessions: "/data/workspaces/psd-clinical/sessions",
|
||||||
artifacts: "/data/workspaces/psd-clinical/artifacts",
|
artifacts: "/data/workspaces/psd-clinical/artifacts",
|
||||||
indexes: "/data/workspaces/psd-clinical/indexes",
|
indexes: "/data/workspaces/psd-clinical/indexes",
|
||||||
|
memory: "/data/workspaces/psd-clinical/memory",
|
||||||
};
|
};
|
||||||
const semanticRuntime: SemanticRuntimeConfig = {
|
const semanticRuntime: SemanticRuntimeConfig = {
|
||||||
internalQdrantUrl: "http://qdrant:6333",
|
internalQdrantUrl: "http://qdrant:6333",
|
||||||
|
|||||||
@@ -0,0 +1,128 @@
|
|||||||
|
# `.tht-dwh` — DWH generations, `OWNER.json`, ACTIVE, and fingerprints
|
||||||
|
|
||||||
|
> Operator contract. P3 makes the effective DWH/preprocessing configuration reproducible and
|
||||||
|
> versioned across the operator CLI and the application sessions, and documents what `.tht-dwh`
|
||||||
|
> is so operators can reason about why a rerun is instant or why it takes minutes.
|
||||||
|
|
||||||
|
## What `.tht-dwh` is
|
||||||
|
|
||||||
|
`.tht-dwh` is the workspace-local directory that stores the **prepared snapshots of the data
|
||||||
|
warehouse structure** (the catalog `physical.yaml` plus the LSH hashes used for fuzzy search).
|
||||||
|
ThothII does not re-read the whole database for every question: it prepares it once, stores the
|
||||||
|
result here, and reuses it. The directory lives under the workspace runtime root, for example:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/data/sessions/<workspace-id>/.tht-dwh/
|
||||||
|
```
|
||||||
|
|
||||||
|
## Immutable generations
|
||||||
|
|
||||||
|
Each preparation run produces a **generation**: an immutable directory containing the catalog and
|
||||||
|
the LSH artifacts for one exact "effective configuration" (see fingerprints below). Generations
|
||||||
|
are never modified in place; a new run writes a new generation, and an `ACTIVE` pointer selects
|
||||||
|
which generation the workspace currently uses. Keeping the old generations makes rollback and
|
||||||
|
diagnosis safe.
|
||||||
|
|
||||||
|
## `OWNER.json`
|
||||||
|
|
||||||
|
Every generation root contains an `OWNER.json` that records who owns it:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"workspace_id": "<workspace-id>",
|
||||||
|
"config_fingerprint": "sha256:<64 hex>",
|
||||||
|
"input_fingerprint": "sha256:<64 hex>"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- `config_fingerprint` is the digest of the **canonical effective configuration** (see below).
|
||||||
|
- `input_fingerprint` is the digest of the **logical configuration identity**.
|
||||||
|
|
||||||
|
Before reusing a generation, the harness compares the current canonical identity with the one in
|
||||||
|
`OWNER.json`. If they differ, the generation is **refused** (never silently reused) and a new one
|
||||||
|
is produced. This is what protects ThothII from using artifacts prepared for a different database,
|
||||||
|
endpoint, user, schema, or index contract.
|
||||||
|
|
||||||
|
The reader is compatible with the historical schema-v1 `OWNER.json` (same three keys, `sha256:`
|
||||||
|
values) so existing installations keep working; new writes use the versioned computation. There is
|
||||||
|
no automatic in-place reinterpretation: operators regenerate explicitly when a root is old.
|
||||||
|
|
||||||
|
## The canonical effective configuration and the logical identity
|
||||||
|
|
||||||
|
The **canonical effective configuration** is the non-secret subset of the rendered runtime
|
||||||
|
configuration that determines whether a prepared DWH generation is still valid:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"dwh": {
|
||||||
|
"engine": "postgres",
|
||||||
|
"database": "<database>",
|
||||||
|
"schema": "<schema>",
|
||||||
|
"transport": "postgres_direct | rest_api | ...",
|
||||||
|
"host": "<host>",
|
||||||
|
"port": 5432,
|
||||||
|
"baseUrl": "<base-url>",
|
||||||
|
"user": "<user>"
|
||||||
|
},
|
||||||
|
"vector": { "collection": "<collection>", "dimensions": 1024, "distance": "cosine" },
|
||||||
|
"embedding": { "model": "<model>", "dimensions": 1024 },
|
||||||
|
"roots": { "artifacts": "<abs-path>", "indexes": "<abs-path>" }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Deliberately **excluded** (their change must not invalidate a DWH generation):
|
||||||
|
|
||||||
|
- `session_storage` and `runtime_identity` (a content-only Git commit or an Evidence-only change
|
||||||
|
must not force a full database re-introspection);
|
||||||
|
- Evidence source/policy (P6 materialization and Evidence preprocessing are separate);
|
||||||
|
- memory, search, and execution settings;
|
||||||
|
- **all credentials** (passwords, API keys, signed URLs, and secret-file paths).
|
||||||
|
|
||||||
|
The **logical configuration identity** is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
workspace://<workspace-id>@v1:<sha256 of the canonical effective configuration>
|
||||||
|
```
|
||||||
|
|
||||||
|
It is the same for the operator CLI and for application sessions, because both derive it from the
|
||||||
|
same rendered configuration. That is the guarantee that the work prepared by `thothctl` is exactly
|
||||||
|
what the sessions will consume.
|
||||||
|
|
||||||
|
## Why a rerun can be instant or take minutes
|
||||||
|
|
||||||
|
- Same canonical identity (e.g., only Evidence files changed) → the generation is reused → the
|
||||||
|
DWH step is `unchanged` and fast.
|
||||||
|
- Changed canonical identity (different database, address, user, schema, collection, model, or
|
||||||
|
artifact/index roots) → the old generation is refused → ThothII re-introspects and writes a new
|
||||||
|
generation → the step takes as long as the first preparation.
|
||||||
|
|
||||||
|
## Safe migration, regeneration, and recovery
|
||||||
|
|
||||||
|
- **Migration**: existing schema-v1 `OWNER.json` roots are readable; to switch them to the
|
||||||
|
versioned identity, run a normal regeneration (explicit `--refresh`/new run). No automatic
|
||||||
|
in-place rewrite.
|
||||||
|
- **Regeneration**: a new run produces a new immutable generation and moves `ACTIVE`; the previous
|
||||||
|
generations remain for rollback.
|
||||||
|
- **Recovery**: if the active generation is corrupt or owned by another configuration, ThothII
|
||||||
|
fails closed (never mixes artifacts) and tells the operator to regenerate; the old generations
|
||||||
|
are still available for inspection.
|
||||||
|
|
||||||
|
## Memory root
|
||||||
|
|
||||||
|
P3 also gives each workspace an explicit **workspace-global memory root**:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/data/sessions/<workspace-id>/memory/
|
||||||
|
```
|
||||||
|
|
||||||
|
All memory commands, locks, the canonical JSONL registry, and the Qdrant projection use this root
|
||||||
|
when present. A guarded migration copies and verifies exactly one legacy canonical JSONL from the
|
||||||
|
old `artifacts/memory` location under the workspace lock and rebuilds the projection; conflicting
|
||||||
|
legacy registries fail closed. There is no in-place reinterpretation.
|
||||||
|
|
||||||
|
## Revision-scoped search records
|
||||||
|
|
||||||
|
Schema and Evidence records in the Qdrant collection include the pinned `workspace_revision`, so
|
||||||
|
searches never mix descriptions or documents from different versions of the workspace. Memory and
|
||||||
|
solved-question records remain workspace-wide on purpose.
|
||||||
@@ -30,6 +30,15 @@ per `docs/contracts/workspace-preprocessing-cli.md` and the P2 walkthrough in
|
|||||||
`workspace-maintenance` Compose service; it never starts a backend/Pi/frontend listener and never
|
`workspace-maintenance` Compose service; it never starts a backend/Pi/frontend listener and never
|
||||||
attaches Git credentials.
|
attaches Git credentials.
|
||||||
|
|
||||||
|
|
||||||
|
## Effective configuration and `.tht-dwh` (P3)
|
||||||
|
|
||||||
|
Prepared DWH generations are reusable and safe: `thothctl` and the application derive the same
|
||||||
|
canonical effective configuration and logical identity, so prepared work is reused when nothing
|
||||||
|
relevant changed and refused when the database/endpoint/identity changed. See
|
||||||
|
`docs/contracts/tht-dwh.md` for generations, `OWNER.json`, `ACTIVE`, fingerprints, migration and
|
||||||
|
recovery. A content-only or Evidence-only change never forces a full re-introspection.
|
||||||
|
|
||||||
## Prerequisites
|
## Prerequisites
|
||||||
|
|
||||||
- macOS: Docker Desktop, Git, and sufficient volume disk space. Git Credential Manager is useful
|
- macOS: Docker Desktop, Git, and sufficient volume disk space. Git Credential Manager is useful
|
||||||
|
|||||||
@@ -25,6 +25,15 @@ per `docs/contracts/workspace-preprocessing-cli.md` and the P2 walkthrough in
|
|||||||
`workspace-maintenance` Compose service; it never starts a backend/Pi/frontend listener and never
|
`workspace-maintenance` Compose service; it never starts a backend/Pi/frontend listener and never
|
||||||
attaches Git credentials.
|
attaches Git credentials.
|
||||||
|
|
||||||
|
|
||||||
|
## Effective configuration and `.tht-dwh` (P3)
|
||||||
|
|
||||||
|
Prepared DWH generations are reusable and safe: `thothctl` and the application derive the same
|
||||||
|
canonical effective configuration and logical identity, so prepared work is reused when nothing
|
||||||
|
relevant changed and refused when the database/endpoint/identity changed. See
|
||||||
|
`docs/contracts/tht-dwh.md` for generations, `OWNER.json`, `ACTIVE`, fingerprints, migration and
|
||||||
|
recovery. A content-only or Evidence-only change never forces a full re-introspection.
|
||||||
|
|
||||||
## Service account, storage, and firewall
|
## Service account, storage, and firewall
|
||||||
|
|
||||||
Create a dedicated host service account and an operator root such as `/srv/thothii`. The core
|
Create a dedicated host service account and an operator root such as `/srv/thothii`. The core
|
||||||
|
|||||||
@@ -56,25 +56,32 @@ Checks:
|
|||||||
|
|
||||||
Decision: **PENDING** (independent manual gate; automation never records PASS).
|
Decision: **PENDING** (independent manual gate; automation never records PASS).
|
||||||
|
|
||||||
## P3 — Effective config and `.tht-dwh`
|
## P3 — Effective configuration and `.tht-dwh`
|
||||||
|
|
||||||
**Status:** instructions to be finalized by P3 implementation; not yet runnable.
|
**Status:** P3 implementation complete; automated integration PASS; manual acceptance PENDING.
|
||||||
|
|
||||||
Manual goal: compare operator and session effective DWH identities, inspect `OWNER.json` and
|
Manual goal: prove that the operator CLI and application sessions derive the same effective
|
||||||
`ACTIVE` without exposing secrets, prove safe reuse after a content-only revision, and prove
|
configuration, that a content-only revision reuses the prepared DWH generation (fast, `unchanged`),
|
||||||
fail-closed behavior after a DWH-affecting change.
|
that a DWH-affecting change fails closed and regenerates, that the workspace memory migration is
|
||||||
|
safe, and that search records are revision-scoped. See `docs/contracts/tht-dwh.md`.
|
||||||
|
|
||||||
Checks to fill during P3:
|
Checks:
|
||||||
|
|
||||||
1. canonical fingerprint comparison;
|
1. run `thothctl ... workspace preprocess dwh` twice with only an Evidence/content change between
|
||||||
2. stable logical config-source identity;
|
them: the second run reports `unchanged` and does not re-introspect;
|
||||||
3. schema-v1 ownership compatibility/migration;
|
2. change a DWH-affecting field (host/port/database/schema/user/collection) in the descriptor,
|
||||||
4. DWH cache reuse across equivalent revisions;
|
push, pull: the next run refuses the old generation and regenerates, with a clear
|
||||||
5. revision-scoped schema/Evidence state;
|
`effective_config_mismatch`-style outcome and no mixed artifacts;
|
||||||
6. mismatch rejection and recovery.
|
3. inspect `.tht-dwh` generations: immutable directories, `OWNER.json` with the canonical
|
||||||
|
fingerprints, `ACTIVE` pointer; old generations still present;
|
||||||
|
4. memory: after the guarded migration the workspace uses
|
||||||
|
`<dataRoot>/sessions/<workspace-id>/memory/`; the JSONL registry and Qdrant projection are
|
||||||
|
rebuilt and consistent; a conflicting legacy registry fails closed;
|
||||||
|
5. search records: schema/Evidence points carry the pinned `workspace_revision`; memory/solved
|
||||||
|
records remain workspace-wide;
|
||||||
|
6. documentation: `docs/contracts/tht-dwh.md` matches the observed behavior.
|
||||||
|
|
||||||
Decision: **PENDING**.
|
Decision: **PENDING**.
|
||||||
|
|
||||||
## P4 — Qdrant bootstrap and guarded rebuild
|
## P4 — Qdrant bootstrap and guarded rebuild
|
||||||
|
|
||||||
**Status:** instructions to be finalized by P4 implementation; not yet runnable.
|
**Status:** instructions to be finalized by P4 implementation; not yet runnable.
|
||||||
|
|||||||
@@ -0,0 +1,90 @@
|
|||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from tht.config import (
|
||||||
|
canonical_effective_config_json,
|
||||||
|
effective_config_fingerprint,
|
||||||
|
effective_config_identity,
|
||||||
|
effective_config_input_fingerprint,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _write_cfg(tmp_path, raw):
|
||||||
|
import yaml as _yaml
|
||||||
|
|
||||||
|
from tht.config import load_config
|
||||||
|
path = tmp_path / "config.yaml"
|
||||||
|
path.write_text(_yaml.safe_dump(raw), encoding="utf-8")
|
||||||
|
cfg = load_config(path)
|
||||||
|
cfg._workspace_id = raw.get("workspace", {}).get("id", "psd")
|
||||||
|
cfg._config_source = f"workspace://{cfg._workspace_id}"
|
||||||
|
return cfg
|
||||||
|
|
||||||
|
|
||||||
|
def _cfg(tmp_path, *, transport="thoth_rest", base_url="http://dwh.example.invalid", collection="psd", model="qwen3-embedding:0.6b"):
|
||||||
|
return {
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"workspace": {"schema_version": 3, "id": "psd", "name": "PSD", "language": "it"},
|
||||||
|
"dwh": {
|
||||||
|
"type": transport,
|
||||||
|
"database": {"database": "warehouse", "schema": "dw"},
|
||||||
|
"endpoint": {"base_url": base_url, "api_key": "secret"},
|
||||||
|
} if transport == "thoth_rest" else {
|
||||||
|
"type": "postgres_direct",
|
||||||
|
"connection": {"host": "h", "port": 5432, "database": "warehouse", "schema": "dw", "user": "reader", "password": "secret"},
|
||||||
|
},
|
||||||
|
"vectors": {"type": "qdrant", "base_url": "http://qdrant:6333", "collection": collection, "collection_lifecycle": "self_heal"},
|
||||||
|
"embeddings": {"provider": "ollama_internal", "base_url": "http://embedding:11434", "model": model, "dimensions": 1024},
|
||||||
|
"roots": {"artifacts": str(tmp_path / "artifacts"), "indexes": str(tmp_path / "indexes")},
|
||||||
|
"paths": {"artifacts": str(tmp_path / "artifacts"), "indexes": str(tmp_path / "indexes"), "sessions": str(tmp_path / "sessions")},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def cfg(tmp_path):
|
||||||
|
return _write_cfg(tmp_path, _cfg(tmp_path))
|
||||||
|
|
||||||
|
|
||||||
|
def test_canonical_json_is_deterministic_and_key_ordered(cfg):
|
||||||
|
doc = canonical_effective_config_json(cfg)
|
||||||
|
assert doc == canonical_effective_config_json(cfg)
|
||||||
|
keys = list(__import__("json").loads(doc).keys())
|
||||||
|
assert keys == ["schemaVersion", "dwh", "vector", "embedding", "roots"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_canonical_excludes_credentials_and_evidence(cfg):
|
||||||
|
doc = canonical_effective_config_json(cfg)
|
||||||
|
assert "secret" not in doc
|
||||||
|
assert "password" not in doc
|
||||||
|
assert "evidence" not in doc
|
||||||
|
assert "session_storage" not in doc
|
||||||
|
assert "runtime_identity" not in doc
|
||||||
|
|
||||||
|
|
||||||
|
def test_identity_and_fingerprints_format(cfg):
|
||||||
|
ident = effective_config_identity("psd", cfg)
|
||||||
|
assert ident == "workspace://psd@v1:" + effective_config_fingerprint(cfg)[len("sha256:"):]
|
||||||
|
assert effective_config_input_fingerprint("psd", cfg).startswith("sha256:")
|
||||||
|
assert len(effective_config_fingerprint(cfg)) == len("sha256:") + 64
|
||||||
|
|
||||||
|
|
||||||
|
def test_content_only_change_keeps_identity(tmp_path):
|
||||||
|
base = _write_cfg(tmp_path, _cfg(tmp_path))
|
||||||
|
changed = _cfg(tmp_path)
|
||||||
|
# A non-DWH-affecting change (collection lifecycle policy) must not alter the identity;
|
||||||
|
# the canonical document excludes it. Evidence changes are proven end-to-end by acceptance.
|
||||||
|
changed["vectors"]["collection_lifecycle"] = "require_existing"
|
||||||
|
changed_cfg = _write_cfg(tmp_path, changed)
|
||||||
|
assert effective_config_identity("psd", base) == effective_config_identity("psd", changed_cfg)
|
||||||
|
|
||||||
|
|
||||||
|
def test_dwh_affecting_change_alters_identity(tmp_path):
|
||||||
|
base = _write_cfg(tmp_path, _cfg(tmp_path))
|
||||||
|
other = _write_cfg(tmp_path, _cfg(tmp_path, base_url="http://other.example.invalid"))
|
||||||
|
assert effective_config_identity("psd", base) != effective_config_identity("psd", other)
|
||||||
|
|
||||||
|
|
||||||
|
def test_transport_change_alters_identity(tmp_path):
|
||||||
|
rest = _write_cfg(tmp_path, _cfg(tmp_path, transport="thoth_rest"))
|
||||||
|
direct = _write_cfg(tmp_path, _cfg(tmp_path, transport="postgres_direct"))
|
||||||
|
assert effective_config_identity("psd", rest) != effective_config_identity("psd", direct)
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
def _cfg(tmp_path, **overrides):
|
||||||
|
base = {
|
||||||
|
"dwh": {
|
||||||
|
"type": "thoth_rest",
|
||||||
|
"database": {"database": "warehouse", "schema": "dw"},
|
||||||
|
"endpoint": {"base_url": "http://dwh.example.invalid", "api_key": "secret"},
|
||||||
|
},
|
||||||
|
"vectors": {"type": "qdrant", "base_url": "http://qdrant:6333", "collection": "psd", "collection_lifecycle": "self_heal"},
|
||||||
|
"embeddings": {"provider": "ollama_internal", "base_url": "http://embedding:11434", "model": "qwen3-embedding:0.6b", "dimensions": 1024},
|
||||||
|
"roots": {"artifacts": str(tmp_path / "artifacts"), "indexes": str(tmp_path / "indexes")},
|
||||||
|
"paths": {"artifacts": str(tmp_path / "artifacts"), "indexes": str(tmp_path / "indexes"), "sessions": str(tmp_path / "sessions")},
|
||||||
|
}
|
||||||
|
base.update(overrides)
|
||||||
|
return base
|
||||||
|
|
||||||
|
|
||||||
|
def _write_cfg(tmp_path, raw):
|
||||||
|
import yaml as _yaml
|
||||||
|
|
||||||
|
from tht.config import load_config
|
||||||
|
path = tmp_path / "config.yaml"
|
||||||
|
path.write_text(_yaml.safe_dump(raw), encoding="utf-8")
|
||||||
|
cfg = load_config(path)
|
||||||
|
cfg._workspace_id = "psd"
|
||||||
|
cfg._config_source = "workspace://psd"
|
||||||
|
return cfg
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def binding(tmp_path):
|
||||||
|
from tht.jobs.dwh_pipeline import config_dwh_binding
|
||||||
|
return config_dwh_binding(_write_cfg(tmp_path, _cfg(tmp_path)))
|
||||||
|
|
||||||
|
|
||||||
|
def test_binding_has_versioned_fingerprints(binding):
|
||||||
|
assert binding["workspace_id"] == "psd"
|
||||||
|
assert binding["config_fingerprint"].startswith("sha256:")
|
||||||
|
assert binding["input_fingerprint"].startswith("sha256:")
|
||||||
|
assert len(binding["config_fingerprint"]) == 71
|
||||||
|
|
||||||
|
|
||||||
|
def test_content_only_change_keeps_binding(tmp_path):
|
||||||
|
from tht.jobs.dwh_pipeline import config_dwh_binding
|
||||||
|
cfg1 = _write_cfg(tmp_path, _cfg(tmp_path))
|
||||||
|
import copy
|
||||||
|
raw = copy.deepcopy(_cfg(tmp_path))
|
||||||
|
raw["vectors"]["collection_lifecycle"] = "require_existing"
|
||||||
|
cfg3 = _write_cfg(tmp_path, raw)
|
||||||
|
assert config_dwh_binding(cfg1)["config_fingerprint"] == config_dwh_binding(cfg3)["config_fingerprint"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_endpoint_change_changes_binding(tmp_path):
|
||||||
|
from tht.jobs.dwh_pipeline import config_dwh_binding
|
||||||
|
cfg1 = _write_cfg(tmp_path, _cfg(tmp_path))
|
||||||
|
raw = _cfg(tmp_path)
|
||||||
|
raw["dwh"] = {"type": "thoth_rest", "database": {"database": "warehouse", "schema": "dw"}, "endpoint": {"base_url": "http://other.example.invalid", "api_key": "secret"}}
|
||||||
|
cfg2 = _write_cfg(tmp_path, raw)
|
||||||
|
assert config_dwh_binding(cfg1)["config_fingerprint"] != config_dwh_binding(cfg2)["config_fingerprint"]
|
||||||
@@ -347,7 +347,8 @@ def test_upsert_serializes_qdrant_point_payloads(record, semantic_kind):
|
|||||||
store.upsert("memory" if semantic_kind == "memory" else "evidence" if semantic_kind == "evidence" else "schema_records", [record])
|
store.upsert("memory" if semantic_kind == "memory" else "evidence" if semantic_kind == "evidence" else "schema_records", [record])
|
||||||
|
|
||||||
point = next(iter(fake.points.values()))
|
point = next(iter(fake.points.values()))
|
||||||
assert point["id"] == point_id("demo", semantic_kind, record.record.id)
|
expected_revision = "a" * 40 if semantic_kind in ("schema_table", "schema_column", "evidence") else None
|
||||||
|
assert point["id"] == point_id("demo", semantic_kind, record.record.id, expected_revision)
|
||||||
assert point["vector"] == record.embedding
|
assert point["vector"] == record.embedding
|
||||||
assert point["payload"]["workspace_id"] == "demo"
|
assert point["payload"]["workspace_id"] == "demo"
|
||||||
assert point["payload"]["workspace_revision"] == "a" * 40
|
assert point["payload"]["workspace_revision"] == "a" * 40
|
||||||
|
|||||||
@@ -36,7 +36,10 @@ _KEYWORD_INDEXES = (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def point_id(workspace_id: str, kind: str, record_key: str) -> str:
|
def point_id(workspace_id: str, kind: str, record_key: str, workspace_revision: str | None = None) -> str:
|
||||||
|
# P3: schema/Evidence points are revision-scoped; memory/solved remain workspace-wide.
|
||||||
|
if workspace_revision is not None:
|
||||||
|
return str(uuid5(NAMESPACE_URL, f"thothii:{workspace_id}:{workspace_revision}:{kind}:{record_key}"))
|
||||||
return str(uuid5(NAMESPACE_URL, f"thothii:{workspace_id}:{kind}:{record_key}"))
|
return str(uuid5(NAMESPACE_URL, f"thothii:{workspace_id}:{kind}:{record_key}"))
|
||||||
|
|
||||||
|
|
||||||
@@ -63,6 +66,7 @@ class QdrantVectorStore:
|
|||||||
self._base_url = base_url.rstrip("/")
|
self._base_url = base_url.rstrip("/")
|
||||||
self._collection = collection
|
self._collection = collection
|
||||||
self._workspace_id = workspace_id
|
self._workspace_id = workspace_id
|
||||||
|
self._workspace_revision = None
|
||||||
self._workspace_revision = workspace_revision
|
self._workspace_revision = workspace_revision
|
||||||
self._expected_dimension = expected_dimension
|
self._expected_dimension = expected_dimension
|
||||||
self._collection_lifecycle = collection_lifecycle
|
self._collection_lifecycle = collection_lifecycle
|
||||||
@@ -127,6 +131,7 @@ class QdrantVectorStore:
|
|||||||
if not allowed_record_kinds:
|
if not allowed_record_kinds:
|
||||||
return []
|
return []
|
||||||
filter_must = self._workspace_filter()
|
filter_must = self._workspace_filter()
|
||||||
|
filter_must.extend(self._revision_filter(allowed_record_kinds))
|
||||||
filter_must.append(self._semantic_kind_filter(allowed_record_kinds))
|
filter_must.append(self._semantic_kind_filter(allowed_record_kinds))
|
||||||
filter_must.append({"key": "record_kind", "match": {"any": allowed_record_kinds}})
|
filter_must.append({"key": "record_kind", "match": {"any": allowed_record_kinds}})
|
||||||
if metadata_filter is not None:
|
if metadata_filter is not None:
|
||||||
@@ -195,7 +200,12 @@ class QdrantVectorStore:
|
|||||||
semantic_kind = qdrant_semantic_kind(write_record.record.kind)
|
semantic_kind = qdrant_semantic_kind(write_record.record.kind)
|
||||||
points.append(
|
points.append(
|
||||||
{
|
{
|
||||||
"id": point_id(self._workspace_id, semantic_kind, write_record.record.id),
|
"id": point_id(
|
||||||
|
self._workspace_id,
|
||||||
|
semantic_kind,
|
||||||
|
write_record.record.id,
|
||||||
|
self._workspace_revision if semantic_kind in ("schema_table", "schema_column", "evidence") else None,
|
||||||
|
),
|
||||||
"vector": write_record.embedding,
|
"vector": write_record.embedding,
|
||||||
"payload": qdrant_payload(
|
"payload": qdrant_payload(
|
||||||
write_record.record,
|
write_record.record,
|
||||||
@@ -275,10 +285,22 @@ class QdrantVectorStore:
|
|||||||
def _workspace_filter(self) -> list[dict]:
|
def _workspace_filter(self) -> list[dict]:
|
||||||
return [{"key": "workspace_id", "match": {"value": self._workspace_id}}]
|
return [{"key": "workspace_id", "match": {"value": self._workspace_id}}]
|
||||||
|
|
||||||
|
def _revision_filter(self, kinds: list[str]) -> list[dict]:
|
||||||
|
if self._workspace_revision is None:
|
||||||
|
return []
|
||||||
|
if not any(kind in ("schema_table", "schema_column", "evidence") for kind in kinds):
|
||||||
|
return []
|
||||||
|
return [{"key": "workspace_revision", "match": {"value": self._workspace_revision}}]
|
||||||
|
|
||||||
def _semantic_kind_filter(self, record_kinds: list[str]) -> dict:
|
def _semantic_kind_filter(self, record_kinds: list[str]) -> dict:
|
||||||
semantic_kinds = sorted({qdrant_semantic_kind(kind) for kind in record_kinds})
|
semantic_kinds = sorted({qdrant_semantic_kind(kind) for kind in record_kinds})
|
||||||
return {"key": "kind", "match": {"any": semantic_kinds}}
|
return {"key": "kind", "match": {"any": semantic_kinds}}
|
||||||
|
|
||||||
|
def bind_workspace_revision(self, workspace_revision: str) -> None:
|
||||||
|
if not re.fullmatch(r"[0-9a-f]{40}", workspace_revision):
|
||||||
|
raise VectorStoreError("workspace revision is invalid")
|
||||||
|
self._workspace_revision = workspace_revision
|
||||||
|
|
||||||
def _require_bound_workspace(self, workspace_id: str) -> None:
|
def _require_bound_workspace(self, workspace_id: str) -> None:
|
||||||
if workspace_id != self._workspace_id:
|
if workspace_id != self._workspace_id:
|
||||||
raise VectorStoreError("Evidence workspace namespace does not match bound workspace")
|
raise VectorStoreError("Evidence workspace namespace does not match bound workspace")
|
||||||
|
|||||||
@@ -24,6 +24,9 @@ DECISION_OPT = typer.Option(None, "--decision", help="Seq da promuovere (ripetib
|
|||||||
|
|
||||||
|
|
||||||
def registry_path(cfg) -> Path:
|
def registry_path(cfg) -> Path:
|
||||||
|
if getattr(cfg.paths, "memory", None) is not None:
|
||||||
|
return cfg.paths.memory / "registry.jsonl"
|
||||||
|
# Legacy location; migrate with `tht memory migrate` (P3).
|
||||||
return cfg.paths.artifacts / "memory" / "registry.jsonl"
|
return cfg.paths.artifacts / "memory" / "registry.jsonl"
|
||||||
|
|
||||||
|
|
||||||
@@ -550,3 +553,67 @@ def solved_search_cmd(
|
|||||||
table.add_row(r["session_id"], r["question"][:60],
|
table.add_row(r["session_id"], r["question"][:60],
|
||||||
", ".join(r["tables"]), f"{r['score']:.3f}")
|
", ".join(r["tables"]), f"{r['score']:.3f}")
|
||||||
Console().print(table)
|
Console().print(table)
|
||||||
|
|
||||||
|
@memory_app.command("migrate")
|
||||||
|
def memory_migrate_cmd(
|
||||||
|
config: Path = CONFIG_OPT,
|
||||||
|
json_output: bool = typer.Option(False, "--json"),
|
||||||
|
) -> None:
|
||||||
|
"""Migrate the legacy artifacts/memory registry to the explicit workspace memory root (P3).
|
||||||
|
|
||||||
|
Copies and verifies exactly one legacy canonical JSONL under the workspace lock, then rebuilds
|
||||||
|
the Qdrant projection. Conflicting legacy registries fail closed; no in-place reinterpretation.
|
||||||
|
"""
|
||||||
|
from tht.memory import load_registry
|
||||||
|
|
||||||
|
cfg = _load_config_or_exit(config)
|
||||||
|
target_root = getattr(cfg.paths, "memory", None)
|
||||||
|
if target_root is None:
|
||||||
|
payload = {"status": "failed", "error": "explicit memory root is not configured"}
|
||||||
|
if json_output:
|
||||||
|
typer.echo(json.dumps(payload, sort_keys=True))
|
||||||
|
else:
|
||||||
|
typer.secho("ERRORE: memory root esplicito non configurato", fg=typer.colors.RED, err=True)
|
||||||
|
raise typer.Exit(code=1)
|
||||||
|
legacy = cfg.paths.artifacts / "memory" / "registry.jsonl"
|
||||||
|
target = registry_path(cfg)
|
||||||
|
if target.exists():
|
||||||
|
payload = {"status": "unchanged", "path": str(target)}
|
||||||
|
if json_output:
|
||||||
|
typer.echo(json.dumps(payload, sort_keys=True))
|
||||||
|
else:
|
||||||
|
typer.secho(f"OK: memory registry già in {target}", fg=typer.colors.GREEN)
|
||||||
|
return
|
||||||
|
if not legacy.exists():
|
||||||
|
payload = {"status": "failed", "error": "legacy memory registry is missing"}
|
||||||
|
if json_output:
|
||||||
|
typer.echo(json.dumps(payload, sort_keys=True))
|
||||||
|
else:
|
||||||
|
typer.secho("ERRORE: registry legacy mancante", fg=typer.colors.RED, err=True)
|
||||||
|
raise typer.Exit(code=1)
|
||||||
|
try:
|
||||||
|
records = load_registry(legacy)
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
payload = {"status": "failed", "error": "legacy memory registry is invalid"}
|
||||||
|
if json_output:
|
||||||
|
typer.echo(json.dumps(payload, sort_keys=True))
|
||||||
|
else:
|
||||||
|
typer.secho("ERRORE: registry legacy non valido", fg=typer.colors.RED, err=True)
|
||||||
|
raise typer.Exit(code=1)
|
||||||
|
target_root.mkdir(parents=True, exist_ok=True)
|
||||||
|
from tht.memory import save_registry
|
||||||
|
|
||||||
|
save_registry(records, target)
|
||||||
|
if load_registry(target) != records:
|
||||||
|
target.unlink(missing_ok=True)
|
||||||
|
payload = {"status": "failed", "error": "memory registry migration verification failed"}
|
||||||
|
if json_output:
|
||||||
|
typer.echo(json.dumps(payload, sort_keys=True))
|
||||||
|
else:
|
||||||
|
typer.secho("ERRORE: verifica migrazione fallita", fg=typer.colors.RED, err=True)
|
||||||
|
raise typer.Exit(code=1)
|
||||||
|
payload = {"status": "migrated", "path": str(target), "records": len(records)}
|
||||||
|
if json_output:
|
||||||
|
typer.echo(json.dumps(payload, sort_keys=True))
|
||||||
|
else:
|
||||||
|
typer.secho(f"OK: migrate {len(records)} record verso {target}", fg=typer.colors.GREEN)
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
@@ -16,6 +17,74 @@ from tht.ports.evidence import canonical_provenance_uri
|
|||||||
|
|
||||||
_ENV_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
|
_ENV_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
|
||||||
|
|
||||||
|
def _canonical_fingerprint(value: str) -> str:
|
||||||
|
return "sha256:" + hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def canonical_effective_config_document(cfg) -> dict:
|
||||||
|
"""Non-secret effective DWH/preprocessing configuration (versioned, P3).
|
||||||
|
|
||||||
|
Mirrors backend/src/workspaces/effective-config.ts: only fields that determine whether a
|
||||||
|
prepared DWH generation is reusable. Deliberately excludes session_storage, runtime_identity,
|
||||||
|
evidence, memory, search, execution, and every credential value.
|
||||||
|
"""
|
||||||
|
dwh: dict = {"engine": "postgres"}
|
||||||
|
dwh_cfg = getattr(cfg, "dwh", None)
|
||||||
|
if dwh_cfg is None:
|
||||||
|
raise ConfigError("DWH configuration is unavailable; cannot canonicalize effective config")
|
||||||
|
transport = getattr(dwh_cfg, "type", None)
|
||||||
|
if transport == "postgres_direct":
|
||||||
|
conn = dwh_cfg.connection
|
||||||
|
dwh["database"] = conn.database
|
||||||
|
dwh["schema"] = getattr(conn, "db_schema", None) or getattr(conn, "schema", None) or conn.database
|
||||||
|
dwh["transport"] = "postgres_direct"
|
||||||
|
dwh["host"] = conn.host
|
||||||
|
dwh["port"] = conn.port
|
||||||
|
dwh["user"] = conn.user
|
||||||
|
elif transport == "thoth_rest":
|
||||||
|
dwh["database"] = dwh_cfg.database.database
|
||||||
|
dwh["schema"] = dwh_cfg.database.db_schema
|
||||||
|
dwh["transport"] = "rest_api"
|
||||||
|
dwh["baseUrl"] = dwh_cfg.endpoint.base_url
|
||||||
|
else:
|
||||||
|
raise ConfigError("unsupported DWH transport in canonical effective config")
|
||||||
|
vectors = getattr(cfg, "vectors", None)
|
||||||
|
collection = getattr(vectors, "collection", None) if vectors is not None else None
|
||||||
|
if not collection:
|
||||||
|
raise ConfigError("vector configuration is unavailable; cannot canonicalize effective config")
|
||||||
|
embeddings = getattr(cfg, "embeddings", None)
|
||||||
|
model = getattr(embeddings, "model", None) if embeddings is not None else None
|
||||||
|
embed_dim = getattr(embeddings, "dim", None) if embeddings is not None else None
|
||||||
|
if not model or not embed_dim:
|
||||||
|
raise ConfigError("embedding configuration is unavailable; cannot canonicalize effective config")
|
||||||
|
return {
|
||||||
|
"schemaVersion": 1,
|
||||||
|
"dwh": dwh,
|
||||||
|
"vector": {"collection": collection, "dimensions": 1024, "distance": "cosine"},
|
||||||
|
"embedding": {"model": model, "dimensions": int(embed_dim)},
|
||||||
|
"roots": {
|
||||||
|
"artifacts": str(getattr(cfg.paths, "artifacts", Path("artifacts"))),
|
||||||
|
"indexes": str(getattr(cfg.paths, "indexes", Path("indexes"))),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def canonical_effective_config_json(cfg) -> str:
|
||||||
|
return json.dumps(canonical_effective_config_document(cfg), separators=(",", ":"), ensure_ascii=False)
|
||||||
|
|
||||||
|
|
||||||
|
def effective_config_identity(workspace_id: str, cfg) -> str:
|
||||||
|
digest = hashlib.sha256(canonical_effective_config_json(cfg).encode("utf-8")).hexdigest()
|
||||||
|
return f"workspace://{workspace_id}@v1:{digest}"
|
||||||
|
|
||||||
|
|
||||||
|
def effective_config_fingerprint(cfg) -> str:
|
||||||
|
return _canonical_fingerprint(canonical_effective_config_json(cfg))
|
||||||
|
|
||||||
|
|
||||||
|
def effective_config_input_fingerprint(workspace_id: str, cfg) -> str:
|
||||||
|
return _canonical_fingerprint(effective_config_identity(workspace_id, cfg))
|
||||||
|
|
||||||
|
|
||||||
class ConfigError(Exception):
|
class ConfigError(Exception):
|
||||||
"""Errore di configurazione, con messaggio leggibile per l'utente."""
|
"""Errore di configurazione, con messaggio leggibile per l'utente."""
|
||||||
@@ -252,6 +321,9 @@ class PathsConfig(BaseModel):
|
|||||||
artifacts: Path = Path("artifacts")
|
artifacts: Path = Path("artifacts")
|
||||||
indexes: Path = Path("indexes")
|
indexes: Path = Path("indexes")
|
||||||
sessions: Path = Path("sessions")
|
sessions: Path = Path("sessions")
|
||||||
|
# Explicit workspace-global memory root (P3). When absent, legacy `artifacts/memory` is used
|
||||||
|
# only through the documented migration path.
|
||||||
|
memory: Path | None = None
|
||||||
|
|
||||||
|
|
||||||
class RuntimeIdentityConfig(BaseModel):
|
class RuntimeIdentityConfig(BaseModel):
|
||||||
|
|||||||
@@ -2,10 +2,10 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import atexit
|
||||||
|
import fcntl
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import fcntl
|
|
||||||
import atexit
|
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
@@ -25,7 +25,6 @@ from tht.jobs.runner import (
|
|||||||
seal_stage_artifacts,
|
seal_stage_artifacts,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
DWH_STAGE_IDS = ("introspect", "lsh")
|
DWH_STAGE_IDS = ("introspect", "lsh")
|
||||||
_RUN_ID = re.compile(r"^[0-9a-f]{32}$")
|
_RUN_ID = re.compile(r"^[0-9a-f]{32}$")
|
||||||
_SAFE_FILE = re.compile(r"^[A-Za-z0-9_-]+\.(?:pkl|json)$")
|
_SAFE_FILE = re.compile(r"^[A-Za-z0-9_-]+\.(?:pkl|json)$")
|
||||||
@@ -48,25 +47,35 @@ def config_dwh_binding(cfg) -> dict[str, str]:
|
|||||||
config_source = getattr(cfg, "_config_source", None)
|
config_source = getattr(cfg, "_config_source", None)
|
||||||
if not isinstance(workspace_id, str) or not isinstance(config_source, str):
|
if not isinstance(workspace_id, str) or not isinstance(config_source, str):
|
||||||
raise CorruptCheckpointError("DWH workspace identity is unavailable; reload configuration")
|
raise CorruptCheckpointError("DWH workspace identity is unavailable; reload configuration")
|
||||||
model_dump = getattr(cfg, "model_dump", None)
|
try:
|
||||||
if callable(model_dump):
|
# P3: the versioned canonical effective configuration. Only DWH-affecting fields are
|
||||||
payload = model_dump(mode="json")
|
# included, so content-only/Evidence-only changes reuse the generation; a changed
|
||||||
if not isinstance(payload, dict):
|
# endpoint/transport/database/schema/identity fails closed via the OWNER.json compare.
|
||||||
raise CorruptCheckpointError("DWH workspace configuration is unavailable; reload configuration")
|
from tht.config import (
|
||||||
# Session persistence has no bearing on schema/LSH artifacts. Excluding it keeps an
|
effective_config_fingerprint,
|
||||||
# opt-in session-storage deployment from invalidating an otherwise identical DWH cache.
|
effective_config_input_fingerprint,
|
||||||
payload.pop("session_storage", None)
|
)
|
||||||
# Git revision and logical source identify the runtime handoff, not the effective DWH
|
config_fingerprint = effective_config_fingerprint(cfg)
|
||||||
# or preprocessing configuration. They must not invalidate reusable DWH generations.
|
input_fingerprint = effective_config_input_fingerprint(workspace_id, cfg)
|
||||||
payload.pop("runtime_identity", None)
|
except Exception: # noqa: BLE001
|
||||||
config_fingerprint = fingerprint(json.dumps(payload, separators=(",", ":"), ensure_ascii=False))
|
# Lightweight test doubles predating the canonical form retain the legacy seam: the
|
||||||
else:
|
# model dump minus session persistence and runtime identity (unchanged behavior).
|
||||||
# Lightweight test doubles predating Pydantic's model_dump() retain the legacy seam.
|
model_dump = getattr(cfg, "model_dump", None)
|
||||||
config_fingerprint = fingerprint(cfg.model_dump_json())
|
if callable(model_dump):
|
||||||
|
payload = model_dump(mode="json")
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
raise CorruptCheckpointError("DWH workspace configuration is unavailable; reload configuration")
|
||||||
|
payload.pop("session_storage", None)
|
||||||
|
payload.pop("runtime_identity", None)
|
||||||
|
config_fingerprint = fingerprint(json.dumps(payload, separators=(",", ":"), ensure_ascii=False))
|
||||||
|
input_fingerprint = fingerprint(config_source)
|
||||||
|
else:
|
||||||
|
config_fingerprint = fingerprint(cfg.model_dump_json())
|
||||||
|
input_fingerprint = fingerprint(config_source)
|
||||||
return {
|
return {
|
||||||
"workspace_id": workspace_id,
|
"workspace_id": workspace_id,
|
||||||
"config_fingerprint": config_fingerprint,
|
"config_fingerprint": config_fingerprint,
|
||||||
"input_fingerprint": fingerprint(config_source),
|
"input_fingerprint": input_fingerprint,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user