feat: derive workspace runtime secret requirements
This commit is contained in:
@@ -0,0 +1,199 @@
|
||||
import { dirname } from "node:path";
|
||||
|
||||
import {
|
||||
buildInstallationContract,
|
||||
type InstallationRole,
|
||||
type InstallationSuffix,
|
||||
type InstallationVariable,
|
||||
} from "./contracts.js";
|
||||
import { resolveRuntimeBindings, type RuntimeBindings } from "./bindings.js";
|
||||
import {
|
||||
DWH_TRANSPORTS,
|
||||
validateWorkspaceDescriptor,
|
||||
type DwhTransport,
|
||||
type WorkspaceDescriptor,
|
||||
} from "./schema.js";
|
||||
import {
|
||||
WorkspaceSecretStore,
|
||||
type WorkspaceSecretMaterialization,
|
||||
} from "./secret-store.js";
|
||||
|
||||
export type WorkspaceSecretInput = "password" | "textarea";
|
||||
|
||||
export interface WorkspaceSecretRequirement {
|
||||
id: string;
|
||||
variable: string;
|
||||
connector: "dwh" | "evidence";
|
||||
label: string;
|
||||
description: string;
|
||||
input: WorkspaceSecretInput;
|
||||
required: boolean;
|
||||
}
|
||||
|
||||
export interface WorkspaceRuntimeBindingLease {
|
||||
bindings: RuntimeBindings;
|
||||
release(): void;
|
||||
}
|
||||
|
||||
interface RequirementDefinition {
|
||||
id: string;
|
||||
label: string;
|
||||
description: string;
|
||||
input: WorkspaceSecretInput;
|
||||
}
|
||||
|
||||
const DEFINITIONS: Readonly<Partial<Record<`${InstallationRole}.${InstallationSuffix}`, RequirementDefinition>>> = {
|
||||
"DWH.PASSWORD_FILE": {
|
||||
id: "dwh.password",
|
||||
label: "Data warehouse password",
|
||||
description: "Password used by the selected data warehouse connection.",
|
||||
input: "password",
|
||||
},
|
||||
"DWH.API_KEY_FILE": {
|
||||
id: "dwh.api_key",
|
||||
label: "Data warehouse API key",
|
||||
description: "API key sent to the configured data warehouse REST endpoint.",
|
||||
input: "password",
|
||||
},
|
||||
"DWH.SSH_PRIVATE_KEY_FILE": {
|
||||
id: "dwh.ssh_private_key",
|
||||
label: "SSH private key",
|
||||
description: "Private key used to open the configured SSH tunnel to the data warehouse.",
|
||||
input: "textarea",
|
||||
},
|
||||
"EVIDENCE.SIGNED_URLS_FILE": {
|
||||
id: "evidence.signed_urls",
|
||||
label: "Evidence signed URLs",
|
||||
description: "Signed URLs that authorize ThothII to retrieve the workspace Evidence sources.",
|
||||
input: "textarea",
|
||||
},
|
||||
"EVIDENCE.ACCESS_KEY_FILE": {
|
||||
id: "evidence.access_key",
|
||||
label: "Evidence access key",
|
||||
description: "Access-key identifier used for the configured S3 Evidence source.",
|
||||
input: "password",
|
||||
},
|
||||
"EVIDENCE.SECRET_KEY_FILE": {
|
||||
id: "evidence.secret_key",
|
||||
label: "Evidence secret key",
|
||||
description: "Secret access key used for the configured S3 Evidence source.",
|
||||
input: "password",
|
||||
},
|
||||
"EVIDENCE.SESSION_TOKEN_FILE": {
|
||||
id: "evidence.session_token",
|
||||
label: "Evidence session token",
|
||||
description: "Optional temporary session token used with the S3 Evidence credentials.",
|
||||
input: "password",
|
||||
},
|
||||
};
|
||||
|
||||
const REQUIRED_DWH_SECRETS: Readonly<Record<DwhTransport, readonly InstallationSuffix[]>> = {
|
||||
postgres_direct: ["PASSWORD_FILE"],
|
||||
rest_api: ["API_KEY_FILE"],
|
||||
ssh_tunnel: ["PASSWORD_FILE", "SSH_PRIVATE_KEY_FILE"],
|
||||
};
|
||||
|
||||
function isTransport(value: string | undefined): value is DwhTransport {
|
||||
return value !== undefined && (DWH_TRANSPORTS as readonly string[]).includes(value);
|
||||
}
|
||||
|
||||
function selectedTransport(
|
||||
descriptor: WorkspaceDescriptor,
|
||||
variables: readonly InstallationVariable[],
|
||||
env: NodeJS.ProcessEnv,
|
||||
): DwhTransport {
|
||||
const transportVariable = variables.find(({ role, suffix }) => role === "DWH" && suffix === "TRANSPORT");
|
||||
const value = transportVariable === undefined ? undefined : env[transportVariable.name];
|
||||
return isTransport(value) && descriptor.dwh.supported_transports.includes(value)
|
||||
? value
|
||||
: descriptor.dwh.supported_transports[0];
|
||||
}
|
||||
|
||||
function requirementFor(
|
||||
variable: InstallationVariable,
|
||||
required: boolean,
|
||||
): WorkspaceSecretRequirement | undefined {
|
||||
const definition = DEFINITIONS[`${variable.role}.${variable.suffix}`];
|
||||
if (definition === undefined) return undefined;
|
||||
return {
|
||||
...definition,
|
||||
variable: variable.name,
|
||||
connector: variable.role === "DWH" ? "dwh" : "evidence",
|
||||
required,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Discover the credential fields for the connector and authentication mechanisms selected by
|
||||
* this installation. Paths, hostnames and trust files remain installation configuration rather
|
||||
* than user-entered secrets.
|
||||
*/
|
||||
export function discoverWorkspaceSecretRequirements(
|
||||
workspace: WorkspaceDescriptor,
|
||||
env: NodeJS.ProcessEnv,
|
||||
): WorkspaceSecretRequirement[] {
|
||||
const descriptor = validateWorkspaceDescriptor(workspace);
|
||||
const variables = buildInstallationContract(descriptor).variables;
|
||||
const transport = selectedTransport(descriptor, variables, env);
|
||||
const restHasNoAuthentication = transport === "rest_api" && descriptor.diagnostics?.dwh_rest?.auth === "none";
|
||||
const requiredDwh = new Set(restHasNoAuthentication ? [] : REQUIRED_DWH_SECRETS[transport]);
|
||||
|
||||
const requirements: WorkspaceSecretRequirement[] = [];
|
||||
for (const variable of variables) {
|
||||
if (variable.role === "DWH") {
|
||||
if (variable.transports !== undefined && !variable.transports.includes(transport)) continue;
|
||||
const requirement = requirementFor(variable, requiredDwh.has(variable.suffix));
|
||||
if (requirement !== undefined && requirement.required) requirements.push(requirement);
|
||||
continue;
|
||||
}
|
||||
const requirement = requirementFor(variable, variable.suffix !== "SESSION_TOKEN_FILE");
|
||||
if (requirement !== undefined) requirements.push(requirement);
|
||||
}
|
||||
return requirements;
|
||||
}
|
||||
|
||||
/**
|
||||
* Materialize only the selected workspace credentials, translate them to the existing file-based
|
||||
* harness contract, and bind cleanup to the returned lease.
|
||||
*/
|
||||
export function resolveRuntimeBindingsWithWorkspaceSecrets(
|
||||
workspace: WorkspaceDescriptor,
|
||||
env: NodeJS.ProcessEnv,
|
||||
secretRoots: readonly string[],
|
||||
store: WorkspaceSecretStore,
|
||||
): WorkspaceRuntimeBindingLease {
|
||||
const descriptor = validateWorkspaceDescriptor(workspace);
|
||||
const requirements = discoverWorkspaceSecretRequirements(descriptor, env);
|
||||
let materialization: WorkspaceSecretMaterialization | undefined;
|
||||
try {
|
||||
materialization = store.materialize(
|
||||
descriptor.workspace.id,
|
||||
requirements.map(({ id }) => id),
|
||||
);
|
||||
const effectiveEnvironment: NodeJS.ProcessEnv = { ...env };
|
||||
const materializedRoots = new Set<string>();
|
||||
for (const requirement of requirements) {
|
||||
const path = materialization.files.get(requirement.id);
|
||||
if (path === undefined) continue;
|
||||
effectiveEnvironment[requirement.variable] = path;
|
||||
materializedRoots.add(dirname(path));
|
||||
}
|
||||
const bindings = resolveRuntimeBindings(
|
||||
descriptor,
|
||||
effectiveEnvironment,
|
||||
[...secretRoots, ...materializedRoots],
|
||||
);
|
||||
let released = false;
|
||||
return {
|
||||
bindings,
|
||||
release: () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
materialization?.release();
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
materialization?.release();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, test } from "vitest";
|
||||
|
||||
import {
|
||||
discoverWorkspaceSecretRequirements,
|
||||
resolveRuntimeBindingsWithWorkspaceSecrets,
|
||||
} from "../src/workspaces/secret-requirements.js";
|
||||
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
|
||||
function workspace(extra = "") {
|
||||
return parseWorkspaceYaml(`workspace:
|
||||
schema_version: 3
|
||||
id: psd-clinical
|
||||
name: Policlinico San Donato
|
||||
language: en
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: postgres
|
||||
schema: datawarehouse
|
||||
supported_transports: [postgres_direct, rest_api, ssh_tunnel]
|
||||
semantic_index:
|
||||
vector_store: { engine: qdrant, collection: psd-clinical, dimensions: 1024, distance: cosine }
|
||||
embedding: { provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024 }
|
||||
llm_policy: { allowed: [zai/glm-5.2] }
|
||||
${extra}`);
|
||||
}
|
||||
|
||||
function store() {
|
||||
const root = mkdtempSync(join(tmpdir(), "thoth-requirement-vault-"));
|
||||
const runtimeRoot = mkdtempSync(join(tmpdir(), "thoth-requirement-runtime-"));
|
||||
roots.push(root, runtimeRoot);
|
||||
return new WorkspaceSecretStore({ root, runtimeRoot, installationId: "test-installation" });
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("requirements follow the selected DWH transport", () => {
|
||||
const descriptor = workspace();
|
||||
const direct = discoverWorkspaceSecretRequirements(descriptor, {
|
||||
THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct",
|
||||
});
|
||||
expect(direct.map(({ id, required }) => ({ id, required }))).toEqual([
|
||||
{ id: "dwh.password", required: true },
|
||||
]);
|
||||
|
||||
const rest = discoverWorkspaceSecretRequirements(descriptor, {
|
||||
THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "rest_api",
|
||||
});
|
||||
expect(rest.map(({ id }) => id)).toEqual(["dwh.api_key"]);
|
||||
|
||||
const ssh = discoverWorkspaceSecretRequirements(descriptor, {
|
||||
THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "ssh_tunnel",
|
||||
});
|
||||
expect(ssh.map(({ id }) => id)).toEqual(["dwh.password", "dwh.ssh_private_key"]);
|
||||
});
|
||||
|
||||
test("REST without authentication does not request an API key", () => {
|
||||
const descriptor = workspace(`diagnostics:
|
||||
dwh_rest:
|
||||
method: GET
|
||||
path: /health
|
||||
auth: none
|
||||
response: { database: database, schema: schema }
|
||||
`);
|
||||
expect(discoverWorkspaceSecretRequirements(descriptor, {
|
||||
THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "rest_api",
|
||||
})).toEqual([]);
|
||||
});
|
||||
|
||||
test("evidence requirements follow the descriptor authentication mechanism", () => {
|
||||
const signed = workspace(`evidence:
|
||||
source:
|
||||
type: http
|
||||
uris: [https://evidence.example.test/guide.md]
|
||||
authentication: signed_urls_file
|
||||
policy: { max_chunk_chars: 4000, retain_published_generations: 2 }
|
||||
`);
|
||||
expect(discoverWorkspaceSecretRequirements(signed, {
|
||||
THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct",
|
||||
}).map(({ id, required }) => ({ id, required }))).toEqual([
|
||||
{ id: "dwh.password", required: true },
|
||||
{ id: "evidence.signed_urls", required: true },
|
||||
]);
|
||||
|
||||
const s3 = workspace(`evidence:
|
||||
source: { type: s3, uri: s3://clinical-evidence/published/, credentials: static_files }
|
||||
policy: { max_chunk_chars: 4000, retain_published_generations: 2 }
|
||||
`);
|
||||
expect(discoverWorkspaceSecretRequirements(s3, {
|
||||
THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct",
|
||||
}).map(({ id, required }) => ({ id, required }))).toEqual([
|
||||
{ id: "dwh.password", required: true },
|
||||
{ id: "evidence.access_key", required: true },
|
||||
{ id: "evidence.secret_key", required: true },
|
||||
{ id: "evidence.session_token", required: false },
|
||||
]);
|
||||
});
|
||||
|
||||
test("vault values are mapped to temporary file bindings and released", () => {
|
||||
const descriptor = workspace();
|
||||
const vault = store();
|
||||
vault.put("psd-clinical", "dwh.password", "runtime-password");
|
||||
const environment = {
|
||||
THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct",
|
||||
THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.internal",
|
||||
THT_WS_PSD_CLINICAL_DWH_PORT: "5432",
|
||||
THT_WS_PSD_CLINICAL_DWH_USER: "reader",
|
||||
};
|
||||
|
||||
const lease = resolveRuntimeBindingsWithWorkspaceSecrets(descriptor, environment, [], vault);
|
||||
expect(lease.bindings.dwh.missing).toEqual([]);
|
||||
const secretPath = lease.bindings.dwh.values.THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE!;
|
||||
expect(readFileSync(secretPath, "utf8")).toBe("runtime-password");
|
||||
lease.release();
|
||||
expect(() => readFileSync(secretPath, "utf8")).toThrow();
|
||||
});
|
||||
|
||||
test("missing vault values remain missing and materialization never mutates the source environment", () => {
|
||||
const descriptor = workspace();
|
||||
const vault = store();
|
||||
const environment = {
|
||||
THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct",
|
||||
THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.internal",
|
||||
THT_WS_PSD_CLINICAL_DWH_PORT: "5432",
|
||||
THT_WS_PSD_CLINICAL_DWH_USER: "reader",
|
||||
};
|
||||
const lease = resolveRuntimeBindingsWithWorkspaceSecrets(descriptor, environment, [], vault);
|
||||
expect(lease.bindings.dwh.missing).toContain("THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE");
|
||||
expect(environment).not.toHaveProperty("THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE");
|
||||
lease.release();
|
||||
});
|
||||
Reference in New Issue
Block a user