feat: derive workspace runtime secret requirements

backend/src/workspaces/secret-requirements.ts

@@ -0,0 +1,199 @@
+import { dirname } from "node:path";
+
+import {
+  buildInstallationContract,
+  type InstallationRole,
+  type InstallationSuffix,
+  type InstallationVariable,
+} from "./contracts.js";
+import { resolveRuntimeBindings, type RuntimeBindings } from "./bindings.js";
+import {
+  DWH_TRANSPORTS,
+  validateWorkspaceDescriptor,
+  type DwhTransport,
+  type WorkspaceDescriptor,
+} from "./schema.js";
+import {
+  WorkspaceSecretStore,
+  type WorkspaceSecretMaterialization,
+} from "./secret-store.js";
+
+export type WorkspaceSecretInput = "password" | "textarea";
+
+export interface WorkspaceSecretRequirement {
+  id: string;
+  variable: string;
+  connector: "dwh" | "evidence";
+  label: string;
+  description: string;
+  input: WorkspaceSecretInput;
+  required: boolean;
+}
+
+export interface WorkspaceRuntimeBindingLease {
+  bindings: RuntimeBindings;
+  release(): void;
+}
+
+interface RequirementDefinition {
+  id: string;
+  label: string;
+  description: string;
+  input: WorkspaceSecretInput;
+}
+
+const DEFINITIONS: Readonly<Partial<Record<`${InstallationRole}.${InstallationSuffix}`, RequirementDefinition>>> = {
+  "DWH.PASSWORD_FILE": {
+    id: "dwh.password",
+    label: "Data warehouse password",
+    description: "Password used by the selected data warehouse connection.",
+    input: "password",
+  },
+  "DWH.API_KEY_FILE": {
+    id: "dwh.api_key",
+    label: "Data warehouse API key",
+    description: "API key sent to the configured data warehouse REST endpoint.",
+    input: "password",
+  },
+  "DWH.SSH_PRIVATE_KEY_FILE": {
+    id: "dwh.ssh_private_key",
+    label: "SSH private key",
+    description: "Private key used to open the configured SSH tunnel to the data warehouse.",
+    input: "textarea",
+  },
+  "EVIDENCE.SIGNED_URLS_FILE": {
+    id: "evidence.signed_urls",
+    label: "Evidence signed URLs",
+    description: "Signed URLs that authorize ThothII to retrieve the workspace Evidence sources.",
+    input: "textarea",
+  },
+  "EVIDENCE.ACCESS_KEY_FILE": {
+    id: "evidence.access_key",
+    label: "Evidence access key",
+    description: "Access-key identifier used for the configured S3 Evidence source.",
+    input: "password",
+  },
+  "EVIDENCE.SECRET_KEY_FILE": {
+    id: "evidence.secret_key",
+    label: "Evidence secret key",
+    description: "Secret access key used for the configured S3 Evidence source.",
+    input: "password",
+  },
+  "EVIDENCE.SESSION_TOKEN_FILE": {
+    id: "evidence.session_token",
+    label: "Evidence session token",
+    description: "Optional temporary session token used with the S3 Evidence credentials.",
+    input: "password",
+  },
+};
+
+const REQUIRED_DWH_SECRETS: Readonly<Record<DwhTransport, readonly InstallationSuffix[]>> = {
+  postgres_direct: ["PASSWORD_FILE"],
+  rest_api: ["API_KEY_FILE"],
+  ssh_tunnel: ["PASSWORD_FILE", "SSH_PRIVATE_KEY_FILE"],
+};
+
+function isTransport(value: string | undefined): value is DwhTransport {
+  return value !== undefined && (DWH_TRANSPORTS as readonly string[]).includes(value);
+}
+
+function selectedTransport(
+  descriptor: WorkspaceDescriptor,
+  variables: readonly InstallationVariable[],
+  env: NodeJS.ProcessEnv,
+): DwhTransport {
+  const transportVariable = variables.find(({ role, suffix }) => role === "DWH" && suffix === "TRANSPORT");
+  const value = transportVariable === undefined ? undefined : env[transportVariable.name];
+  return isTransport(value) && descriptor.dwh.supported_transports.includes(value)
+    ? value
+    : descriptor.dwh.supported_transports[0];
+}
+
+function requirementFor(
+  variable: InstallationVariable,
+  required: boolean,
+): WorkspaceSecretRequirement | undefined {
+  const definition = DEFINITIONS[`${variable.role}.${variable.suffix}`];
+  if (definition === undefined) return undefined;
+  return {
+    ...definition,
+    variable: variable.name,
+    connector: variable.role === "DWH" ? "dwh" : "evidence",
+    required,
+  };
+}
+
+/**
+ * Discover the credential fields for the connector and authentication mechanisms selected by
+ * this installation. Paths, hostnames and trust files remain installation configuration rather
+ * than user-entered secrets.
+ */
+export function discoverWorkspaceSecretRequirements(
+  workspace: WorkspaceDescriptor,
+  env: NodeJS.ProcessEnv,
+): WorkspaceSecretRequirement[] {
+  const descriptor = validateWorkspaceDescriptor(workspace);
+  const variables = buildInstallationContract(descriptor).variables;
+  const transport = selectedTransport(descriptor, variables, env);
+  const restHasNoAuthentication = transport === "rest_api" && descriptor.diagnostics?.dwh_rest?.auth === "none";
+  const requiredDwh = new Set(restHasNoAuthentication ? [] : REQUIRED_DWH_SECRETS[transport]);
+
+  const requirements: WorkspaceSecretRequirement[] = [];
+  for (const variable of variables) {
+    if (variable.role === "DWH") {
+      if (variable.transports !== undefined && !variable.transports.includes(transport)) continue;
+      const requirement = requirementFor(variable, requiredDwh.has(variable.suffix));
+      if (requirement !== undefined && requirement.required) requirements.push(requirement);
+      continue;
+    }
+    const requirement = requirementFor(variable, variable.suffix !== "SESSION_TOKEN_FILE");
+    if (requirement !== undefined) requirements.push(requirement);
+  }
+  return requirements;
+}
+
+/**
+ * Materialize only the selected workspace credentials, translate them to the existing file-based
+ * harness contract, and bind cleanup to the returned lease.
+ */
+export function resolveRuntimeBindingsWithWorkspaceSecrets(
+  workspace: WorkspaceDescriptor,
+  env: NodeJS.ProcessEnv,
+  secretRoots: readonly string[],
+  store: WorkspaceSecretStore,
+): WorkspaceRuntimeBindingLease {
+  const descriptor = validateWorkspaceDescriptor(workspace);
+  const requirements = discoverWorkspaceSecretRequirements(descriptor, env);
+  let materialization: WorkspaceSecretMaterialization | undefined;
+  try {
+    materialization = store.materialize(
+      descriptor.workspace.id,
+      requirements.map(({ id }) => id),
+    );
+    const effectiveEnvironment: NodeJS.ProcessEnv = { ...env };
+    const materializedRoots = new Set<string>();
+    for (const requirement of requirements) {
+      const path = materialization.files.get(requirement.id);
+      if (path === undefined) continue;
+      effectiveEnvironment[requirement.variable] = path;
+      materializedRoots.add(dirname(path));
+    }
+    const bindings = resolveRuntimeBindings(
+      descriptor,
+      effectiveEnvironment,
+      [...secretRoots, ...materializedRoots],
+    );
+    let released = false;
+    return {
+      bindings,
+      release: () => {
+        if (released) return;
+        released = true;
+        materialization?.release();
+      },
+    };
+  } catch (error) {
+    materialization?.release();
+    throw error;
+  }
+}

backend/test/workspace-secret-requirements.test.ts

@@ -0,0 +1,138 @@
+import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, expect, test } from "vitest";
+
+import {
+  discoverWorkspaceSecretRequirements,
+  resolveRuntimeBindingsWithWorkspaceSecrets,
+} from "../src/workspaces/secret-requirements.js";
+import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
+import { parseWorkspaceYaml } from "../src/workspaces/schema.js";
+
+const roots: string[] = [];
+
+function workspace(extra = "") {
+  return parseWorkspaceYaml(`workspace:
+  schema_version: 3
+  id: psd-clinical
+  name: Policlinico San Donato
+  language: en
+dwh:
+  engine: postgres
+  database: postgres
+  schema: datawarehouse
+  supported_transports: [postgres_direct, rest_api, ssh_tunnel]
+semantic_index:
+  vector_store: { engine: qdrant, collection: psd-clinical, dimensions: 1024, distance: cosine }
+  embedding: { provider: ollama_internal, model: qwen3-embedding:0.6b, dimensions: 1024 }
+llm_policy: { allowed: [zai/glm-5.2] }
+${extra}`);
+}
+
+function store() {
+  const root = mkdtempSync(join(tmpdir(), "thoth-requirement-vault-"));
+  const runtimeRoot = mkdtempSync(join(tmpdir(), "thoth-requirement-runtime-"));
+  roots.push(root, runtimeRoot);
+  return new WorkspaceSecretStore({ root, runtimeRoot, installationId: "test-installation" });
+}
+
+afterEach(() => {
+  for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
+});
+
+test("requirements follow the selected DWH transport", () => {
+  const descriptor = workspace();
+  const direct = discoverWorkspaceSecretRequirements(descriptor, {
+    THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct",
+  });
+  expect(direct.map(({ id, required }) => ({ id, required }))).toEqual([
+    { id: "dwh.password", required: true },
+  ]);
+
+  const rest = discoverWorkspaceSecretRequirements(descriptor, {
+    THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "rest_api",
+  });
+  expect(rest.map(({ id }) => id)).toEqual(["dwh.api_key"]);
+
+  const ssh = discoverWorkspaceSecretRequirements(descriptor, {
+    THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "ssh_tunnel",
+  });
+  expect(ssh.map(({ id }) => id)).toEqual(["dwh.password", "dwh.ssh_private_key"]);
+});
+
+test("REST without authentication does not request an API key", () => {
+  const descriptor = workspace(`diagnostics:
+  dwh_rest:
+    method: GET
+    path: /health
+    auth: none
+    response: { database: database, schema: schema }
+`);
+  expect(discoverWorkspaceSecretRequirements(descriptor, {
+    THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "rest_api",
+  })).toEqual([]);
+});
+
+test("evidence requirements follow the descriptor authentication mechanism", () => {
+  const signed = workspace(`evidence:
+  source:
+    type: http
+    uris: [https://evidence.example.test/guide.md]
+    authentication: signed_urls_file
+  policy: { max_chunk_chars: 4000, retain_published_generations: 2 }
+`);
+  expect(discoverWorkspaceSecretRequirements(signed, {
+    THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct",
+  }).map(({ id, required }) => ({ id, required }))).toEqual([
+    { id: "dwh.password", required: true },
+    { id: "evidence.signed_urls", required: true },
+  ]);
+
+  const s3 = workspace(`evidence:
+  source: { type: s3, uri: s3://clinical-evidence/published/, credentials: static_files }
+  policy: { max_chunk_chars: 4000, retain_published_generations: 2 }
+`);
+  expect(discoverWorkspaceSecretRequirements(s3, {
+    THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct",
+  }).map(({ id, required }) => ({ id, required }))).toEqual([
+    { id: "dwh.password", required: true },
+    { id: "evidence.access_key", required: true },
+    { id: "evidence.secret_key", required: true },
+    { id: "evidence.session_token", required: false },
+  ]);
+});
+
+test("vault values are mapped to temporary file bindings and released", () => {
+  const descriptor = workspace();
+  const vault = store();
+  vault.put("psd-clinical", "dwh.password", "runtime-password");
+  const environment = {
+    THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct",
+    THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.internal",
+    THT_WS_PSD_CLINICAL_DWH_PORT: "5432",
+    THT_WS_PSD_CLINICAL_DWH_USER: "reader",
+  };
+
+  const lease = resolveRuntimeBindingsWithWorkspaceSecrets(descriptor, environment, [], vault);
+  expect(lease.bindings.dwh.missing).toEqual([]);
+  const secretPath = lease.bindings.dwh.values.THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE!;
+  expect(readFileSync(secretPath, "utf8")).toBe("runtime-password");
+  lease.release();
+  expect(() => readFileSync(secretPath, "utf8")).toThrow();
+});
+
+test("missing vault values remain missing and materialization never mutates the source environment", () => {
+  const descriptor = workspace();
+  const vault = store();
+  const environment = {
+    THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "postgres_direct",
+    THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.internal",
+    THT_WS_PSD_CLINICAL_DWH_PORT: "5432",
+    THT_WS_PSD_CLINICAL_DWH_USER: "reader",
+  };
+  const lease = resolveRuntimeBindingsWithWorkspaceSecrets(descriptor, environment, [], vault);
+  expect(lease.bindings.dwh.missing).toContain("THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE");
+  expect(environment).not.toHaveProperty("THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE");
+  lease.release();
+});