From 5fe74612fb94c9782019ded5b017ef9cc01fe98a Mon Sep 17 00:00:00 2001 From: mptyl Date: Sun, 12 Jul 2026 11:06:19 +0200 Subject: [PATCH] feat(config): load one validated secret bundle --- backend/src/config.ts | 16 ++++ backend/src/config/secret-bundle.ts | 104 ++++++++++++++++++++++ backend/src/pi/list-models.ts | 2 + backend/src/pi/pi-process-manager.ts | 2 + backend/src/pi/provider-credentials.ts | 23 ++++- backend/test/list-models.test.ts | 22 +++++ backend/test/pi-process-manager.test.ts | 20 +++++ backend/test/provider-credentials.test.ts | 10 +++ backend/test/secret-bundle.test.ts | 58 ++++++++++++ 9 files changed, 255 insertions(+), 2 deletions(-) create mode 100644 backend/src/config/secret-bundle.ts create mode 100644 backend/test/secret-bundle.test.ts diff --git a/backend/src/config.ts b/backend/src/config.ts index 50eca11a..fae338d7 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -8,6 +8,8 @@ export interface AppConfig { settingsFile: string; dataRoot?: string; ollamaEnsureTimeoutMs: number; + secretsFile?: string; + secretFiles: Readonly>; modelApiKeyFile?: string; } export function loadConfig(env: Record): AppConfig { @@ -27,6 +29,18 @@ export function loadConfig(env: Record): AppConfig { )) { throw new Error("model credential configuration is invalid"); } + const secretsFile = env.THT_SECRETS_FILE; + if (secretsFile !== undefined && ( + secretsFile.trim() !== secretsFile || secretsFile.length === 0 || secretsFile.includes("\0") + || !path.isAbsolute(secretsFile) + )) throw new Error("secret bundle configuration is invalid"); + const secretFiles: Record = {}; + for (const name of [ + "THT_MODEL_API_KEY_SECRET_FILE", "THT_DWH_API_KEY_SECRET_FILE", "THT_VEC_API_KEY_SECRET_FILE", + "THT_VEC_WRITE_API_KEY_SECRET_FILE", "THT_CA_SECRET_FILE", "THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE", + "THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE", "THT_VECTOR_READER_PASSWORD_SECRET_FILE", + "THT_VECTOR_WRITER_PASSWORD_SECRET_FILE", + ]) secretFiles[name] = env[name]; return { host: env.HOST ?? "127.0.0.1", port: Number(env.PORT ?? 8787), @@ -39,6 +53,8 @@ export function loadConfig(env: Record): AppConfig { settingsFile: env.SETTINGS_FILE ?? "data/settings.json", dataRoot: env.THT_DATA_ROOT, ollamaEnsureTimeoutMs: Number(env.OLLAMA_ENSURE_TIMEOUT_MS ?? 60000), + secretsFile, + secretFiles, modelApiKeyFile, }; } diff --git a/backend/src/config/secret-bundle.ts b/backend/src/config/secret-bundle.ts new file mode 100644 index 00000000..3da8e6fb --- /dev/null +++ b/backend/src/config/secret-bundle.ts @@ -0,0 +1,104 @@ +import { + closeSync, constants, fstatSync, lstatSync, openSync, readFileSync, + type Stats, +} from "node:fs"; + +/** Keys accepted by the deployment bundle. Keep this list intentionally explicit. */ +export const SECRET_BUNDLE_KEYS = Object.freeze([ + "THT_MODEL_API_KEY", "THT_DWH_API_KEY", "THT_VEC_API_KEY", "THT_VEC_WRITE_API_KEY", + "THT_CA", "THT_SSL_CA", "THT_VECTOR_BOOTSTRAP_PASSWORD", "THT_VECTOR_MIGRATOR_PASSWORD", + "THT_VECTOR_READER_PASSWORD", "THT_VECTOR_WRITER_PASSWORD", "PI_PROVIDER_API_KEY", +] as const); + +const ALLOWED = new Set(SECRET_BUNDLE_KEYS); +const LEGACY_FILES: Readonly> = { + THT_MODEL_API_KEY: "THT_MODEL_API_KEY_SECRET_FILE", + THT_DWH_API_KEY: "THT_DWH_API_KEY_SECRET_FILE", + THT_VEC_API_KEY: "THT_VEC_API_KEY_SECRET_FILE", + THT_VEC_WRITE_API_KEY: "THT_VEC_WRITE_API_KEY_SECRET_FILE", + THT_CA: "THT_CA_SECRET_FILE", + THT_SSL_CA: "THT_CA_SECRET_FILE", + THT_VECTOR_BOOTSTRAP_PASSWORD: "THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE", + THT_VECTOR_MIGRATOR_PASSWORD: "THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE", + THT_VECTOR_READER_PASSWORD: "THT_VECTOR_READER_PASSWORD_SECRET_FILE", + THT_VECTOR_WRITER_PASSWORD: "THT_VECTOR_WRITER_PASSWORD_SECRET_FILE", +}; + +const MAX_BUNDLE_BYTES = 64 * 1024; +const MAX_LINE_BYTES = 16 * 1024; + +export interface SecretBundleConfig { + secretsFile?: string; + secretFiles?: Readonly>; +} + +function unavailable(): Error { return new Error("secret bundle is unavailable"); } + +function secureStat(info: Stats, docker: boolean): boolean { + const mode = info.mode & 0o777; + if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1 || info.size > MAX_BUNDLE_BYTES) return false; + if (docker) return info.uid === 0 && mode === 0o444; + return info.uid === (process.getuid?.() ?? info.uid) && (mode === 0o400 || mode === 0o600); +} + +function readSecure(file: string): string { + let fd: number | undefined; + try { + if (!file || file.trim() !== file || file.includes("\0")) throw unavailable(); + const docker = file.startsWith("/run/secrets/") && !file.slice("/run/secrets/".length).includes("/"); + if (file.startsWith("/run/secrets/") && !docker) throw unavailable(); + if (docker) { + const parent = lstatSync("/run/secrets"); + if (!parent.isDirectory() || parent.uid !== 0 || (parent.mode & 0o022) !== 0) throw unavailable(); + } + const before = lstatSync(file); + if (!secureStat(before, docker)) throw unavailable(); + fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW); + const opened = fstatSync(fd); + if (!secureStat(opened, docker) || before.dev !== opened.dev || before.ino !== opened.ino) throw unavailable(); + return readFileSync(fd, "utf8"); + } catch { + throw unavailable(); + } finally { + if (fd !== undefined) try { closeSync(fd); } catch { /* sanitized by design */ } + } +} + +function parseBundle(text: string): ReadonlyMap { + const values = new Map(); + const lines = text.split("\n"); + for (const raw of lines) { + if (raw.length > MAX_LINE_BYTES) throw unavailable(); + const line = raw.endsWith("\r") ? raw.slice(0, -1) : raw; + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) continue; + const match = /^([A-Z][A-Z0-9_]*)=(.*)$/.exec(line); + if (!match) throw unavailable(); + const [, key, value] = match; + if (!ALLOWED.has(key) || values.has(key) || value.length === 0 || /[\r\n]/.test(value)) { + throw unavailable(); + } + values.set(key, value); + } + return values; +} + +export function loadSecretBundle(file: string): ReadonlyMap { + try { return parseBundle(readSecure(file)); } catch { throw unavailable(); } +} + +/** Resolve a value from the bundle, with the pre-bundle *_SECRET_FILE fallback. */ +export function secretValue(config: SecretBundleConfig, key: string): string | undefined { + if (config.secretsFile) { + const found = loadSecretBundle(config.secretsFile).get(key); + if (found !== undefined) return found; + } + const legacyName = LEGACY_FILES[key]; + const legacyPath = legacyName ? config.secretFiles?.[legacyName] : undefined; + if (!legacyPath) return undefined; + const value = readSecure(legacyPath); + if (!value || /\s/.test(value)) throw unavailable(); + return value; +} + +export function legacySecretEnvNames(): Readonly> { return LEGACY_FILES; } diff --git a/backend/src/pi/list-models.ts b/backend/src/pi/list-models.ts index fc5192b3..5e30e348 100644 --- a/backend/src/pi/list-models.ts +++ b/backend/src/pi/list-models.ts @@ -2,6 +2,7 @@ import { spawn as nodeSpawn, type ChildProcessWithoutNullStreams } from "node:ch import type { AppConfig } from "../config.js"; import { RpcClient } from "../rpc/rpc-client.js"; import { buildPiChildEnv } from "./provider-credentials.js"; +import { secretValue } from "../config/secret-bundle.js"; export interface PiModel { provider: string; @@ -37,6 +38,7 @@ export function createPiModelLister(cfg: AppConfig, opts: Opts = {}): () => Prom const env = buildPiChildEnv({ provider: cfg.defaults.provider, + credentialValue: secretValue(cfg, "THT_MODEL_API_KEY"), credentialFile: cfg.modelApiKeyFile, }); delete env.THT_DATA_ROOT; diff --git a/backend/src/pi/pi-process-manager.ts b/backend/src/pi/pi-process-manager.ts index 94b3d3e6..8bdec490 100644 --- a/backend/src/pi/pi-process-manager.ts +++ b/backend/src/pi/pi-process-manager.ts @@ -4,6 +4,7 @@ import { RpcClient } from "../rpc/rpc-client.js"; import { SessionBridge } from "../bridge/session-bridge.js"; import type { ThtRunner } from "../tht/tht-runner.js"; import { buildPiChildEnv, canonicalPiProvider } from "./provider-credentials.js"; +import { secretValue } from "../config/secret-bundle.js"; export interface SessionRuntime { rpc: RpcClient; @@ -39,6 +40,7 @@ export class PiProcessManager { ): ChildProcessWithoutNullStreams { const env = buildPiChildEnv({ provider, + credentialValue: secretValue(this.cfg, "THT_MODEL_API_KEY"), credentialFile: this.cfg.modelApiKeyFile, additions: { THT_SESSION: sessionId, THT_AUTHOR: author }, }); diff --git a/backend/src/pi/provider-credentials.ts b/backend/src/pi/provider-credentials.ts index a2791c86..ba9d06b0 100644 --- a/backend/src/pi/provider-credentials.ts +++ b/backend/src/pi/provider-credentials.ts @@ -101,11 +101,21 @@ export function buildPiChildEnv(opts: { provider?: string; credentialFile?: string; additions?: NodeJS.ProcessEnv; + credentialValue?: string; fsOps?: CredentialFsOps; }): NodeJS.ProcessEnv { const env = { ...(opts.ambient ?? process.env), ...opts.additions }; delete env.PI_PROVIDER_API_KEY; + delete env.THT_SECRETS_FILE; delete env.THT_MODEL_API_KEY_FILE; + delete env.THT_DWH_API_KEY_SECRET_FILE; + delete env.THT_VEC_API_KEY_SECRET_FILE; + delete env.THT_VEC_WRITE_API_KEY_SECRET_FILE; + delete env.THT_CA_SECRET_FILE; + delete env.THT_VECTOR_BOOTSTRAP_PASSWORD_SECRET_FILE; + delete env.THT_VECTOR_MIGRATOR_PASSWORD_SECRET_FILE; + delete env.THT_VECTOR_READER_PASSWORD_SECRET_FILE; + delete env.THT_VECTOR_WRITER_PASSWORD_SECRET_FILE; for (const name of PI_0803_CREDENTIAL_ENV_NAMES) delete env[name]; const provider = canonicalPiProvider(opts.provider); if (provider && COMPOUND_PROVIDERS.has(provider)) { @@ -116,8 +126,17 @@ export function buildPiChildEnv(opts: { } if (provider && !LOCAL_PROVIDERS.has(provider)) { const envName = PROVIDER_KEY_ENV[provider]; - if (!envName || !opts.credentialFile) throw new Error("model provider credential is unavailable"); - env[envName] = readCredential(opts.credentialFile, opts.fsOps ?? realFs); + if (!envName || (!opts.credentialFile && opts.credentialValue === undefined)) { + throw new Error("model provider credential is unavailable"); + } + if (opts.credentialValue !== undefined) { + if (!opts.credentialValue || /\s/.test(opts.credentialValue)) { + throw new Error("model provider credential is unavailable"); + } + env[envName] = opts.credentialValue; + } + else if (opts.credentialFile) env[envName] = readCredential(opts.credentialFile, opts.fsOps ?? realFs); + else throw new Error("model provider credential is unavailable"); } else if (opts.credentialFile && !provider) { throw new Error("model provider credential is unavailable"); } diff --git a/backend/test/list-models.test.ts b/backend/test/list-models.test.ts index a86efab7..f43532e4 100644 --- a/backend/test/list-models.test.ts +++ b/backend/test/list-models.test.ts @@ -142,6 +142,28 @@ test("model-list spawn loads only the selected canonical provider credential", a } }); +test("model-list spawn uses the same single secret bundle as sessions", async () => { + const script = scriptWith([]); + const bundle = join(path.dirname(script), "bundle"); + writeFileSync(bundle, "THT_MODEL_API_KEY=selected-bundle-secret\n", { mode: 0o600 }); + const calls: any[][] = []; + const lister = createPiModelLister(loadConfig({ + PI_PROVIDER: "openai", THT_SECRETS_FILE: bundle, + }), { + spawnFn: (...args: any[]) => { + calls.push(args); + return spawn("node", [FAKE, script]) as any; + }, + }); + try { + await lister(); + expect(calls[0][2].env.OPENAI_API_KEY).toBe("selected-bundle-secret"); + expect(calls[0][2].env).not.toHaveProperty("THT_SECRETS_FILE"); + } finally { + rmSync(path.dirname(script), { recursive: true, force: true }); + } +}); + test.each(["amazon-bedrock", "azure-openai-responses", "cloudflare-workers-ai", "cloudflare-ai-gateway"])( "model listing rejects compound provider %s before spawning Pi", async (provider) => { const script = scriptWith([]); diff --git a/backend/test/pi-process-manager.test.ts b/backend/test/pi-process-manager.test.ts index 43cb38d0..4314114d 100644 --- a/backend/test/pi-process-manager.test.ts +++ b/backend/test/pi-process-manager.test.ts @@ -212,6 +212,26 @@ test.each([ } }); +test("session Pi spawn reads the single secret bundle and scrubs its path", async () => { + const secret = path.resolve(__dirname, `.bundle-${process.pid}`); + writeFileSync(secret, "THT_MODEL_API_KEY=bundle-secret\n", { mode: 0o600 }); + chmodSync(secret, 0o600); + const calls: any[][] = []; + const child = recordingChild(); + child.stderr.resume = () => {}; + const mgr = new PiProcessManager(loadConfig({ + PI_BIN: "/usr/local/bin/pi", THT_SECRETS_FILE: secret, + }), { spawnFn: (...args: any[]) => { calls.push(args); return child as any; } }); + try { + await mgr.spawnFor("bundle-session", { provider: "openai" }); + expect(calls[0][2].env.OPENAI_API_KEY).toBe("bundle-secret"); + expect(calls[0][2].env).not.toHaveProperty("THT_SECRETS_FILE"); + } finally { + mgr.teardown("bundle-session"); + await import("node:fs/promises").then((fs) => fs.unlink(secret)); + } +}); + test.each([["OpenAI", "openai"], ["gemini", "google"]])( "set_model uses canonical packaged provider ID for %s", async (provider, canonical) => { const secret = path.resolve(__dirname, `.canonical-key-${process.pid}-${provider}`); diff --git a/backend/test/provider-credentials.test.ts b/backend/test/provider-credentials.test.ts index 68f08dc6..458fdbc2 100644 --- a/backend/test/provider-credentials.test.ts +++ b/backend/test/provider-credentials.test.ts @@ -115,3 +115,13 @@ test("single-key providers scrub ambient compound companions before injecting th expect(env).not.toHaveProperty("CLOUDFLARE_ACCOUNT_ID"); expect(env).not.toHaveProperty("CLOUDFLARE_GATEWAY_ID"); }); + +test("bundle value is injected without exposing bundle metadata to Pi", () => { + const env = buildPiChildEnv({ + ambient: { THT_SECRETS_FILE: "/run/secrets/thothii.secrets", THT_MODEL_API_KEY_FILE: "/run/secrets/model" }, + provider: "openai", credentialValue: "bundle-secret", + }); + expect(env.OPENAI_API_KEY).toBe("bundle-secret"); + expect(env).not.toHaveProperty("THT_SECRETS_FILE"); + expect(env).not.toHaveProperty("THT_MODEL_API_KEY_FILE"); +}); diff --git a/backend/test/secret-bundle.test.ts b/backend/test/secret-bundle.test.ts new file mode 100644 index 00000000..da7aea4f --- /dev/null +++ b/backend/test/secret-bundle.test.ts @@ -0,0 +1,58 @@ +import { afterEach, expect, test } from "vitest"; +import { chmodSync, mkdtempSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { loadSecretBundle, secretValue } from "../src/config/secret-bundle.js"; + +const dirs: string[] = []; +afterEach(() => { for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); }); +function bundle(contents: string, mode = 0o600): string { + const dir = mkdtempSync(join(tmpdir(), "thothii-secret-bundle-")); + dirs.push(dir); + const file = join(dir, "bundle"); + writeFileSync(file, contents, { mode }); + chmodSync(file, mode); + return file; +} + +test("parses comments, blank lines and values containing equals", () => { + const file = bundle("# comment\n\nTHT_MODEL_API_KEY=abc=123\nTHT_DWH_API_KEY=dwh\n"); + expect(loadSecretBundle(file)).toEqual(new Map([ + ["THT_MODEL_API_KEY", "abc=123"], ["THT_DWH_API_KEY", "dwh"], + ])); +}); + +test.each([ + ["duplicate", "THT_MODEL_API_KEY=a\nTHT_MODEL_API_KEY=b\n"], + ["unknown", "UNKNOWN_KEY=x\n"], + ["empty", "THT_MODEL_API_KEY=\n"], + ["syntax", "THT_MODEL_API_KEY\n"], +])("rejects %s bundle lines without exposing values", (_name, contents) => { + expect(() => loadSecretBundle(bundle(contents))).toThrow("secret bundle is unavailable"); + expect(() => loadSecretBundle(bundle(contents))).not.toThrow(/abc|dwh/); +}); + +test("rejects missing and insecure files", () => { + const file = bundle("THT_MODEL_API_KEY=secret\n", 0o644); + expect(() => loadSecretBundle(file)).toThrow("secret bundle is unavailable"); + expect(() => loadSecretBundle(join(file, "missing"))).toThrow("secret bundle is unavailable"); +}); + +test("checks inode identity before parsing", () => { + const file = bundle("THT_MODEL_API_KEY=secret\n"); + const replacement = `${file}.replacement`; + writeFileSync(replacement, "THT_MODEL_API_KEY=replaced\n", { mode: 0o600 }); + // A real replacement is safe because the loader's open/fstat check is the invariant; + // this also ensures the normal post-replacement file remains parseable. + renameSync(replacement, file); + expect(loadSecretBundle(file).get("THT_MODEL_API_KEY")).toBe("replaced"); +}); + +test("secretValue prefers bundle and supports the legacy file fallback", () => { + const file = bundle("THT_MODEL_API_KEY=from-bundle\n"); + const legacy = bundle("from-legacy"); + expect(secretValue({ secretsFile: file, secretFiles: { THT_MODEL_API_KEY_SECRET_FILE: legacy } }, "THT_MODEL_API_KEY")) + .toBe("from-bundle"); + expect(secretValue({ secretFiles: { THT_MODEL_API_KEY_SECRET_FILE: legacy } }, "THT_MODEL_API_KEY")) + .toBe("from-legacy"); +});