fix: harden pi management verification
This commit is contained in:
@@ -7,6 +7,7 @@ import {
|
|||||||
type Settings,
|
type Settings,
|
||||||
} from "../settings/settings-store.js";
|
} from "../settings/settings-store.js";
|
||||||
import type { PiModel } from "./list-models.js";
|
import type { PiModel } from "./list-models.js";
|
||||||
|
import { createPiProviderSmoke, type PiProviderSmoke } from "./provider-smoke.js";
|
||||||
|
|
||||||
const execFile = promisify(nodeExecFile);
|
const execFile = promisify(nodeExecFile);
|
||||||
const REASONING_CHOICES = ["low", "medium", "high"] as const;
|
const REASONING_CHOICES = ["low", "medium", "high"] as const;
|
||||||
@@ -80,6 +81,7 @@ export class PiManagementError extends Error {
|
|||||||
interface PiManagementDeps {
|
interface PiManagementDeps {
|
||||||
execute?: PiExecFile;
|
execute?: PiExecFile;
|
||||||
listModels: () => Promise<PiModel[]>;
|
listModels: () => Promise<PiModel[]>;
|
||||||
|
smokeProvider?: PiProviderSmoke;
|
||||||
readSettings?: () => Settings;
|
readSettings?: () => Settings;
|
||||||
saveSettings?: (settings: Settings) => Settings;
|
saveSettings?: (settings: Settings) => Settings;
|
||||||
readLogs?: () => string | Promise<string>;
|
readLogs?: () => string | Promise<string>;
|
||||||
@@ -97,6 +99,7 @@ export function createPiManagement(config: AppConfig, deps: PiManagementDeps): P
|
|||||||
const readSettings = deps.readSettings ?? (() => loadSettings(config));
|
const readSettings = deps.readSettings ?? (() => loadSettings(config));
|
||||||
const persistSettings = deps.saveSettings ?? ((settings) => saveSettings(config, settings));
|
const persistSettings = deps.saveSettings ?? ((settings) => saveSettings(config, settings));
|
||||||
const readLogs = deps.readLogs ?? (() => diagnostics.join("\n"));
|
const readLogs = deps.readLogs ?? (() => diagnostics.join("\n"));
|
||||||
|
const smokeProvider = deps.smokeProvider ?? createPiProviderSmoke(config);
|
||||||
|
|
||||||
const closedOptions = async (): Promise<Omit<PiOptions, "checkedAt">> => {
|
const closedOptions = async (): Promise<Omit<PiOptions, "checkedAt">> => {
|
||||||
let listed: PiModel[];
|
let listed: PiModel[];
|
||||||
@@ -123,12 +126,12 @@ export function createPiManagement(config: AppConfig, deps: PiManagementDeps): P
|
|||||||
return { providers, models, reasoning: [...REASONING_CHOICES] };
|
return { providers, models, reasoning: [...REASONING_CHOICES] };
|
||||||
};
|
};
|
||||||
|
|
||||||
const version = async (): Promise<string> => {
|
const version = async (timeoutMs = config.piManagementTimeoutMs): Promise<string> => {
|
||||||
let output: { stdout: string; stderr: string };
|
let output: { stdout: string; stderr: string };
|
||||||
try {
|
try {
|
||||||
// The Pi executable and every argument are installation-owned constants. Do not add a shell.
|
// The Pi executable and every argument are installation-owned constants. Do not add a shell.
|
||||||
output = await execute(config.piBin, ["--version"], {
|
output = await execute(config.piBin, ["--version"], {
|
||||||
timeout: config.piManagementTimeoutMs,
|
timeout: timeoutMs,
|
||||||
maxBuffer: MAX_EXEC_OUTPUT_BYTES,
|
maxBuffer: MAX_EXEC_OUTPUT_BYTES,
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -193,20 +196,40 @@ export function createPiManagement(config: AppConfig, deps: PiManagementDeps): P
|
|||||||
|
|
||||||
async test(): Promise<PiTestResult> {
|
async test(): Promise<PiTestResult> {
|
||||||
const checkedAt = now().toISOString();
|
const checkedAt = now().toISOString();
|
||||||
|
const deadline = Date.now() + config.piManagementTimeoutMs;
|
||||||
|
let timer: NodeJS.Timeout | undefined;
|
||||||
try {
|
try {
|
||||||
await version();
|
const check = async (): Promise<void> => {
|
||||||
|
await version(remainingBudget(deadline));
|
||||||
const current = installationConfig();
|
const current = installationConfig();
|
||||||
if (!current.provider || !current.model || !current.reasoning) {
|
if (!current.provider || !current.model || !current.reasoning) {
|
||||||
return smokeFailure("Pi installation configuration is incomplete", checkedAt, addDiagnostic);
|
throw new PiManagementError(
|
||||||
}
|
"pi_management_unavailable",
|
||||||
const choices = await closedOptions();
|
"Pi installation configuration is incomplete",
|
||||||
if (!choices.models.some((model) => model.provider === current.provider && model.id === current.model)) {
|
);
|
||||||
return smokeFailure("Configured Pi provider and model are unavailable", checkedAt, addDiagnostic);
|
|
||||||
}
|
}
|
||||||
|
await smokeProvider({
|
||||||
|
provider: current.provider,
|
||||||
|
model: current.model,
|
||||||
|
reasoning: current.reasoning,
|
||||||
|
timeoutMs: remainingBudget(deadline),
|
||||||
|
});
|
||||||
|
};
|
||||||
|
await Promise.race([
|
||||||
|
check(),
|
||||||
|
new Promise<never>((_resolve, reject) => {
|
||||||
|
timer = setTimeout(
|
||||||
|
() => reject(new PiManagementError("pi_management_unavailable", "Pi smoke check timed out")),
|
||||||
|
config.piManagementTimeoutMs,
|
||||||
|
);
|
||||||
|
}),
|
||||||
|
]);
|
||||||
addDiagnostic("Pi smoke check succeeded");
|
addDiagnostic("Pi smoke check succeeded");
|
||||||
return { ready: true, checkedAt };
|
return { ready: true, checkedAt };
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
return smokeFailure(stableMessage(error, "Pi smoke check failed"), checkedAt, addDiagnostic);
|
return smokeFailure(stableMessage(error, "Pi provider smoke check failed"), checkedAt, addDiagnostic);
|
||||||
|
} finally {
|
||||||
|
if (timer) clearTimeout(timer);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -279,7 +302,11 @@ function smokeFailure(
|
|||||||
export function redact(value: string): string {
|
export function redact(value: string): string {
|
||||||
return value
|
return value
|
||||||
.replace(/(\bauthorization\b\s*:\s*Bearer\s+)[^\s,;]+/giu, "$1[REDACTED]")
|
.replace(/(\bauthorization\b\s*:\s*Bearer\s+)[^\s,;]+/giu, "$1[REDACTED]")
|
||||||
.replace(/(\b(?:api[_-]?key|token|password|secret|authorization)\b\s*(?:=|:)\s*)(?:"[^"\r\n]*"|'[^'\r\n]*'|[^\s,;]+)/giu, "$1[REDACTED]")
|
.replace(/((?:["']?)[A-Za-z0-9_-]*(?:api[_-]?key|token|password|secret|authorization)[A-Za-z0-9_-]*(?:["']?)\s*(?:=|:)\s*)(?:"[^"\r\n]*"|'[^'\r\n]*'|[^\s,;}]+)/giu, "$1[REDACTED]")
|
||||||
.replace(/(\bBearer\s+)[^\s,;]+/giu, "$1[REDACTED]")
|
.replace(/(\bBearer\s+)[^\s,;]+/giu, "$1[REDACTED]")
|
||||||
.replace(/(\w+:\/\/[^:/\s]+:)[^@/\s]+@/gu, "$1[REDACTED]@");
|
.replace(/(\w+:\/\/[^:/\s]+:)[^@/\s]+@/gu, "$1[REDACTED]@");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function remainingBudget(deadline: number): number {
|
||||||
|
return Math.max(1, deadline - Date.now());
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,121 @@
|
|||||||
|
import { spawn as nodeSpawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||||
|
import type { AppConfig } from "../config.js";
|
||||||
|
import { secretValue } from "../config/secret-bundle.js";
|
||||||
|
import { clearPrincipalEnvironment } from "../auth/principal.js";
|
||||||
|
import { RpcClient } from "../rpc/rpc-client.js";
|
||||||
|
import { loadPiAuthProviders } from "./auth-providers.js";
|
||||||
|
import { buildPiChildEnv, canonicalPiProvider } from "./provider-credentials.js";
|
||||||
|
import type { PiReasoning } from "./management.js";
|
||||||
|
|
||||||
|
const SMOKE_PROMPT = "Provider health check only. Reply with exactly OK without using tools.";
|
||||||
|
|
||||||
|
export interface PiProviderSmokeRequest {
|
||||||
|
provider: string;
|
||||||
|
model: string;
|
||||||
|
reasoning: PiReasoning;
|
||||||
|
timeoutMs: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type PiProviderSmoke = (request: PiProviderSmokeRequest) => Promise<void>;
|
||||||
|
|
||||||
|
interface ProviderSmokeOptions {
|
||||||
|
spawnFn?: (
|
||||||
|
command: string,
|
||||||
|
args: string[],
|
||||||
|
options: { cwd: string; env: NodeJS.ProcessEnv },
|
||||||
|
) => ChildProcessWithoutNullStreams;
|
||||||
|
authProviders?: () => ReadonlySet<string>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createPiProviderSmoke(
|
||||||
|
config: AppConfig,
|
||||||
|
options: ProviderSmokeOptions = {},
|
||||||
|
): PiProviderSmoke {
|
||||||
|
const spawnFn = options.spawnFn ?? nodeSpawn;
|
||||||
|
const authProviders = options.authProviders ?? (() => loadPiAuthProviders());
|
||||||
|
|
||||||
|
return async ({ provider, model, reasoning, timeoutMs }): Promise<void> => {
|
||||||
|
let child: ChildProcessWithoutNullStreams | undefined;
|
||||||
|
let timer: NodeJS.Timeout | undefined;
|
||||||
|
try {
|
||||||
|
const canonicalProvider = canonicalPiProvider(provider);
|
||||||
|
if (!canonicalProvider || timeoutMs <= 0) throw providerFailure();
|
||||||
|
const env = buildPiChildEnv({
|
||||||
|
provider: canonicalProvider,
|
||||||
|
authProviders: authProviders(),
|
||||||
|
credentialValue: secretValue(config, "THT_MODEL_API_KEY"),
|
||||||
|
credentialFile: config.modelApiKeyFile,
|
||||||
|
});
|
||||||
|
clearPrincipalEnvironment(env);
|
||||||
|
delete env.THT_DATA_ROOT;
|
||||||
|
if (config.dataRoot !== undefined) env.THT_DATA_ROOT = config.dataRoot;
|
||||||
|
|
||||||
|
child = spawnFn(config.piBin, ["--mode", "rpc"], { cwd: config.harnessDir, env });
|
||||||
|
child.stderr.resume();
|
||||||
|
const rpc = new RpcClient(child);
|
||||||
|
const turn = async (): Promise<void> => {
|
||||||
|
requireSuccessfulResponse(await rpc.request({
|
||||||
|
type: "set_model", provider: canonicalProvider, modelId: model,
|
||||||
|
} as object & { type: string }));
|
||||||
|
requireSuccessfulResponse(await rpc.request({
|
||||||
|
type: "set_thinking_level", level: reasoning,
|
||||||
|
} as object & { type: string }));
|
||||||
|
await waitForProviderTurn(rpc, child!);
|
||||||
|
};
|
||||||
|
await Promise.race([
|
||||||
|
turn(),
|
||||||
|
new Promise<never>((_resolve, reject) => {
|
||||||
|
timer = setTimeout(() => reject(providerTimeout()), timeoutMs);
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
} catch (error) {
|
||||||
|
if (isProviderTimeout(error)) throw providerTimeout();
|
||||||
|
throw providerFailure();
|
||||||
|
} finally {
|
||||||
|
if (timer) clearTimeout(timer);
|
||||||
|
if (child) {
|
||||||
|
try { child.kill(); } catch { /* preserve the sanitized smoke outcome */ }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function waitForProviderTurn(rpc: RpcClient, child: ChildProcessWithoutNullStreams): Promise<void> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
let failed = false;
|
||||||
|
rpc.on("event", (event) => {
|
||||||
|
if (event?.type === "message_end" && event.message?.role === "assistant"
|
||||||
|
&& event.message.stopReason === "error") {
|
||||||
|
failed = true;
|
||||||
|
reject(providerFailure());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (event?.type === "agent_end") {
|
||||||
|
const messages = Array.isArray(event.messages) ? event.messages : [];
|
||||||
|
const eventFailed = messages.some((message: any) => (
|
||||||
|
message?.role === "assistant" && message?.stopReason === "error"
|
||||||
|
));
|
||||||
|
if (failed || eventFailed) reject(providerFailure());
|
||||||
|
else resolve();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
child.once("exit", () => reject(providerFailure()));
|
||||||
|
rpc.send({ type: "prompt", message: SMOKE_PROMPT });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function requireSuccessfulResponse(response: any): void {
|
||||||
|
if (!response || response.success !== true) throw providerFailure();
|
||||||
|
}
|
||||||
|
|
||||||
|
function providerFailure(): Error {
|
||||||
|
return new Error("Pi provider smoke check failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
function providerTimeout(): Error {
|
||||||
|
return Object.assign(new Error("Pi smoke check timed out"), { code: "ETIMEDOUT" });
|
||||||
|
}
|
||||||
|
|
||||||
|
function isProviderTimeout(error: unknown): boolean {
|
||||||
|
return Boolean(error && typeof error === "object" && (error as { code?: unknown }).code === "ETIMEDOUT");
|
||||||
|
}
|
||||||
@@ -29,6 +29,10 @@ async function run<T>(
|
|||||||
if (!managementAllowed(deps.config, principal.isAdmin)) {
|
if (!managementAllowed(deps.config, principal.isAdmin)) {
|
||||||
return reply.code(403).send({ code: "pi_management_forbidden", error: "Pi management is not permitted" });
|
return reply.code(403).send({ code: "pi_management_forbidden", error: "Pi management is not permitted" });
|
||||||
}
|
}
|
||||||
|
if (deps.config.authMode === "none" && isManagementWrite(request.method)
|
||||||
|
&& !sameOriginOrNonBrowser(request)) {
|
||||||
|
return reply.code(403).send({ code: "pi_management_forbidden", error: "Pi management is not permitted" });
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
return await action();
|
return await action();
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -44,3 +48,20 @@ function managementAllowed(config: AppConfig, isAdmin: boolean): boolean {
|
|||||||
return (config.authMode === "none" && !config.publicExposure)
|
return (config.authMode === "none" && !config.publicExposure)
|
||||||
|| (config.authMode === "upstream" && isAdmin);
|
|| (config.authMode === "upstream" && isAdmin);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isManagementWrite(method: string): boolean {
|
||||||
|
return method === "POST" || method === "PUT" || method === "PATCH" || method === "DELETE";
|
||||||
|
}
|
||||||
|
|
||||||
|
function sameOriginOrNonBrowser(request: FastifyRequest): boolean {
|
||||||
|
const origin = request.headers.origin;
|
||||||
|
if (origin === undefined) return true;
|
||||||
|
if (typeof origin !== "string" || typeof request.headers.host !== "string") return false;
|
||||||
|
try {
|
||||||
|
const supplied = new URL(origin);
|
||||||
|
const expected = new URL(`${request.protocol}://${request.headers.host}`);
|
||||||
|
return supplied.origin === expected.origin;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
|
import { mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
|
||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import { expect, test } from "vitest";
|
import { expect, test, vi } from "vitest";
|
||||||
import { loadConfig } from "../src/config.js";
|
import { loadConfig } from "../src/config.js";
|
||||||
import {
|
import {
|
||||||
PiManagementError,
|
PiManagementError,
|
||||||
@@ -140,12 +140,95 @@ test("smoke uses the configured timeout and reports a sanitized timeout", async
|
|||||||
expect(calls).toEqual([{ command: "/usr/local/bin/pi", args: ["--version"], timeout: 750 }]);
|
expect(calls).toEqual([{ command: "/usr/local/bin/pi", args: ["--version"], timeout: 750 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Catches a smoke endpoint that validates only the Pi binary/model catalogue and never makes a
|
||||||
|
// request through the configured provider and model.
|
||||||
|
test("smoke exercises the configured provider and model", async () => {
|
||||||
|
const providerChecks: unknown[] = [];
|
||||||
|
const service = createPiManagement(configFor(), {
|
||||||
|
execute: successfulExec([]),
|
||||||
|
listModels: async () => supportedModels,
|
||||||
|
readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }),
|
||||||
|
smokeProvider: async (request) => { providerChecks.push(request); },
|
||||||
|
now: () => new Date("2026-08-05T10:00:00.000Z"),
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(service.test()).resolves.toEqual({
|
||||||
|
ready: true,
|
||||||
|
checkedAt: "2026-08-05T10:00:00.000Z",
|
||||||
|
});
|
||||||
|
expect(providerChecks).toEqual([{
|
||||||
|
provider: "zai", model: "glm-5.2", reasoning: "medium", timeoutMs: expect.any(Number),
|
||||||
|
}]);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Catches expired provider credentials being treated as ready or raw provider diagnostics being
|
||||||
|
// reflected through the management API.
|
||||||
|
test("smoke fails closed and sanitizes configured-provider authentication errors", async () => {
|
||||||
|
const service = createPiManagement(configFor(), {
|
||||||
|
execute: successfulExec([]),
|
||||||
|
listModels: async () => supportedModels,
|
||||||
|
readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }),
|
||||||
|
smokeProvider: async () => {
|
||||||
|
throw new Error('401 {"token":"raw-expired-token","output":"raw-provider-output"}');
|
||||||
|
},
|
||||||
|
now: () => new Date("2026-08-05T10:00:00.000Z"),
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await service.test();
|
||||||
|
expect(result).toEqual({
|
||||||
|
ready: false,
|
||||||
|
message: "Pi provider smoke check failed",
|
||||||
|
checkedAt: "2026-08-05T10:00:00.000Z",
|
||||||
|
});
|
||||||
|
expect(JSON.stringify(result)).not.toMatch(/raw-expired-token|raw-provider-output/);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Catches separate per-phase timeouts that allow a later provider turn to exceed the one
|
||||||
|
// end-to-end Pi Management smoke budget.
|
||||||
|
test("smoke applies one deadline across version and a hung provider turn", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
vi.setSystemTime(new Date("2026-08-05T10:00:00.000Z"));
|
||||||
|
try {
|
||||||
|
const providerTimeouts: number[] = [];
|
||||||
|
const service = createPiManagement(configFor(), {
|
||||||
|
execute: async () => await new Promise((resolve) => setTimeout(
|
||||||
|
() => resolve({ stdout: "pi 0.80.3\n", stderr: "" }),
|
||||||
|
500,
|
||||||
|
)),
|
||||||
|
listModels: async () => supportedModels,
|
||||||
|
readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }),
|
||||||
|
smokeProvider: async ({ timeoutMs }) => {
|
||||||
|
providerTimeouts.push(timeoutMs);
|
||||||
|
await new Promise(() => {});
|
||||||
|
},
|
||||||
|
now: () => new Date("2026-08-05T10:00:00.000Z"),
|
||||||
|
});
|
||||||
|
|
||||||
|
let settled = false;
|
||||||
|
const pending = service.test().finally(() => { settled = true; });
|
||||||
|
await vi.advanceTimersByTimeAsync(500);
|
||||||
|
expect(providerTimeouts).toEqual([250]);
|
||||||
|
await vi.advanceTimersByTimeAsync(249);
|
||||||
|
expect(settled).toBe(false);
|
||||||
|
await vi.advanceTimersByTimeAsync(1);
|
||||||
|
await expect(pending).resolves.toEqual({
|
||||||
|
ready: false,
|
||||||
|
message: "Pi smoke check timed out",
|
||||||
|
checkedAt: "2026-08-05T10:00:00.000Z",
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
// Catches an unbounded diagnostics endpoint or one that returns bearer tokens and connection
|
// Catches an unbounded diagnostics endpoint or one that returns bearer tokens and connection
|
||||||
// passwords captured in Pi output.
|
// passwords captured in Pi output.
|
||||||
test("logs keep only the latest 200 redacted lines", async () => {
|
test("logs keep only the latest 200 redacted lines", async () => {
|
||||||
const source = Array.from({ length: 205 }, (_, index) => `line-${index + 1}`);
|
const source = Array.from({ length: 205 }, (_, index) => `line-${index + 1}`);
|
||||||
source[203] = "Authorization: Bearer raw-bearer-token";
|
source[203] = "Authorization: Bearer raw-bearer-token";
|
||||||
source[204] = "database_url=postgres://thoth:raw-db-password@example.invalid/db";
|
source[204] = "database_url=postgres://thoth:raw-db-password@example.invalid/db";
|
||||||
|
source[202] = '{"token":"raw-json-secret","password":"raw-json-password"}';
|
||||||
|
source[201] = "THT_MODEL_API_KEY=raw-env-secret";
|
||||||
const service = createPiManagement(configFor(), {
|
const service = createPiManagement(configFor(), {
|
||||||
execute: successfulExec([]),
|
execute: successfulExec([]),
|
||||||
listModels: async () => supportedModels,
|
listModels: async () => supportedModels,
|
||||||
@@ -159,5 +242,8 @@ test("logs keep only the latest 200 redacted lines", async () => {
|
|||||||
expect(logs.lines[0]).toBe("line-6");
|
expect(logs.lines[0]).toBe("line-6");
|
||||||
expect(logs.lines.join("\n")).not.toContain("raw-bearer-token");
|
expect(logs.lines.join("\n")).not.toContain("raw-bearer-token");
|
||||||
expect(logs.lines.join("\n")).not.toContain("raw-db-password");
|
expect(logs.lines.join("\n")).not.toContain("raw-db-password");
|
||||||
|
expect(logs.lines.join("\n")).not.toContain("raw-json-secret");
|
||||||
|
expect(logs.lines.join("\n")).not.toContain("raw-json-password");
|
||||||
|
expect(logs.lines.join("\n")).not.toContain("raw-env-secret");
|
||||||
expect(logs.lines.join("\n")).toContain("[REDACTED]");
|
expect(logs.lines.join("\n")).toContain("[REDACTED]");
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,94 @@
|
|||||||
|
import { EventEmitter } from "node:events";
|
||||||
|
import { expect, test, vi } from "vitest";
|
||||||
|
import { loadConfig } from "../src/config.js";
|
||||||
|
import { createPiProviderSmoke } from "../src/pi/provider-smoke.js";
|
||||||
|
|
||||||
|
function rpcChild(onCommand: (command: any, emit: (message: unknown) => void) => void) {
|
||||||
|
const child: any = new EventEmitter();
|
||||||
|
child.stdout = new EventEmitter();
|
||||||
|
child.stderr = { resume: vi.fn() };
|
||||||
|
child.kill = vi.fn();
|
||||||
|
const emit = (message: unknown) => queueMicrotask(() => {
|
||||||
|
child.stdout.emit("data", `${JSON.stringify(message)}\n`);
|
||||||
|
});
|
||||||
|
child.stdin = {
|
||||||
|
write: (data: unknown) => {
|
||||||
|
onCommand(JSON.parse(String(data)), emit);
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
return child;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Catches a provider smoke implementation that merely selects a model, leaks generated output,
|
||||||
|
// or fails to terminate its ephemeral Pi process after a real model turn.
|
||||||
|
test("provider smoke selects the configured model and completes a fixed output-discarding turn", async () => {
|
||||||
|
const commands: any[] = [];
|
||||||
|
const child = rpcChild((command, emit) => {
|
||||||
|
commands.push(command);
|
||||||
|
if (command.type === "set_model" || command.type === "set_thinking_level") {
|
||||||
|
emit({ type: "response", id: command.id, success: true });
|
||||||
|
}
|
||||||
|
if (command.type === "prompt") {
|
||||||
|
emit({
|
||||||
|
type: "message_end",
|
||||||
|
message: { role: "assistant", stopReason: "stop", content: "raw-provider-output" },
|
||||||
|
});
|
||||||
|
emit({
|
||||||
|
type: "agent_end",
|
||||||
|
messages: [{ role: "assistant", stopReason: "stop", content: "raw-provider-output" }],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
const smoke = createPiProviderSmoke(loadConfig({
|
||||||
|
THT_HARNESS_DIR: "/app/harness",
|
||||||
|
PI_BIN: "/usr/local/bin/pi",
|
||||||
|
}), {
|
||||||
|
spawnFn: () => child,
|
||||||
|
authProviders: () => new Set(["zai"]),
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(smoke({
|
||||||
|
provider: "zai", model: "glm-5.2", reasoning: "medium", timeoutMs: 750,
|
||||||
|
})).resolves.toBeUndefined();
|
||||||
|
expect(commands.map(({ id: _id, ...command }) => command)).toEqual([
|
||||||
|
{ type: "set_model", provider: "zai", modelId: "glm-5.2" },
|
||||||
|
{ type: "set_thinking_level", level: "medium" },
|
||||||
|
{ type: "prompt", message: expect.stringMatching(/health check/i) },
|
||||||
|
]);
|
||||||
|
expect(child.kill).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
|
||||||
|
// Catches provider errors that are accepted as a successful health check or returned with raw
|
||||||
|
// credential/output diagnostics.
|
||||||
|
test("provider smoke rejects a failed model turn with a stable non-secret error", async () => {
|
||||||
|
const child = rpcChild((command, emit) => {
|
||||||
|
if (command.type === "set_model" || command.type === "set_thinking_level") {
|
||||||
|
emit({ type: "response", id: command.id, success: true });
|
||||||
|
}
|
||||||
|
if (command.type === "prompt") {
|
||||||
|
emit({
|
||||||
|
type: "message_end",
|
||||||
|
message: {
|
||||||
|
role: "assistant", stopReason: "error",
|
||||||
|
errorMessage: '401 {"token":"raw-provider-secret"}',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
emit({ type: "agent_end", messages: [] });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
const smoke = createPiProviderSmoke(loadConfig({}), {
|
||||||
|
spawnFn: () => child,
|
||||||
|
authProviders: () => new Set(["zai"]),
|
||||||
|
});
|
||||||
|
|
||||||
|
let caught: unknown;
|
||||||
|
try {
|
||||||
|
await smoke({ provider: "zai", model: "glm-5.2", reasoning: "medium", timeoutMs: 750 });
|
||||||
|
} catch (error) {
|
||||||
|
caught = error;
|
||||||
|
}
|
||||||
|
expect(caught).toBeInstanceOf(Error);
|
||||||
|
expect((caught as Error).message).toBe("Pi provider smoke check failed");
|
||||||
|
expect(String(caught)).not.toContain("raw-provider-secret");
|
||||||
|
});
|
||||||
@@ -85,6 +85,51 @@ test("loopback-only AUTH_MODE=none may read the sanitized Pi status", async () =
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Catches an arbitrary website using browser CORS to mutate a loopback-only installation's Pi
|
||||||
|
// defaults or trigger provider work with the local user's authority.
|
||||||
|
test("loopback-only management rejects cross-origin writes", async () => {
|
||||||
|
const service = fakeService();
|
||||||
|
const app = appWith(service);
|
||||||
|
try {
|
||||||
|
const configured = await app.inject({
|
||||||
|
method: "PUT", url: "/pi-management/config",
|
||||||
|
headers: { host: "127.0.0.1:8080", origin: "https://evil.example" },
|
||||||
|
payload: { provider: "zai", model: "glm-5.2", reasoning: "high" },
|
||||||
|
});
|
||||||
|
const smoke = await app.inject({
|
||||||
|
method: "POST", url: "/pi-management/test",
|
||||||
|
headers: { host: "127.0.0.1:8080", origin: "https://evil.example" },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(configured.statusCode).toBe(403);
|
||||||
|
expect(smoke.statusCode).toBe(403);
|
||||||
|
expect(service.configure).not.toHaveBeenCalled();
|
||||||
|
expect(service.test).not.toHaveBeenCalled();
|
||||||
|
} finally {
|
||||||
|
await app.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Catches an origin guard that also blocks the same-origin Docker frontend or non-browser local
|
||||||
|
// lifecycle clients that do not send Origin.
|
||||||
|
test("loopback-only management preserves same-origin frontend and origin-less local writes", async () => {
|
||||||
|
const service = fakeService();
|
||||||
|
const app = appWith(service);
|
||||||
|
try {
|
||||||
|
const sameOrigin = await app.inject({
|
||||||
|
method: "PUT", url: "/pi-management/config",
|
||||||
|
headers: { host: "127.0.0.1:8080", origin: "http://127.0.0.1:8080" },
|
||||||
|
payload: { provider: "zai", model: "glm-5.2", reasoning: "high" },
|
||||||
|
});
|
||||||
|
const lifecycleClient = await app.inject({ method: "POST", url: "/pi-management/test" });
|
||||||
|
|
||||||
|
expect(sameOrigin.statusCode).toBe(200);
|
||||||
|
expect(lifecycleClient.statusCode).toBe(200);
|
||||||
|
} finally {
|
||||||
|
await app.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
// Catches route wiring that bypasses closed service validation or gives the browser a Docker/image
|
// Catches route wiring that bypasses closed service validation or gives the browser a Docker/image
|
||||||
// lifecycle endpoint rather than only installation-default configuration and diagnostics.
|
// lifecycle endpoint rather than only installation-default configuration and diagnostics.
|
||||||
test("trusted admins receive only configuration, smoke, options, and log endpoints", async () => {
|
test("trusted admins receive only configuration, smoke, options, and log endpoints", async () => {
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
# Host nginx example. The auth service MUST authenticate every request and return a stable
|
# Host nginx example. The auth service MUST authenticate every request and return only the
|
||||||
# identity in X-Authenticated-User. ThothII itself remains on 127.0.0.1:8080.
|
# normalized X-Thoth-* identity/admin claims below. ThothII remains on 127.0.0.1:8080.
|
||||||
server {
|
server {
|
||||||
listen 443 ssl;
|
listen 443 ssl;
|
||||||
server_name thoth.example.test;
|
server_name thoth.example.test;
|
||||||
@@ -13,12 +13,33 @@ server {
|
|||||||
proxy_pass_request_body off;
|
proxy_pass_request_body off;
|
||||||
proxy_set_header Content-Length "";
|
proxy_set_header Content-Length "";
|
||||||
proxy_set_header X-Original-URI $request_uri;
|
proxy_set_header X-Original-URI $request_uri;
|
||||||
|
proxy_set_header X-Authenticated-User "";
|
||||||
|
proxy_set_header X-Thoth-Principal-Issuer "";
|
||||||
|
proxy_set_header X-Thoth-Principal-Subject "";
|
||||||
|
proxy_set_header X-Thoth-Principal-Display-Name "";
|
||||||
|
proxy_set_header X-Thoth-Is-Admin "";
|
||||||
|
proxy_set_header X-Thoth-Trusted-Principal-Issuer "";
|
||||||
|
proxy_set_header X-Thoth-Trusted-Principal-Subject "";
|
||||||
|
proxy_set_header X-Thoth-Trusted-Principal-Display-Name "";
|
||||||
|
proxy_set_header X-Thoth-Trusted-Is-Admin "";
|
||||||
}
|
}
|
||||||
|
|
||||||
location / {
|
location / {
|
||||||
auth_request /_authenticate;
|
auth_request /_authenticate;
|
||||||
auth_request_set $authenticated_user $upstream_http_x_authenticated_user;
|
auth_request_set $thoth_principal_issuer $upstream_http_x_thoth_principal_issuer;
|
||||||
proxy_set_header X-Authenticated-User $authenticated_user;
|
auth_request_set $thoth_principal_subject $upstream_http_x_thoth_principal_subject;
|
||||||
|
auth_request_set $thoth_principal_display_name $upstream_http_x_thoth_principal_display_name;
|
||||||
|
auth_request_set $thoth_is_admin $upstream_http_x_thoth_is_admin;
|
||||||
|
# Clear public normalized claims and carry auth_request results over the private hop.
|
||||||
|
proxy_set_header X-Authenticated-User "";
|
||||||
|
proxy_set_header X-Thoth-Principal-Issuer "";
|
||||||
|
proxy_set_header X-Thoth-Principal-Subject "";
|
||||||
|
proxy_set_header X-Thoth-Principal-Display-Name "";
|
||||||
|
proxy_set_header X-Thoth-Is-Admin "";
|
||||||
|
proxy_set_header X-Thoth-Trusted-Principal-Issuer $thoth_principal_issuer;
|
||||||
|
proxy_set_header X-Thoth-Trusted-Principal-Subject $thoth_principal_subject;
|
||||||
|
proxy_set_header X-Thoth-Trusted-Principal-Display-Name $thoth_principal_display_name;
|
||||||
|
proxy_set_header X-Thoth-Trusted-Is-Admin $thoth_is_admin;
|
||||||
proxy_set_header X-Forwarded-Proto https;
|
proxy_set_header X-Forwarded-Proto https;
|
||||||
proxy_set_header Host $host;
|
proxy_set_header Host $host;
|
||||||
proxy_pass http://127.0.0.1:8080;
|
proxy_pass http://127.0.0.1:8080;
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ server {
|
|||||||
location = /health {
|
location = /health {
|
||||||
proxy_pass ${THT_FRONTEND_API_UPSTREAM}/health;
|
proxy_pass ${THT_FRONTEND_API_UPSTREAM}/health;
|
||||||
proxy_http_version 1.1;
|
proxy_http_version 1.1;
|
||||||
proxy_set_header Host $host;
|
proxy_set_header Host $http_host;
|
||||||
proxy_cache off;
|
proxy_cache off;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -14,13 +14,21 @@ server {
|
|||||||
# The trailing slash replaces the matched /api/ prefix before the private hop.
|
# The trailing slash replaces the matched /api/ prefix before the private hop.
|
||||||
proxy_pass ${THT_FRONTEND_API_UPSTREAM}/;
|
proxy_pass ${THT_FRONTEND_API_UPSTREAM}/;
|
||||||
proxy_http_version 1.1;
|
proxy_http_version 1.1;
|
||||||
proxy_set_header Host $host;
|
proxy_set_header Host $http_host;
|
||||||
proxy_set_header X-Real-IP $remote_addr;
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
# Trusted only when AUTH_MODE=upstream and this frontend port is reachable solely
|
# Public normalized claims are discarded by mapping only the private-hop values from the
|
||||||
# from the authenticated host proxy documented in deploy/.
|
# authenticated host proxy. Private-hop headers are then cleared before reaching core.
|
||||||
proxy_set_header X-Authenticated-User $http_x_authenticated_user;
|
proxy_set_header X-Authenticated-User "";
|
||||||
|
proxy_set_header X-Thoth-Principal-Issuer $http_x_thoth_trusted_principal_issuer;
|
||||||
|
proxy_set_header X-Thoth-Principal-Subject $http_x_thoth_trusted_principal_subject;
|
||||||
|
proxy_set_header X-Thoth-Principal-Display-Name $http_x_thoth_trusted_principal_display_name;
|
||||||
|
proxy_set_header X-Thoth-Is-Admin $http_x_thoth_trusted_is_admin;
|
||||||
|
proxy_set_header X-Thoth-Trusted-Principal-Issuer "";
|
||||||
|
proxy_set_header X-Thoth-Trusted-Principal-Subject "";
|
||||||
|
proxy_set_header X-Thoth-Trusted-Principal-Display-Name "";
|
||||||
|
proxy_set_header X-Thoth-Trusted-Is-Admin "";
|
||||||
proxy_buffering off;
|
proxy_buffering off;
|
||||||
proxy_cache off;
|
proxy_cache off;
|
||||||
proxy_read_timeout 3600s;
|
proxy_read_timeout 3600s;
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ nginx_config=docker/nginx.conf.template
|
|||||||
for setting in \
|
for setting in \
|
||||||
'proxy_pass ${THT_FRONTEND_API_UPSTREAM}/;' \
|
'proxy_pass ${THT_FRONTEND_API_UPSTREAM}/;' \
|
||||||
'proxy_http_version 1.1;' \
|
'proxy_http_version 1.1;' \
|
||||||
|
'proxy_set_header Host $http_host;' \
|
||||||
'proxy_buffering off;' \
|
'proxy_buffering off;' \
|
||||||
'proxy_read_timeout 3600s;'; do
|
'proxy_read_timeout 3600s;'; do
|
||||||
if ! grep -Fq "$setting" "$nginx_config"; then
|
if ! grep -Fq "$setting" "$nginx_config"; then
|
||||||
|
|||||||
@@ -152,7 +152,11 @@ import it into the maintenance shell. Explicitly export the non-secret source an
|
|||||||
before running the commands below.
|
before running the commands below.
|
||||||
|
|
||||||
Configure the portal proxy so the frontend and `/api` share one origin. It authenticates first and
|
Configure the portal proxy so the frontend and `/api` share one origin. It authenticates first and
|
||||||
forwards the trusted identity expected by `AUTH_MODE=upstream`; it is the only public listener.
|
clears client identity headers, carries auth-request claims over the private hop as
|
||||||
|
`X-Thoth-Trusted-*`, and lets the frontend proxy inject only the normalized
|
||||||
|
`X-Thoth-Principal-Issuer`, `X-Thoth-Principal-Subject`, `X-Thoth-Principal-Display-Name`, and
|
||||||
|
`X-Thoth-Is-Admin` claims expected by `AUTH_MODE=upstream`; it is the only public listener. Use
|
||||||
|
`deploy/nginx-authenticated-proxy.conf.example` as the forwarding contract.
|
||||||
From a trusted maintenance shell:
|
From a trusted maintenance shell:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
|
|||||||
@@ -137,7 +137,11 @@ docker compose exec core /opt/venv/bin/tht doctor --json
|
|||||||
```
|
```
|
||||||
|
|
||||||
Se si abilita l'overlay production, il proxy autenticato TLS deve essere l'unico listener pubblico
|
Se si abilita l'overlay production, il proxy autenticato TLS deve essere l'unico listener pubblico
|
||||||
e deve iniettare `X-Authenticated-User`; non esporre direttamente la porta pubblicata da nginx.
|
e deve sostituire gli header client con i claim restituiti dal proprio `auth_request`. L'esempio
|
||||||
|
usa header `X-Thoth-Trusted-*` soltanto sul collegamento privato; nginx frontend li converte nei
|
||||||
|
claim normalizzati `X-Thoth-Principal-Issuer`, `X-Thoth-Principal-Subject`,
|
||||||
|
`X-Thoth-Principal-Display-Name` e `X-Thoth-Is-Admin` attesi dal core. Non esporre direttamente
|
||||||
|
la porta pubblicata da nginx.
|
||||||
|
|
||||||
Se il server deve essere raggiungibile da altri host, sostituire `COMPOSE_FILE` con
|
Se il server deve essere raggiungibile da altri host, sostituire `COMPOSE_FILE` con
|
||||||
`compose.yaml:deploy/compose.production.yaml`, configurare il proxy autenticato e impostare
|
`compose.yaml:deploy/compose.production.yaml`, configurare il proxy autenticato e impostare
|
||||||
|
|||||||
Executable
+45
@@ -0,0 +1,45 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
|
||||||
|
outer=deploy/nginx-authenticated-proxy.conf.example
|
||||||
|
inner=docker/nginx.conf.template
|
||||||
|
|
||||||
|
# Catches a documented public proxy that forwards client-supplied normalized identity/admin
|
||||||
|
# headers instead of replacing them with claims returned by auth_request.
|
||||||
|
for suffix in Principal-Issuer Principal-Subject Principal-Display-Name Is-Admin; do
|
||||||
|
variable=$(printf '%s' "$suffix" | tr '[:upper:]-' '[:lower:]_')
|
||||||
|
grep -Fq "auth_request_set \$thoth_${variable} \$upstream_http_x_thoth_${variable};" "$outer"
|
||||||
|
if [[ $(grep -Fc "proxy_set_header X-Thoth-${suffix} \"\";" "$outer") -lt 2 ]]; then
|
||||||
|
echo "public proxy does not clear X-Thoth-${suffix} at both ingress hops" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
grep -Fq "proxy_set_header X-Thoth-Trusted-${suffix} \$thoth_${variable};" "$outer"
|
||||||
|
done
|
||||||
|
if rg -n 'proxy_set_header X-Thoth-[^;]+\$http_x_thoth_' "$outer"; then
|
||||||
|
echo "public proxy trusts client-supplied normalized Thoth claims" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Catches an included frontend hop that preserves a client normalized claim or drops the original
|
||||||
|
# Host port needed for exact same-origin management checks.
|
||||||
|
for suffix in Principal-Issuer Principal-Subject Principal-Display-Name Is-Admin; do
|
||||||
|
variable=$(printf '%s' "$suffix" | tr '[:upper:]-' '[:lower:]_')
|
||||||
|
grep -Fq "proxy_set_header X-Thoth-${suffix} \$http_x_thoth_trusted_${variable};" "$inner"
|
||||||
|
grep -Fq "proxy_set_header X-Thoth-Trusted-${suffix} \"\";" "$inner"
|
||||||
|
done
|
||||||
|
grep -Fq 'proxy_set_header Host $http_host;' "$inner"
|
||||||
|
if rg -n 'proxy_set_header X-Thoth-(Principal|Is-Admin)[^;]+\$http_x_thoth_(principal|is_admin)' "$inner"; then
|
||||||
|
echo "frontend proxy trusts a client-supplied normalized Thoth claim" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
for config in "$outer" "$inner"; do
|
||||||
|
grep -Fq 'proxy_set_header X-Authenticated-User "";' "$config"
|
||||||
|
done
|
||||||
|
if rg --pcre2 -n 'proxy_set_header X-Authenticated-User\s+(?!"";)' "$outer" "$inner"; then
|
||||||
|
echo "legacy unnormalized identity is forwarded by the proxy chain" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "authenticated proxy identity contract: ok"
|
||||||
Reference in New Issue
Block a user