mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix: reject executable pi configuration

Browse Source
This commit is contained in:
marcopan
2026-08-05 04:49:19 +02:00
parent 6b828288e3
commit 2703864572
9 changed files with 479 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/test/list-models.test.ts
+131
View File
@@ -20,6 +20,9 @@ function enabled(...ids: string[]) {
return () => ({ ids, warnings: [], source: "/test/settings.json" });
}
const noManagedModels = { readModelsStore: () => undefined };
const MANAGED_CONFIG_ERROR = "Pi provider/model configuration is invalid";
test("createPiModelLister returns mapped PiModel[] from get_available_models", async () => {
const script = scriptWith([
{ provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true, extra: "ignored" },
@@ -27,6 +30,7 @@ test("createPiModelLister returns mapped PiModel[] from get_available_models", a
]);
try {
const lister = createPiModelLister(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
...noManagedModels,
loadEnabledModels: enabled("zai/glm-5.2", "anthropic/claude-opus-4-8"),
spawnFn: () => spawn("node", [FAKE, script]) as any,
});
@@ -45,6 +49,7 @@ test("createPiModelLister caches within ttl (spawns once for two calls)", async
try {
let spawns = 0;
const lister = createPiModelLister(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
...noManagedModels,
loadEnabledModels: enabled("zai/glm-5.2"),
spawnFn: () => { spawns++; return spawn("node", [FAKE, script]) as any; },
ttlMs: 10_000,
@@ -69,6 +74,7 @@ test("production model-list spawn preserves PATH and passes the portable data ro
PI_BIN: "/usr/local/bin/pi",
THT_DATA_ROOT: "/data",
}), {
...noManagedModels,
loadEnabledModels: enabled("test/unavailable"),
spawnFn: (...args: any[]) => {
calls.push(args);
@@ -101,6 +107,7 @@ test("model-list spawn scrubs ambient provider credentials and generic secret me
process.env.CLOUDFLARE_ACCOUNT_ID = "must-not-leak";
try {
const lister = createPiModelLister(loadConfig({ PI_BIN: "/usr/local/bin/pi" }), {
...noManagedModels,
loadEnabledModels: enabled("test/unavailable"),
spawnFn: (...args: any[]) => {
calls.push(args);
@@ -135,6 +142,7 @@ test("model listing does not require PI_PROVIDER or read the generic credential"
PI_BIN: "/usr/local/bin/pi",
THT_MODEL_API_KEY_FILE: "/missing-and-must-not-be-read",
}), {
...noManagedModels,
loadEnabledModels: enabled("zai/glm-5.2"),
spawnFn: (...args: any[]) => {
calls.push(args);
@@ -158,6 +166,7 @@ test("returns only enabled available models in enabledModels order", async () =>
]);
try {
const lister = createPiModelLister(loadConfig({}), {
...noManagedModels,
loadEnabledModels: enabled(
"zai/glm-5.2",
"deepseek/deepseek-v4-flash",
@@ -179,6 +188,7 @@ test("empty enabled model scope fails closed without spawning Pi", async () => {
let spawns = 0;
const warnings: string[] = [];
const lister = createPiModelLister(loadConfig({}), {
...noManagedModels,
loadEnabledModels: () => ({ ids: [], warnings: ["scope invalid"] }),
warn: (message) => warnings.push(message),
spawnFn: () => { spawns += 1; throw new Error("must not spawn"); },
@@ -195,6 +205,7 @@ test("warns and returns empty when enabled identifiers are unavailable", async (
const warnings: string[] = [];
try {
const lister = createPiModelLister(loadConfig({}), {
...noManagedModels,
loadEnabledModels: enabled("zai/glm-5.2"),
warn: (message) => warnings.push(message),
spawnFn: () => spawn("node", [FAKE, script]) as any,
@@ -205,3 +216,123 @@ test("warns and returns empty when enabled identifiers are unavailable", async (
rmSync(path.dirname(script), { recursive: true, force: true });
}
});
const executableModelsConfigCases: Array<[string, unknown]> = [
["nested provider headers", {
providers: {
selected: {
headers: { Authorization: "!sensitive-header-command /private/header-path" },
},
},
}],
["provider apiKey", {
providers: {
selected: { apiKey: "!sensitive-api-key-command /private/key-path" },
},
}],
["selected model objects", {
providers: {
selected: {
models: [{ id: "model", name: "!sensitive-model-command /private/model-path" }],
},
},
}],
["selected model overrides", {
providers: {
selected: {
modelOverrides: {
model: { headers: { "X-Override": "!sensitive-override-command /private/override-path" } },
},
},
},
}],
["nested arrays", {
providers: {
selected: {
compat: { nested: ["literal", { value: "!sensitive-array-command /private/array-path" }] },
},
},
}],
];
// Catches Task 8 model discovery delegating raw managed models.json values to Pi. Pi 0.80.3
// executes leading-! values at request time, so the complete managed store must be rejected before
// it can become an authoritative source of API choices.
test.each(executableModelsConfigCases)(
"managed models ingestion rejects executable strings in %s",
async (_name, modelsConfig) => {
let spawns = 0;
const lister = createPiModelLister(loadConfig({}), {
loadEnabledModels: enabled("selected/model"),
readModelsStore: () => JSON.stringify(modelsConfig),
spawnFn: () => {
spawns += 1;
throw new Error("unsafe model-list spawn");
},
});
let caught: unknown;
try {
await lister();
} catch (error) {
caught = error;
}
expect(caught).toBeInstanceOf(Error);
expect((caught as Error).message).toBe(MANAGED_CONFIG_ERROR);
expect(String(caught)).not.toMatch(/sensitive|private|command|path/i);
expect(spawns).toBe(0);
},
);
// Selection-time smoke filtering intentionally ignores unrelated providers, but the full
// installation-owned models.json is invalid at the model-choice ingestion boundary.
test("managed models ingestion rejects an executable string in an unrelated provider", async () => {
let spawns = 0;
const lister = createPiModelLister(loadConfig({}), {
loadEnabledModels: enabled("selected/model"),
readModelsStore: () => JSON.stringify({
providers: {
selected: { models: [{ id: "model" }] },
unrelated: { apiKey: "!sensitive-unrelated-command /private/unrelated-path" },
},
}),
spawnFn: () => {
spawns += 1;
throw new Error("unsafe model-list spawn");
},
});
await expect(lister()).rejects.toThrow(MANAGED_CONFIG_ERROR);
expect(spawns).toBe(0);
});
test("managed models ingestion revalidates the store before serving a cached choice", async () => {
const script = scriptWith([
{ provider: "selected", id: "model", name: "Selected model", reasoning: true },
]);
let managedModels = JSON.stringify({
providers: { selected: { models: [{ id: "model" }] } },
});
let spawns = 0;
try {
const lister = createPiModelLister(loadConfig({}), {
loadEnabledModels: enabled("selected/model"),
readModelsStore: () => managedModels,
spawnFn: () => {
spawns += 1;
return spawn("node", [FAKE, script]) as any;
},
ttlMs: 10_000,
});
await expect(lister()).resolves.toHaveLength(1);
managedModels = JSON.stringify({
providers: { selected: { headers: { Authorization: "!new-unsafe-value" } } },
});
await expect(lister()).rejects.toThrow(MANAGED_CONFIG_ERROR);
expect(spawns).toBe(1);
} finally {
rmSync(path.dirname(script), { recursive: true, force: true });
}
});
backend/test/pi-management.test.ts
+51
View File
@@ -73,6 +73,32 @@ test("options expose only closed provider, model, and reasoning choices", async
});
});
// Catches raw managed models.json validation details being collapsed into an ambiguous model-list
// failure or escaping through the Pi Management options API.
test("options report invalid managed model configuration with a stable sanitized error", async () => {
const service = createPiManagement(configFor(), {
execute: successfulExec([]),
listModels: async () => {
throw Object.assign(
new Error("!sensitive-command /private/models.json raw-secret"),
{ code: "PI_MANAGED_CONFIG_INVALID" },
);
},
});
let caught: unknown;
try {
await service.options();
} catch (error) {
caught = error;
}
expect(caught).toMatchObject<PiManagementError>({
code: "pi_management_unavailable",
message: "Pi provider/model configuration is invalid",
});
expect(String(caught)).not.toMatch(/sensitive|private|models\.json|secret/i);
});
// Catches configuration writes that accept whitespace, unknown choices, or extra free-form fields
// before reaching the durable installation settings file.
test("config rejects invalid free-form values before writing settings", async () => {
@@ -186,6 +212,31 @@ test("smoke fails closed and sanitizes configured-provider authentication errors
expect(JSON.stringify(result)).not.toMatch(/raw-expired-token|raw-provider-output/);
});
// Catches selected auth/models validation failures being downgraded to a generic provider error
// or exposing the rejected command, path, or secret through POST /pi-management/test.
test("smoke reports invalid managed provider configuration with a stable sanitized error", async () => {
const service = createPiManagement(configFor(), {
execute: successfulExec([]),
listModels: async () => supportedModels,
readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }),
smokeProvider: async () => {
throw Object.assign(
new Error("!sensitive-command /private/models.json raw-secret"),
{ code: "PI_MANAGED_CONFIG_INVALID" },
);
},
now: () => new Date("2026-08-05T10:00:00.000Z"),
});
const result = await service.test();
expect(result).toEqual({
ready: false,
message: "Pi provider/model configuration is invalid",
checkedAt: "2026-08-05T10:00:00.000Z",
});
expect(JSON.stringify(result)).not.toMatch(/sensitive|private|models\.json|secret/i);
});
// Catches separate per-phase timeouts that allow a later provider turn to exceed the one
// end-to-end Pi Management smoke budget.
test("smoke applies one deadline across version and a hung provider turn", async () => {
backend/test/pi-provider-smoke.test.ts
+128 -2
View File
@@ -24,6 +24,26 @@ function rpcChild(onCommand: (command: any, emit: (message: unknown) => void) =>
return child;
}
function successfulProviderChild() {
return rpcChild((command, emit) => {
if (command.type === "set_model" || command.type === "set_thinking_level") {
emit({ type: "response", id: command.id, success: true });
}
if (command.type === "prompt") {
emit({
type: "message_end",
message: { role: "assistant", stopReason: "stop", content: "must-not-be-returned" },
});
emit({
type: "agent_end",
messages: [{ role: "assistant", stopReason: "stop", content: "must-not-be-returned" }],
});
}
});
}
const MANAGED_CONFIG_ERROR = "Pi provider/model configuration is invalid";
// Catches an isolated smoke agent that copies auth.json but drops the selected custom
// provider/model from models.json, causing set_model to fail before the real request.
test("provider smoke reaches the selected custom provider from an isolated models.json", async () => {
@@ -61,23 +81,30 @@ test("provider smoke reaches the selected custom provider from an isolated model
providers: {
"custom-openai": {
baseUrl: "https://selected.invalid/v1",
apiKey: "$CUSTOM_OPENAI_API_KEY",
api: "openai-completions",
headers: { "X-Literal-Bang": "$!literal-value" },
models: [{ id: "selected-model", name: "Selected model", reasoning: true }],
},
},
});
expect(JSON.parse(readFileSync(`${isolatedAgentDir}/auth.json`, "utf8"))).toEqual({
"custom-openai": { type: "api_key", key: "${CUSTOM_OPENAI_API_KEY}" },
});
return child;
},
authProviders: () => new Set(["custom-openai"]),
readAuthStore: () => JSON.stringify({
"custom-openai": { type: "api_key", key: "test-only" },
unrelated: { type: "api_key", key: "must-not-enter-isolated-context" },
"custom-openai": { type: "api_key", key: "${CUSTOM_OPENAI_API_KEY}" },
unrelated: { type: "api_key", key: "!must-not-run-or-enter-isolated-context" },
}),
readModelsStore: () => JSON.stringify({
providers: {
"custom-openai": {
baseUrl: "https://selected.invalid/v1",
apiKey: "$CUSTOM_OPENAI_API_KEY",
api: "openai-completions",
headers: { "X-Literal-Bang": "$!literal-value" },
models: [
{ id: "selected-model", name: "Selected model", reasoning: true },
{ id: "unrelated-model", name: "Must not enter isolated context" },
@@ -101,6 +128,105 @@ test("provider smoke reaches the selected custom provider from an isolated model
expect(isolatedAgentDir && existsSync(dirname(isolatedAgentDir))).toBe(false);
});
const selectedExecutableConfigCases: Array<{
name: string;
selectedAuth?: unknown;
selectedProvider: Record<string, unknown>;
}> = [
{
name: "selected auth credential",
selectedAuth: {
type: "api_key",
key: "!sensitive-credential-command /private/credential-path",
},
selectedProvider: {},
},
{
name: "selected provider apiKey",
selectedProvider: {
apiKey: "!sensitive-api-key-command /private/key-path",
},
},
{
name: "nested selected-provider headers",
selectedProvider: {
headers: { Authorization: "!sensitive-header-command /private/header-path" },
},
},
{
name: "selected model object",
selectedProvider: {
models: [{
id: "selected-model",
name: "!sensitive-model-command /private/model-path",
}],
},
},
{
name: "selected model override",
selectedProvider: {
modelOverrides: {
"selected-model": {
headers: { "X-Override": "!sensitive-override-command /private/override-path" },
},
},
},
},
{
name: "array nested in selected provider configuration",
selectedProvider: {
compat: {
nested: ["literal", { value: "!sensitive-array-command /private/array-path" }],
},
},
},
];
// Catches a defense that validates only known top-level fields or waits until after the isolated
// Pi process starts. Every selected value crossing into auth.json/models.json must be declarative.
test.each(selectedExecutableConfigCases)(
"provider smoke rejects executable config in $name before isolated Pi spawn",
async ({ selectedAuth, selectedProvider }) => {
const child = successfulProviderChild();
const spawnFn = vi.fn(() => child);
const smoke = createPiProviderSmoke(loadConfig({}), {
spawnFn,
authProviders: () => new Set(["custom-openai"]),
readAuthStore: () => JSON.stringify({
"custom-openai": selectedAuth ?? { type: "api_key", key: "test-only" },
unrelated: { type: "api_key", key: "!must-not-contaminate-selected-provider" },
}),
readModelsStore: () => JSON.stringify({
providers: {
"custom-openai": {
baseUrl: "https://selected.invalid/v1",
api: "openai-completions",
models: [{ id: "selected-model", name: "Selected model" }],
...selectedProvider,
},
unrelated: {
apiKey: "!must-not-contaminate-selected-provider",
models: [{ id: "unrelated-model" }],
},
},
}),
});
let caught: unknown;
try {
await smoke({
provider: "custom-openai", model: "selected-model", reasoning: "medium", timeoutMs: 750,
});
} catch (error) {
caught = error;
}
expect(caught).toBeInstanceOf(Error);
expect((caught as Error).message).toBe(MANAGED_CONFIG_ERROR);
expect(String(caught)).not.toMatch(/sensitive|private|command|path/i);
expect(spawnFn).not.toHaveBeenCalled();
},
);
// Catches a provider smoke process that runs from the trusted harness or leaves Pi tools,
// extensions, skills, context files, templates, themes, or session persistence enabled.
test("provider smoke makes one configured request from an isolated no-capability Pi process", async () => {