fix: isolate pi provider smoke
This commit is contained in:
@@ -47,7 +47,10 @@ test("status parses only a Pi version from a fixed execFile argument array", asy
|
||||
config: { provider: "zai", model: "glm-5.2", reasoning: "medium" },
|
||||
checkedAt: "2026-08-05T10:00:00.000Z",
|
||||
});
|
||||
expect(calls).toEqual([{ command: "/usr/local/bin/pi", args: ["--version"], timeout: 750 }]);
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls[0]).toMatchObject({ command: "/usr/local/bin/pi", args: ["--version"] });
|
||||
expect(calls[0].timeout).toBeGreaterThan(0);
|
||||
expect(calls[0].timeout).toBeLessThanOrEqual(750);
|
||||
});
|
||||
|
||||
// Catches an options response that leaks provider metadata or lets callers choose model IDs that
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import { expect, test, vi } from "vitest";
|
||||
import { existsSync, readFileSync, readdirSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import { loadConfig } from "../src/config.js";
|
||||
import { createPiProviderSmoke } from "../src/pi/provider-smoke.js";
|
||||
|
||||
afterEach(() => vi.unstubAllEnvs());
|
||||
|
||||
function rpcChild(onCommand: (command: any, emit: (message: unknown) => void) => void) {
|
||||
const child: any = new EventEmitter();
|
||||
child.stdout = new EventEmitter();
|
||||
@@ -20,10 +24,17 @@ function rpcChild(onCommand: (command: any, emit: (message: unknown) => void) =>
|
||||
return child;
|
||||
}
|
||||
|
||||
// Catches a provider smoke implementation that merely selects a model, leaks generated output,
|
||||
// or fails to terminate its ephemeral Pi process after a real model turn.
|
||||
test("provider smoke selects the configured model and completes a fixed output-discarding turn", async () => {
|
||||
// Catches a provider smoke process that runs from the trusted harness or leaves Pi tools,
|
||||
// extensions, skills, context files, templates, themes, or session persistence enabled.
|
||||
test("provider smoke makes one configured request from an isolated no-capability Pi process", async () => {
|
||||
vi.stubEnv("THT_DATA_ROOT", "/mounted-workflow-state");
|
||||
vi.stubEnv("THT_SESSION", "mounted-session-id");
|
||||
vi.stubEnv("THT_AUTHOR", "mounted-author");
|
||||
vi.stubEnv("THT_CONFIG", "/mounted-workflow-state/config.yaml");
|
||||
vi.stubEnv("PI_CODING_AGENT_DIR", "/home/thoth/.pi/agent");
|
||||
vi.stubEnv("PI_CODING_AGENT_SESSION_DIR", "/mounted-session-state");
|
||||
const commands: any[] = [];
|
||||
const spawns: any[][] = [];
|
||||
const child = rpcChild((command, emit) => {
|
||||
commands.push(command);
|
||||
if (command.type === "set_model" || command.type === "set_thinking_level") {
|
||||
@@ -43,14 +54,47 @@ test("provider smoke selects the configured model and completes a fixed output-d
|
||||
const smoke = createPiProviderSmoke(loadConfig({
|
||||
THT_HARNESS_DIR: "/app/harness",
|
||||
PI_BIN: "/usr/local/bin/pi",
|
||||
THT_DATA_ROOT: "/mounted-workflow-state",
|
||||
}), {
|
||||
spawnFn: () => child,
|
||||
spawnFn: (...args) => {
|
||||
spawns.push(args);
|
||||
expect(args[2].cwd).not.toBe("/app/harness");
|
||||
expect(readdirSync(args[2].cwd)).toEqual([]);
|
||||
expect(args[2].env.PI_CODING_AGENT_DIR).not.toBe("/home/thoth/.pi/agent");
|
||||
expect(readdirSync(args[2].env.PI_CODING_AGENT_DIR)).toEqual(["auth.json"]);
|
||||
expect(JSON.parse(readFileSync(`${args[2].env.PI_CODING_AGENT_DIR}/auth.json`, "utf8")))
|
||||
.toEqual({ zai: { type: "api_key", key: "test-only" } });
|
||||
return child;
|
||||
},
|
||||
authProviders: () => new Set(["zai"]),
|
||||
readAuthStore: () => JSON.stringify({
|
||||
zai: { type: "api_key", key: "test-only" },
|
||||
deepseek: { type: "api_key", key: "must-not-enter-isolated-context" },
|
||||
}),
|
||||
});
|
||||
|
||||
await expect(smoke({
|
||||
provider: "zai", model: "glm-5.2", reasoning: "medium", timeoutMs: 750,
|
||||
})).resolves.toBeUndefined();
|
||||
expect(spawns).toHaveLength(1);
|
||||
expect(spawns[0][0]).toBe("/usr/local/bin/pi");
|
||||
expect(spawns[0][1]).toEqual([
|
||||
"--mode", "rpc",
|
||||
"--no-session",
|
||||
"--no-tools",
|
||||
"--no-extensions",
|
||||
"--no-skills",
|
||||
"--no-prompt-templates",
|
||||
"--no-themes",
|
||||
"--no-context-files",
|
||||
"--no-approve",
|
||||
]);
|
||||
expect(spawns[0][2].env).not.toHaveProperty("THT_DATA_ROOT");
|
||||
expect(spawns[0][2].env).not.toHaveProperty("THT_SESSION");
|
||||
expect(spawns[0][2].env).not.toHaveProperty("THT_AUTHOR");
|
||||
expect(spawns[0][2].env).not.toHaveProperty("THT_CONFIG");
|
||||
expect(spawns[0][2].env).not.toHaveProperty("PI_CODING_AGENT_SESSION_DIR");
|
||||
expect(existsSync(dirname(spawns[0][2].cwd))).toBe(false);
|
||||
expect(commands.map(({ id: _id, ...command }) => command)).toEqual([
|
||||
{ type: "set_model", provider: "zai", modelId: "glm-5.2" },
|
||||
{ type: "set_thinking_level", level: "medium" },
|
||||
@@ -59,6 +103,55 @@ test("provider smoke selects the configured model and completes a fixed output-d
|
||||
expect(child.kill).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
const unexpectedToolEvents = [
|
||||
{
|
||||
name: "streamed tool call",
|
||||
event: {
|
||||
type: "message_update",
|
||||
assistantMessageEvent: { type: "toolcall_start", contentIndex: 0 },
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "tool execution",
|
||||
event: { type: "tool_execution_start", toolCallId: "tool-1", toolName: "read" },
|
||||
},
|
||||
{
|
||||
name: "completed message tool call",
|
||||
event: {
|
||||
type: "message_end",
|
||||
message: { role: "assistant", stopReason: "toolUse", content: [{ type: "toolCall" }] },
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "turn tool result",
|
||||
event: { type: "turn_end", toolResults: [{ role: "toolResult" }] },
|
||||
},
|
||||
];
|
||||
|
||||
// Catches Pi/provider regressions that surface a tool capability despite the fixed no-tools argv;
|
||||
// accepting agent_end after any such event could hide a mounted-state read or mutation.
|
||||
test.each(unexpectedToolEvents)("provider smoke fails closed on an unexpected $name event", async ({ event }) => {
|
||||
const child = rpcChild((command, emit) => {
|
||||
if (command.type === "set_model" || command.type === "set_thinking_level") {
|
||||
emit({ type: "response", id: command.id, success: true });
|
||||
}
|
||||
if (command.type === "prompt") {
|
||||
emit(event);
|
||||
emit({ type: "agent_end", messages: [] });
|
||||
}
|
||||
});
|
||||
const smoke = createPiProviderSmoke(loadConfig({}), {
|
||||
spawnFn: () => child,
|
||||
authProviders: () => new Set(["zai"]),
|
||||
readAuthStore: () => '{"zai":{"type":"api_key","key":"test-only"}}',
|
||||
});
|
||||
|
||||
await expect(smoke({
|
||||
provider: "zai", model: "glm-5.2", reasoning: "medium", timeoutMs: 750,
|
||||
})).rejects.toThrow("Pi provider smoke check failed");
|
||||
expect(child.kill).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
// Catches provider errors that are accepted as a successful health check or returned with raw
|
||||
// credential/output diagnostics.
|
||||
test("provider smoke rejects a failed model turn with a stable non-secret error", async () => {
|
||||
@@ -80,6 +173,7 @@ test("provider smoke rejects a failed model turn with a stable non-secret error"
|
||||
const smoke = createPiProviderSmoke(loadConfig({}), {
|
||||
spawnFn: () => child,
|
||||
authProviders: () => new Set(["zai"]),
|
||||
readAuthStore: () => '{"zai":{"type":"api_key","key":"test-only"}}',
|
||||
});
|
||||
|
||||
let caught: unknown;
|
||||
|
||||
Reference in New Issue
Block a user