95 lines
3.4 KiB
TypeScript
95 lines
3.4 KiB
TypeScript
import { EventEmitter } from "node:events";
|
|
import { expect, test, vi } from "vitest";
|
|
import { loadConfig } from "../src/config.js";
|
|
import { createPiProviderSmoke } from "../src/pi/provider-smoke.js";
|
|
|
|
function rpcChild(onCommand: (command: any, emit: (message: unknown) => void) => void) {
|
|
const child: any = new EventEmitter();
|
|
child.stdout = new EventEmitter();
|
|
child.stderr = { resume: vi.fn() };
|
|
child.kill = vi.fn();
|
|
const emit = (message: unknown) => queueMicrotask(() => {
|
|
child.stdout.emit("data", `${JSON.stringify(message)}\n`);
|
|
});
|
|
child.stdin = {
|
|
write: (data: unknown) => {
|
|
onCommand(JSON.parse(String(data)), emit);
|
|
return true;
|
|
},
|
|
};
|
|
return child;
|
|
}
|
|
|
|
// Catches a provider smoke implementation that merely selects a model, leaks generated output,
|
|
// or fails to terminate its ephemeral Pi process after a real model turn.
|
|
test("provider smoke selects the configured model and completes a fixed output-discarding turn", async () => {
|
|
const commands: any[] = [];
|
|
const child = rpcChild((command, emit) => {
|
|
commands.push(command);
|
|
if (command.type === "set_model" || command.type === "set_thinking_level") {
|
|
emit({ type: "response", id: command.id, success: true });
|
|
}
|
|
if (command.type === "prompt") {
|
|
emit({
|
|
type: "message_end",
|
|
message: { role: "assistant", stopReason: "stop", content: "raw-provider-output" },
|
|
});
|
|
emit({
|
|
type: "agent_end",
|
|
messages: [{ role: "assistant", stopReason: "stop", content: "raw-provider-output" }],
|
|
});
|
|
}
|
|
});
|
|
const smoke = createPiProviderSmoke(loadConfig({
|
|
THT_HARNESS_DIR: "/app/harness",
|
|
PI_BIN: "/usr/local/bin/pi",
|
|
}), {
|
|
spawnFn: () => child,
|
|
authProviders: () => new Set(["zai"]),
|
|
});
|
|
|
|
await expect(smoke({
|
|
provider: "zai", model: "glm-5.2", reasoning: "medium", timeoutMs: 750,
|
|
})).resolves.toBeUndefined();
|
|
expect(commands.map(({ id: _id, ...command }) => command)).toEqual([
|
|
{ type: "set_model", provider: "zai", modelId: "glm-5.2" },
|
|
{ type: "set_thinking_level", level: "medium" },
|
|
{ type: "prompt", message: expect.stringMatching(/health check/i) },
|
|
]);
|
|
expect(child.kill).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
// Catches provider errors that are accepted as a successful health check or returned with raw
|
|
// credential/output diagnostics.
|
|
test("provider smoke rejects a failed model turn with a stable non-secret error", async () => {
|
|
const child = rpcChild((command, emit) => {
|
|
if (command.type === "set_model" || command.type === "set_thinking_level") {
|
|
emit({ type: "response", id: command.id, success: true });
|
|
}
|
|
if (command.type === "prompt") {
|
|
emit({
|
|
type: "message_end",
|
|
message: {
|
|
role: "assistant", stopReason: "error",
|
|
errorMessage: '401 {"token":"raw-provider-secret"}',
|
|
},
|
|
});
|
|
emit({ type: "agent_end", messages: [] });
|
|
}
|
|
});
|
|
const smoke = createPiProviderSmoke(loadConfig({}), {
|
|
spawnFn: () => child,
|
|
authProviders: () => new Set(["zai"]),
|
|
});
|
|
|
|
let caught: unknown;
|
|
try {
|
|
await smoke({ provider: "zai", model: "glm-5.2", reasoning: "medium", timeoutMs: 750 });
|
|
} catch (error) {
|
|
caught = error;
|
|
}
|
|
expect(caught).toBeInstanceOf(Error);
|
|
expect((caught as Error).message).toBe("Pi provider smoke check failed");
|
|
expect(String(caught)).not.toContain("raw-provider-secret");
|
|
});
|