fix(resume): recover cleanly after Pi exits
This commit is contained in:
+2
-1
@@ -20,6 +20,7 @@ export interface BuildAppDeps {
|
||||
listModels?: ListModelsFn;
|
||||
getSettings?: () => Settings;
|
||||
readiness?: ReadinessManager;
|
||||
hub?: SseHub;
|
||||
}
|
||||
|
||||
export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstance {
|
||||
@@ -39,7 +40,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
dataRoot: config.dataRoot,
|
||||
});
|
||||
const mgr = deps?.mgr ?? new PiProcessManager(config, deps?.spawnFn ? { spawnFn: deps.spawnFn } : undefined);
|
||||
const hub = new SseHub();
|
||||
const hub = deps?.hub ?? new SseHub();
|
||||
const readiness = deps?.readiness ?? new ReadinessManager(
|
||||
tht as ThtRunner,
|
||||
Math.round(config.ollamaEnsureTimeoutMs / 1000),
|
||||
|
||||
@@ -33,7 +33,7 @@ export class SessionBridge {
|
||||
m.message?.role === "assistant" &&
|
||||
m.message?.stopReason === "error"
|
||||
) {
|
||||
this.state = "failed";
|
||||
this.markFailed();
|
||||
this.fan({
|
||||
type: "info",
|
||||
level: "error",
|
||||
@@ -69,6 +69,9 @@ export class SessionBridge {
|
||||
|
||||
beginTurn(): void { this.state = "running"; }
|
||||
|
||||
/** Record a backend-detected failure; callers own any sanitized client message. */
|
||||
markFailed(): void { this.state = "failed"; }
|
||||
|
||||
onClientEvent(cb: (e: ClientEvent) => void): void { this.cbs.add(cb); }
|
||||
|
||||
/** Eventi generati dal backend stesso (es. exit inatteso del child Pi), non da Pi. */
|
||||
|
||||
@@ -103,6 +103,7 @@ export class PiProcessManager {
|
||||
child.on("exit", (code) => {
|
||||
console.error(`[pi:${sessionId}] exited code=${code ?? "?"} mapped=${this.runtimes.get(sessionId) === rt}`);
|
||||
if (this.runtimes.get(sessionId) === rt) {
|
||||
rt.bridge.markFailed();
|
||||
this.runtimes.delete(sessionId);
|
||||
rt.bridge.emitClientEvent({
|
||||
type: "info",
|
||||
|
||||
@@ -106,8 +106,6 @@ export function sessionRoutes(
|
||||
if (state === "running" || state === "waiting") {
|
||||
return reply.code(200).send({ id, alreadyActive: true });
|
||||
}
|
||||
d.mgr.teardown(id);
|
||||
d.hub.clear(id);
|
||||
}
|
||||
const manifest = (await d.tht.sessionShow(id, d.getSettings().workspace)) as { status?: string; archived?: boolean } | null;
|
||||
if (manifest?.status === "finalized" || manifest?.archived) {
|
||||
@@ -124,6 +122,8 @@ export function sessionRoutes(
|
||||
author: getUser(req).id,
|
||||
mode: "resume" as const,
|
||||
};
|
||||
if (existing) d.mgr.teardown(id);
|
||||
d.hub.clear(id);
|
||||
await d.tht.reopenSession(id, settings.workspace);
|
||||
const rt = d.mgr.createFor(id, options);
|
||||
bindRuntime(id, rt);
|
||||
|
||||
@@ -92,6 +92,7 @@ test("un exit INATTESO del child notifica il client (info error + agent_end)", a
|
||||
const seen: any[] = [];
|
||||
rt.bridge.onClientEvent((e) => seen.push(e));
|
||||
child.emit("exit", 137);
|
||||
expect(rt.bridge.turnState()).toBe("failed");
|
||||
expect(seen).toEqual([
|
||||
{ type: "info", level: "error", text: expect.stringContaining("137") },
|
||||
{ type: "system_event", event: "session_failed" },
|
||||
|
||||
@@ -137,18 +137,21 @@ test.each(["running", "waiting"])(
|
||||
"POST resume preserves a %s runtime",
|
||||
async (state) => {
|
||||
let tornDown = false;
|
||||
let cleared = false;
|
||||
const existing = { bridge: { turnState: () => state } } as any;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
mgr: {
|
||||
get: () => existing,
|
||||
teardown: () => { tornDown = true; },
|
||||
} as any,
|
||||
hub: { clear: () => { cleared = true; } } as any,
|
||||
thtRunner: {} as any,
|
||||
getSettings: () => ({ workspace: "psd" }) as any,
|
||||
});
|
||||
const response = await app.inject({ method: "POST", url: "/sessions/s1/resume" });
|
||||
expect(response.json()).toEqual({ id: "s1", alreadyActive: true });
|
||||
expect(tornDown).toBe(false);
|
||||
expect(cleared).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -166,6 +169,10 @@ test.each(["idle", "failed"])(
|
||||
configure: async () => {},
|
||||
start: () => order.push("start"),
|
||||
} as any,
|
||||
hub: {
|
||||
clear: (id: string) => order.push(`clear:${id}`),
|
||||
publish: () => {},
|
||||
} as any,
|
||||
thtRunner: {
|
||||
sessionShow: async () => ({
|
||||
status: "open", archived: false,
|
||||
@@ -179,10 +186,95 @@ test.each(["idle", "failed"])(
|
||||
const response = await app.inject({ method: "POST", url: "/sessions/s1/resume" });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(response.json()).toEqual({ id: "s1" });
|
||||
expect(order).toEqual(["teardown:s1", "reopen", "create", "start"]);
|
||||
expect(order).toEqual(["teardown:s1", "clear:s1", "reopen", "create", "start"]);
|
||||
},
|
||||
);
|
||||
|
||||
test("POST resume without a runtime clears stale SSE state before cold start", async () => {
|
||||
const order: string[] = [];
|
||||
let createOptions: any;
|
||||
const newRuntime = { bridge: { onClientEvent: () => {}, emitClientEvent: () => {} } } as any;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
mgr: {
|
||||
get: () => undefined,
|
||||
createFor: (_id: string, options: any) => {
|
||||
createOptions = options;
|
||||
order.push("create");
|
||||
return newRuntime;
|
||||
},
|
||||
configure: async () => {},
|
||||
start: () => order.push("start"),
|
||||
} as any,
|
||||
hub: {
|
||||
clear: (id: string) => order.push(`clear:${id}`),
|
||||
publish: () => {},
|
||||
} as any,
|
||||
thtRunner: {
|
||||
sessionShow: async () => ({
|
||||
status: "open", archived: false,
|
||||
provider: "local-qwen", model: "qwen3.6-35b-a3b", thinking: "low",
|
||||
}),
|
||||
reopenSession: async () => order.push("reopen"),
|
||||
} as any,
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
getSettings: () => ({ workspace: "local", thinking: "medium" }) as any,
|
||||
});
|
||||
|
||||
const response = await app.inject({ method: "POST", url: "/sessions/crashed/resume" });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(response.json()).toEqual({ id: "crashed" });
|
||||
expect(order).toEqual(["clear:crashed", "reopen", "create", "start"]);
|
||||
expect(createOptions).toMatchObject({
|
||||
provider: "local-qwen", model: "qwen3.6-35b-a3b", thinking: "low", mode: "resume",
|
||||
});
|
||||
});
|
||||
|
||||
test("POST resume keeps an idle runtime stream attached when the manifest is read-only", async () => {
|
||||
let tornDown = false;
|
||||
let cleared = false;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
mgr: {
|
||||
get: () => ({ bridge: { turnState: () => "idle" } }),
|
||||
teardown: () => { tornDown = true; },
|
||||
} as any,
|
||||
hub: { clear: () => { cleared = true; } } as any,
|
||||
thtRunner: {
|
||||
sessionShow: async () => ({ status: "finalized", archived: false }),
|
||||
} as any,
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
|
||||
const response = await app.inject({ method: "POST", url: "/sessions/s1/resume" });
|
||||
|
||||
expect(response.statusCode).toBe(409);
|
||||
expect(tornDown).toBe(false);
|
||||
expect(cleared).toBe(false);
|
||||
});
|
||||
|
||||
test("POST resume keeps a failed runtime stream attached when readiness fails", async () => {
|
||||
let tornDown = false;
|
||||
let cleared = false;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
mgr: {
|
||||
get: () => ({ bridge: { turnState: () => "failed" } }),
|
||||
teardown: () => { tornDown = true; },
|
||||
} as any,
|
||||
hub: { clear: () => { cleared = true; } } as any,
|
||||
thtRunner: {
|
||||
sessionShow: async () => ({ status: "open", archived: false }),
|
||||
} as any,
|
||||
readiness: { ensure: async () => ({ ok: false, error: "not ready" }) } as any,
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
|
||||
const response = await app.inject({ method: "POST", url: "/sessions/s1/resume" });
|
||||
|
||||
expect(response.statusCode).toBe(503);
|
||||
expect(tornDown).toBe(false);
|
||||
expect(cleared).toBe(false);
|
||||
});
|
||||
|
||||
test("POST /sessions/:id/response inoltra al bridge (no error)", async () => {
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: {
|
||||
|
||||
@@ -68,6 +68,19 @@ test("assistant provider errors are sanitized and leave the turn failed", () =>
|
||||
expect(JSON.stringify(seen)).not.toContain("DO_NOT_LEAK");
|
||||
});
|
||||
|
||||
test("markFailed records backend-detected failure without emitting raw detail", () => {
|
||||
const { rpc } = fakeRpc();
|
||||
const bridge = new SessionBridge(rpc);
|
||||
const seen: any[] = [];
|
||||
bridge.onClientEvent((event) => seen.push(event));
|
||||
bridge.beginTurn();
|
||||
|
||||
bridge.markFailed();
|
||||
|
||||
expect(bridge.turnState()).toBe("failed");
|
||||
expect(seen).toEqual([]);
|
||||
});
|
||||
|
||||
test("reviewer wait and response transition waiting back to running", () => {
|
||||
const { rpc, fire } = fakeRpc();
|
||||
const bridge = new SessionBridge(rpc);
|
||||
|
||||
Reference in New Issue
Block a user