fix(resume): recover cleanly after Pi exits

This commit is contained in:
User
2026-07-14 21:35:50 +02:00
parent 7e1d72c9a4
commit df1e7dea9c
13 changed files with 214 additions and 17 deletions
+2 -1
View File
@@ -20,6 +20,7 @@ export interface BuildAppDeps {
listModels?: ListModelsFn;
getSettings?: () => Settings;
readiness?: ReadinessManager;
hub?: SseHub;
}
export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstance {
@@ -39,7 +40,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
dataRoot: config.dataRoot,
});
const mgr = deps?.mgr ?? new PiProcessManager(config, deps?.spawnFn ? { spawnFn: deps.spawnFn } : undefined);
const hub = new SseHub();
const hub = deps?.hub ?? new SseHub();
const readiness = deps?.readiness ?? new ReadinessManager(
tht as ThtRunner,
Math.round(config.ollamaEnsureTimeoutMs / 1000),
+4 -1
View File
@@ -33,7 +33,7 @@ export class SessionBridge {
m.message?.role === "assistant" &&
m.message?.stopReason === "error"
) {
this.state = "failed";
this.markFailed();
this.fan({
type: "info",
level: "error",
@@ -69,6 +69,9 @@ export class SessionBridge {
beginTurn(): void { this.state = "running"; }
/** Record a backend-detected failure; callers own any sanitized client message. */
markFailed(): void { this.state = "failed"; }
onClientEvent(cb: (e: ClientEvent) => void): void { this.cbs.add(cb); }
/** Eventi generati dal backend stesso (es. exit inatteso del child Pi), non da Pi. */
+1
View File
@@ -103,6 +103,7 @@ export class PiProcessManager {
child.on("exit", (code) => {
console.error(`[pi:${sessionId}] exited code=${code ?? "?"} mapped=${this.runtimes.get(sessionId) === rt}`);
if (this.runtimes.get(sessionId) === rt) {
rt.bridge.markFailed();
this.runtimes.delete(sessionId);
rt.bridge.emitClientEvent({
type: "info",
+2 -2
View File
@@ -106,8 +106,6 @@ export function sessionRoutes(
if (state === "running" || state === "waiting") {
return reply.code(200).send({ id, alreadyActive: true });
}
d.mgr.teardown(id);
d.hub.clear(id);
}
const manifest = (await d.tht.sessionShow(id, d.getSettings().workspace)) as { status?: string; archived?: boolean } | null;
if (manifest?.status === "finalized" || manifest?.archived) {
@@ -124,6 +122,8 @@ export function sessionRoutes(
author: getUser(req).id,
mode: "resume" as const,
};
if (existing) d.mgr.teardown(id);
d.hub.clear(id);
await d.tht.reopenSession(id, settings.workspace);
const rt = d.mgr.createFor(id, options);
bindRuntime(id, rt);
+1
View File
@@ -92,6 +92,7 @@ test("un exit INATTESO del child notifica il client (info error + agent_end)", a
const seen: any[] = [];
rt.bridge.onClientEvent((e) => seen.push(e));
child.emit("exit", 137);
expect(rt.bridge.turnState()).toBe("failed");
expect(seen).toEqual([
{ type: "info", level: "error", text: expect.stringContaining("137") },
{ type: "system_event", event: "session_failed" },
+93 -1
View File
@@ -137,18 +137,21 @@ test.each(["running", "waiting"])(
"POST resume preserves a %s runtime",
async (state) => {
let tornDown = false;
let cleared = false;
const existing = { bridge: { turnState: () => state } } as any;
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
mgr: {
get: () => existing,
teardown: () => { tornDown = true; },
} as any,
hub: { clear: () => { cleared = true; } } as any,
thtRunner: {} as any,
getSettings: () => ({ workspace: "psd" }) as any,
});
const response = await app.inject({ method: "POST", url: "/sessions/s1/resume" });
expect(response.json()).toEqual({ id: "s1", alreadyActive: true });
expect(tornDown).toBe(false);
expect(cleared).toBe(false);
},
);
@@ -166,6 +169,10 @@ test.each(["idle", "failed"])(
configure: async () => {},
start: () => order.push("start"),
} as any,
hub: {
clear: (id: string) => order.push(`clear:${id}`),
publish: () => {},
} as any,
thtRunner: {
sessionShow: async () => ({
status: "open", archived: false,
@@ -179,10 +186,95 @@ test.each(["idle", "failed"])(
const response = await app.inject({ method: "POST", url: "/sessions/s1/resume" });
await new Promise((resolve) => setImmediate(resolve));
expect(response.json()).toEqual({ id: "s1" });
expect(order).toEqual(["teardown:s1", "reopen", "create", "start"]);
expect(order).toEqual(["teardown:s1", "clear:s1", "reopen", "create", "start"]);
},
);
test("POST resume without a runtime clears stale SSE state before cold start", async () => {
const order: string[] = [];
let createOptions: any;
const newRuntime = { bridge: { onClientEvent: () => {}, emitClientEvent: () => {} } } as any;
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
mgr: {
get: () => undefined,
createFor: (_id: string, options: any) => {
createOptions = options;
order.push("create");
return newRuntime;
},
configure: async () => {},
start: () => order.push("start"),
} as any,
hub: {
clear: (id: string) => order.push(`clear:${id}`),
publish: () => {},
} as any,
thtRunner: {
sessionShow: async () => ({
status: "open", archived: false,
provider: "local-qwen", model: "qwen3.6-35b-a3b", thinking: "low",
}),
reopenSession: async () => order.push("reopen"),
} as any,
readiness: { ensure: async () => ({ ok: true }) } as any,
getSettings: () => ({ workspace: "local", thinking: "medium" }) as any,
});
const response = await app.inject({ method: "POST", url: "/sessions/crashed/resume" });
await new Promise((resolve) => setImmediate(resolve));
expect(response.json()).toEqual({ id: "crashed" });
expect(order).toEqual(["clear:crashed", "reopen", "create", "start"]);
expect(createOptions).toMatchObject({
provider: "local-qwen", model: "qwen3.6-35b-a3b", thinking: "low", mode: "resume",
});
});
test("POST resume keeps an idle runtime stream attached when the manifest is read-only", async () => {
let tornDown = false;
let cleared = false;
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
mgr: {
get: () => ({ bridge: { turnState: () => "idle" } }),
teardown: () => { tornDown = true; },
} as any,
hub: { clear: () => { cleared = true; } } as any,
thtRunner: {
sessionShow: async () => ({ status: "finalized", archived: false }),
} as any,
getSettings: () => ({ workspace: "local" }) as any,
});
const response = await app.inject({ method: "POST", url: "/sessions/s1/resume" });
expect(response.statusCode).toBe(409);
expect(tornDown).toBe(false);
expect(cleared).toBe(false);
});
test("POST resume keeps a failed runtime stream attached when readiness fails", async () => {
let tornDown = false;
let cleared = false;
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
mgr: {
get: () => ({ bridge: { turnState: () => "failed" } }),
teardown: () => { tornDown = true; },
} as any,
hub: { clear: () => { cleared = true; } } as any,
thtRunner: {
sessionShow: async () => ({ status: "open", archived: false }),
} as any,
readiness: { ensure: async () => ({ ok: false, error: "not ready" }) } as any,
getSettings: () => ({ workspace: "local" }) as any,
});
const response = await app.inject({ method: "POST", url: "/sessions/s1/resume" });
expect(response.statusCode).toBe(503);
expect(tornDown).toBe(false);
expect(cleared).toBe(false);
});
test("POST /sessions/:id/response inoltra al bridge (no error)", async () => {
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
thtRunner: {
+13
View File
@@ -68,6 +68,19 @@ test("assistant provider errors are sanitized and leave the turn failed", () =>
expect(JSON.stringify(seen)).not.toContain("DO_NOT_LEAK");
});
test("markFailed records backend-detected failure without emitting raw detail", () => {
const { rpc } = fakeRpc();
const bridge = new SessionBridge(rpc);
const seen: any[] = [];
bridge.onClientEvent((event) => seen.push(event));
bridge.beginTurn();
bridge.markFailed();
expect(bridge.turnState()).toBe("failed");
expect(seen).toEqual([]);
});
test("reviewer wait and response transition waiting back to running", () => {
const { rpc, fire } = fakeRpc();
const bridge = new SessionBridge(rpc);