fix: make session resume atomic across restarts
This commit is contained in:
@@ -68,9 +68,14 @@ export class PiProcessManager {
|
||||
cwd: this.cfg.harnessDir,
|
||||
env,
|
||||
});
|
||||
// Log stderr for debugging (was silently drained)
|
||||
child.stderr.on("data", (d: Buffer) => console.error(`[pi:${sessionId}] stderr:`, d.toString().trim()));
|
||||
return child;
|
||||
try {
|
||||
// Log stderr for debugging (was silently drained)
|
||||
child.stderr.on("data", (d: Buffer) => console.error(`[pi:${sessionId}] stderr:`, d.toString().trim()));
|
||||
return child;
|
||||
} catch (error) {
|
||||
try { child.kill(); } catch { /* preserve the initialization error */ }
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
count(): number { return this.runtimes.size; }
|
||||
@@ -91,31 +96,39 @@ export class PiProcessManager {
|
||||
const author = o.author ?? "dev@local";
|
||||
const provider = canonicalPiProvider(o.provider ?? this.cfg.defaults.provider);
|
||||
const child = this.spawnFn(sessionId, author, provider);
|
||||
const rpc = new RpcClient(child);
|
||||
const bridge = new SessionBridge(rpc);
|
||||
const rt: SessionRuntime = { rpc, bridge, child };
|
||||
bridge.beginTurn();
|
||||
this.runtimes.set(sessionId, rt);
|
||||
// Identity-checked: a stale child's exit must not evict a newer runtime.
|
||||
// Expected teardowns (teardown()/respawn) delete the runtime from the map BEFORE the
|
||||
// exit event fires, so reaching this branch with `rt` still mapped means the child
|
||||
// died on its own: tell the client, or the UI spins forever waiting for a turn end.
|
||||
child.on("exit", (code) => {
|
||||
console.error(`[pi:${sessionId}] exited code=${code ?? "?"} mapped=${this.runtimes.get(sessionId) === rt}`);
|
||||
if (this.runtimes.get(sessionId) === rt) {
|
||||
rt.bridge.markFailed();
|
||||
this.runtimes.delete(sessionId);
|
||||
rt.bridge.emitClientEvent({
|
||||
type: "info",
|
||||
level: "error",
|
||||
text: `Pi process exited unexpectedly (code ${code ?? "?"})`,
|
||||
});
|
||||
rt.bridge.emitClientEvent({ type: "system_event", event: "session_failed" });
|
||||
rt.bridge.emitClientEvent({ type: "system_event", event: "agent_end" });
|
||||
}
|
||||
});
|
||||
let rt: SessionRuntime | undefined;
|
||||
try {
|
||||
const rpc = new RpcClient(child);
|
||||
const bridge = new SessionBridge(rpc);
|
||||
const runtime: SessionRuntime = { rpc, bridge, child };
|
||||
rt = runtime;
|
||||
bridge.beginTurn();
|
||||
this.runtimes.set(sessionId, runtime);
|
||||
// Identity-checked: a stale child's exit must not evict a newer runtime.
|
||||
// Expected teardowns (teardown()/respawn) delete the runtime from the map BEFORE the
|
||||
// exit event fires, so reaching this branch with `rt` still mapped means the child
|
||||
// died on its own: tell the client, or the UI spins forever waiting for a turn end.
|
||||
child.on("exit", (code) => {
|
||||
console.error(`[pi:${sessionId}] exited code=${code ?? "?"} mapped=${this.runtimes.get(sessionId) === runtime}`);
|
||||
if (this.runtimes.get(sessionId) === runtime) {
|
||||
runtime.bridge.markFailed();
|
||||
this.runtimes.delete(sessionId);
|
||||
runtime.bridge.emitClientEvent({
|
||||
type: "info",
|
||||
level: "error",
|
||||
text: `Pi process exited unexpectedly (code ${code ?? "?"})`,
|
||||
});
|
||||
runtime.bridge.emitClientEvent({ type: "system_event", event: "session_failed" });
|
||||
runtime.bridge.emitClientEvent({ type: "system_event", event: "agent_end" });
|
||||
}
|
||||
});
|
||||
|
||||
return rt;
|
||||
return runtime;
|
||||
} catch (error) {
|
||||
if (rt && this.runtimes.get(sessionId) === rt) this.runtimes.delete(sessionId);
|
||||
try { child.kill(); } catch { /* preserve the initialization error */ }
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/** Configure model and thinking. Safe to run alongside deterministic retrieval. */
|
||||
|
||||
+109
-36
@@ -10,6 +10,8 @@ const BOOTSTRAP_FAILURE_MESSAGE =
|
||||
"Session startup failed. Check configuration and connectivity, then Resume the session.";
|
||||
const READINESS_FAILURE_MESSAGE =
|
||||
"Session services are not ready. Check configuration and connectivity, then try again.";
|
||||
const RESUME_FAILURE_MESSAGE =
|
||||
"Session could not be resumed. Check configuration and connectivity, then try again.";
|
||||
|
||||
function eventCursor(...values: unknown[]): number {
|
||||
let cursor = 0;
|
||||
@@ -25,16 +27,58 @@ export function sessionRoutes(
|
||||
app: FastifyInstance,
|
||||
d: { mgr: PiProcessManager; tht: ThtRunner; hub: SseHub; getSettings: () => Settings; readiness: ReadinessManager },
|
||||
) {
|
||||
const resumeTails = new Map<string, Promise<void>>();
|
||||
const boundRuntimes = new Map<
|
||||
string,
|
||||
ReturnType<PiProcessManager["createFor"]>
|
||||
>();
|
||||
|
||||
const withResumeLock = async <T>(id: string, work: () => Promise<T>): Promise<T> => {
|
||||
const previous = resumeTails.get(id) ?? Promise.resolve();
|
||||
let release!: () => void;
|
||||
const gate = new Promise<void>((resolve) => { release = resolve; });
|
||||
const tail = previous.then(() => gate);
|
||||
resumeTails.set(id, tail);
|
||||
await previous;
|
||||
try {
|
||||
return await work();
|
||||
} finally {
|
||||
release();
|
||||
if (resumeTails.get(id) === tail) resumeTails.delete(id);
|
||||
}
|
||||
};
|
||||
|
||||
const info = (id: string, text: string, level = "info") =>
|
||||
d.hub.publish(id, "info", { type: "info", level, text });
|
||||
|
||||
const bindRuntime = (id: string, rt: ReturnType<PiProcessManager["createFor"]>) =>
|
||||
rt.bridge.onClientEvent((e) => {
|
||||
if (e.type === "system_event" && e.event === "session_failed") {
|
||||
void d.tht.failSession(id, d.getSettings().workspace).catch(() => undefined);
|
||||
}
|
||||
d.hub.publish(id, e.type, e);
|
||||
});
|
||||
const bindRuntime = (id: string, rt: ReturnType<PiProcessManager["createFor"]>) => {
|
||||
const previous = boundRuntimes.get(id);
|
||||
boundRuntimes.set(id, rt);
|
||||
try {
|
||||
rt.bridge.onClientEvent((e) => {
|
||||
// Child termination is asynchronous. Ignore queued events from a runtime once a newer
|
||||
// identity is bound or the session is explicitly closed/deleted. The active identity
|
||||
// remains bound after an unexpected exit so its public failure events still reach SSE.
|
||||
if (boundRuntimes.get(id) !== rt) return;
|
||||
if (e.type === "system_event" && e.event === "session_failed") {
|
||||
void d.tht.failSession(id, d.getSettings().workspace).catch(() => undefined);
|
||||
}
|
||||
d.hub.publish(id, e.type, e);
|
||||
if (
|
||||
e.type === "system_event"
|
||||
&& e.event === "agent_end"
|
||||
&& d.mgr.get(id) !== rt
|
||||
&& boundRuntimes.get(id) === rt
|
||||
) {
|
||||
boundRuntimes.delete(id);
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
if (previous) boundRuntimes.set(id, previous);
|
||||
else if (boundRuntimes.get(id) === rt) boundRuntimes.delete(id);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
const bootstrap = (
|
||||
id: string,
|
||||
@@ -114,42 +158,69 @@ export function sessionRoutes(
|
||||
});
|
||||
app.post("/sessions/:id/resume", async (req, reply) => {
|
||||
const id = (req.params as any).id;
|
||||
const existing = d.mgr.get(id);
|
||||
if (existing) {
|
||||
const state = existing.bridge.turnState();
|
||||
if (state === "running" || state === "waiting") {
|
||||
return reply.code(200).send({ id, alreadyActive: true });
|
||||
return withResumeLock(id, async () => {
|
||||
// This check belongs inside the per-session lock: a preceding cold Resume may have
|
||||
// installed a running runtime while this request was waiting.
|
||||
const existing = d.mgr.get(id);
|
||||
if (existing) {
|
||||
const state = existing.bridge.turnState();
|
||||
if (state === "running" || state === "waiting") {
|
||||
return reply.code(200).send({ id, alreadyActive: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
const manifest = (await d.tht.sessionShow(id, d.getSettings().workspace)) as { status?: string; archived?: boolean } | null;
|
||||
if (manifest?.status === "finalized" || manifest?.archived) {
|
||||
return reply.code(409).send({ error: "sessione in sola lettura (finalizzata o archiviata)" });
|
||||
}
|
||||
const settings = d.getSettings();
|
||||
const ensure = await d.readiness.ensure(settings.workspace ?? "");
|
||||
if (!ensure.ok) return reply.code(503).send({ error: READINESS_FAILURE_MESSAGE });
|
||||
const saved = manifest as { provider?: string; model?: string; thinking?: string } | null;
|
||||
const options = {
|
||||
provider: saved?.provider,
|
||||
model: saved?.model,
|
||||
thinking: saved?.thinking ?? settings.thinking,
|
||||
author: getUser(req).id,
|
||||
mode: "resume" as const,
|
||||
};
|
||||
if (existing) d.mgr.teardown(id);
|
||||
d.hub.clear(id);
|
||||
await d.tht.reopenSession(id, settings.workspace);
|
||||
const rt = d.mgr.createFor(id, options);
|
||||
bindRuntime(id, rt);
|
||||
info(id, "Resuming session");
|
||||
bootstrap(id, rt, d.mgr.configure(rt, options), null, () => d.mgr.start(id, rt, options));
|
||||
return reply.code(200).send({ id, alreadyActive: false });
|
||||
const manifest = (await d.tht.sessionShow(id, d.getSettings().workspace)) as { status?: string; archived?: boolean } | null;
|
||||
if (manifest?.status === "finalized" || manifest?.archived) {
|
||||
return reply.code(409).send({ error: "sessione in sola lettura (finalizzata o archiviata)" });
|
||||
}
|
||||
const settings = d.getSettings();
|
||||
const ensure = await d.readiness.ensure(settings.workspace ?? "");
|
||||
if (!ensure.ok) return reply.code(503).send({ error: READINESS_FAILURE_MESSAGE });
|
||||
const saved = manifest as { provider?: string; model?: string; thinking?: string } | null;
|
||||
const options = {
|
||||
provider: saved?.provider,
|
||||
model: saved?.model,
|
||||
thinking: saved?.thinking ?? settings.thinking,
|
||||
author: getUser(req).id,
|
||||
mode: "resume" as const,
|
||||
};
|
||||
|
||||
// Reopening is validation, not the transport commit point. Keep the old hub intact if
|
||||
// persistence cannot be reopened.
|
||||
try {
|
||||
await d.tht.reopenSession(id, settings.workspace);
|
||||
} catch {
|
||||
return reply.code(503).send({ error: RESUME_FAILURE_MESSAGE });
|
||||
}
|
||||
|
||||
let rt: ReturnType<PiProcessManager["createFor"]> | undefined;
|
||||
try {
|
||||
if (existing) {
|
||||
boundRuntimes.delete(id);
|
||||
d.mgr.teardown(id);
|
||||
}
|
||||
rt = d.mgr.createFor(id, options);
|
||||
bindRuntime(id, rt);
|
||||
} catch {
|
||||
// A created-but-unbound runtime is not usable. The old hub remains attached because
|
||||
// clear() has not happened yet.
|
||||
if (rt) d.mgr.teardown(id);
|
||||
return reply.code(503).send({ error: RESUME_FAILURE_MESSAGE });
|
||||
}
|
||||
|
||||
// Commit the replacement only after reopen + runtime creation/binding succeeded, and
|
||||
// immediately before the first event produced by the new Resume.
|
||||
d.hub.clear(id);
|
||||
info(id, "Resuming session");
|
||||
bootstrap(id, rt, d.mgr.configure(rt, options), null, () => d.mgr.start(id, rt, options));
|
||||
return reply.code(200).send({ id, alreadyActive: false });
|
||||
});
|
||||
});
|
||||
app.post("/sessions/:id/close", async (req) => {
|
||||
const id = (req.params as { id: string }).id;
|
||||
try {
|
||||
await d.tht.closeSession(id, d.getSettings().workspace);
|
||||
} finally {
|
||||
boundRuntimes.delete(id);
|
||||
d.mgr.teardown(id);
|
||||
d.hub.clear(id);
|
||||
}
|
||||
@@ -202,8 +273,10 @@ export function sessionRoutes(
|
||||
});
|
||||
app.delete("/sessions/:id", async (req, reply) => {
|
||||
const id = (req.params as any).id;
|
||||
boundRuntimes.delete(id);
|
||||
d.mgr.teardown(id); // drop any live runtime before deleting on disk
|
||||
await d.tht.deleteSession(id, d.getSettings().workspace);
|
||||
d.hub.forget(id);
|
||||
return reply.code(204).send();
|
||||
});
|
||||
app.get("/sessions/:id/documents", async (req) => d.tht.documents((req.params as any).id));
|
||||
|
||||
@@ -91,4 +91,10 @@ export class SseHub {
|
||||
this.buffers.delete(sessionId);
|
||||
this.subs.delete(sessionId);
|
||||
}
|
||||
|
||||
/** Permanently discard transport state for a deleted session id. */
|
||||
forget(sessionId: string): void {
|
||||
this.clear(sessionId);
|
||||
this.lastIds.delete(sessionId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,6 +84,36 @@ function recordingChild() {
|
||||
return ch;
|
||||
}
|
||||
|
||||
test("createFor kills a spawned child when post-spawn initialization throws", () => {
|
||||
const child = recordingChild();
|
||||
child.kill = vi.fn();
|
||||
child.stdout = {
|
||||
on: () => { throw new Error("READER_INIT_SENTINEL"); },
|
||||
};
|
||||
const mgr = new PiProcessManager(loadConfig({}), { spawnFn: () => child as any });
|
||||
|
||||
expect(() => mgr.createFor("broken-init", {})).toThrow("READER_INIT_SENTINEL");
|
||||
|
||||
expect(child.kill).toHaveBeenCalledOnce();
|
||||
expect(mgr.get("broken-init")).toBeUndefined();
|
||||
expect(mgr.count()).toBe(0);
|
||||
});
|
||||
|
||||
test("createFor kills a spawned child when spawn boundary initialization throws", () => {
|
||||
const child = recordingChild();
|
||||
child.kill = vi.fn();
|
||||
child.stderr = {
|
||||
on: () => { throw new Error("STDERR_INIT_SENTINEL"); },
|
||||
};
|
||||
const mgr = new PiProcessManager(loadConfig({}), { spawnFn: () => child as any });
|
||||
|
||||
expect(() => mgr.createFor("broken-spawn-init", {})).toThrow("STDERR_INIT_SENTINEL");
|
||||
|
||||
expect(child.kill).toHaveBeenCalledOnce();
|
||||
expect(mgr.get("broken-spawn-init")).toBeUndefined();
|
||||
expect(mgr.count()).toBe(0);
|
||||
});
|
||||
|
||||
test("un exit INATTESO del child notifica il client (info error + agent_end)", async () => {
|
||||
const cfg = loadConfig({});
|
||||
const child = recordingChild();
|
||||
|
||||
@@ -5,6 +5,7 @@ import os from "node:os";
|
||||
import { chmodSync, unlinkSync, writeFileSync } from "node:fs";
|
||||
import { buildApp } from "../src/app.js";
|
||||
import { loadConfig } from "../src/config.js";
|
||||
import { SseHub } from "../src/sse/sse-hub.js";
|
||||
|
||||
const FAKE = path.resolve("../harness/tests/fake_pi/fake_pi_rpc.mjs");
|
||||
const SCRIPT = path.resolve("../harness/tests/fake_pi/scripts/f1_disambiguation.json");
|
||||
@@ -186,7 +187,7 @@ test.each(["idle", "failed"])(
|
||||
const response = await app.inject({ method: "POST", url: "/sessions/s1/resume" });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(response.json()).toEqual({ id: "s1", alreadyActive: false });
|
||||
expect(order).toEqual(["teardown:s1", "clear:s1", "reopen", "create", "start"]);
|
||||
expect(order).toEqual(["reopen", "teardown:s1", "create", "clear:s1", "start"]);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -224,7 +225,7 @@ test("POST resume without a runtime clears stale SSE state before cold start", a
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(response.json()).toEqual({ id: "crashed", alreadyActive: false });
|
||||
expect(order).toEqual(["clear:crashed", "reopen", "create", "start"]);
|
||||
expect(order).toEqual(["reopen", "create", "clear:crashed", "start"]);
|
||||
expect(createOptions).toMatchObject({
|
||||
provider: "local-qwen", model: "qwen3.6-35b-a3b", thinking: "low", mode: "resume",
|
||||
});
|
||||
@@ -275,6 +276,364 @@ test("POST resume keeps a failed runtime stream attached when readiness fails",
|
||||
expect(cleared).toBe(false);
|
||||
});
|
||||
|
||||
test("POST resume sanitizes reopen failure and preserves the old hub attachment", async () => {
|
||||
const delivered: string[] = [];
|
||||
const actualHub = new SseHub();
|
||||
let clearCalls = 0;
|
||||
let tornDown = false;
|
||||
const hub = {
|
||||
subscribe: actualHub.subscribe.bind(actualHub),
|
||||
publish: actualHub.publish.bind(actualHub),
|
||||
clear: (id: string) => { clearCalls += 1; actualHub.clear(id); },
|
||||
} as any;
|
||||
hub.subscribe("s1", (_event: string, data: any) => delivered.push(data.text));
|
||||
hub.publish("s1", "info", { text: "before" });
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
mgr: {
|
||||
get: () => ({ bridge: { turnState: () => "idle" } }),
|
||||
teardown: () => { tornDown = true; },
|
||||
} as any,
|
||||
hub,
|
||||
thtRunner: {
|
||||
sessionShow: async () => ({ status: "open", archived: false }),
|
||||
reopenSession: async () => { throw new Error("REOPEN_SENTINEL"); },
|
||||
} as any,
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
|
||||
const response = await app.inject({ method: "POST", url: "/sessions/s1/resume" });
|
||||
hub.publish("s1", "info", { text: "post-failure probe" });
|
||||
|
||||
expect(response.statusCode).toBe(503);
|
||||
expect(response.json()).toEqual({
|
||||
error: "Session could not be resumed. Check configuration and connectivity, then try again.",
|
||||
});
|
||||
expect(response.body).not.toContain("REOPEN_SENTINEL");
|
||||
expect(clearCalls).toBe(0);
|
||||
expect(tornDown).toBe(false);
|
||||
expect(delivered).toEqual(["before", "post-failure probe"]);
|
||||
});
|
||||
|
||||
test("POST resume sanitizes runtime creation failure and preserves the old hub attachment", async () => {
|
||||
const delivered: string[] = [];
|
||||
const actualHub = new SseHub();
|
||||
let clearCalls = 0;
|
||||
let createCalls = 0;
|
||||
const hub = {
|
||||
subscribe: actualHub.subscribe.bind(actualHub),
|
||||
publish: actualHub.publish.bind(actualHub),
|
||||
clear: (id: string) => { clearCalls += 1; actualHub.clear(id); },
|
||||
} as any;
|
||||
hub.subscribe("s1", (_event: string, data: any) => delivered.push(data.text));
|
||||
hub.publish("s1", "info", { text: "before" });
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
mgr: {
|
||||
get: () => ({ bridge: { turnState: () => "failed" } }),
|
||||
teardown: () => {},
|
||||
createFor: () => { createCalls += 1; throw new Error("CREATE_SENTINEL"); },
|
||||
} as any,
|
||||
hub,
|
||||
thtRunner: {
|
||||
sessionShow: async () => ({ status: "open", archived: false }),
|
||||
reopenSession: async () => {},
|
||||
} as any,
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
|
||||
const response = await app.inject({ method: "POST", url: "/sessions/s1/resume" });
|
||||
hub.publish("s1", "info", { text: "post-failure probe" });
|
||||
|
||||
expect(response.statusCode).toBe(503);
|
||||
expect(response.json()).toEqual({
|
||||
error: "Session could not be resumed. Check configuration and connectivity, then try again.",
|
||||
});
|
||||
expect(response.body).not.toContain("CREATE_SENTINEL");
|
||||
expect(createCalls).toBe(1);
|
||||
expect(clearCalls).toBe(0);
|
||||
expect(delivered).toEqual(["before", "post-failure probe"]);
|
||||
});
|
||||
|
||||
test("concurrent cold Resume requests serialize and create one runtime", async () => {
|
||||
let runtime: any;
|
||||
let showCalls = 0;
|
||||
let readinessCalls = 0;
|
||||
let reopenCalls = 0;
|
||||
let createCalls = 0;
|
||||
let clearCalls = 0;
|
||||
let releaseReopen!: () => void;
|
||||
let markReopenStarted!: () => void;
|
||||
const reopenStarted = new Promise<void>((resolve) => { markReopenStarted = resolve; });
|
||||
const reopenReleased = new Promise<void>((resolve) => { releaseReopen = resolve; });
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
mgr: {
|
||||
get: () => runtime,
|
||||
createFor: () => {
|
||||
createCalls += 1;
|
||||
runtime = {
|
||||
bridge: {
|
||||
turnState: () => "running",
|
||||
onClientEvent: () => {},
|
||||
emitClientEvent: () => {},
|
||||
},
|
||||
};
|
||||
return runtime;
|
||||
},
|
||||
configure: async () => {},
|
||||
start: () => {},
|
||||
teardown: () => {},
|
||||
} as any,
|
||||
hub: {
|
||||
clear: () => { clearCalls += 1; },
|
||||
publish: () => {},
|
||||
} as any,
|
||||
thtRunner: {
|
||||
sessionShow: async () => {
|
||||
showCalls += 1;
|
||||
return { status: "open", archived: false };
|
||||
},
|
||||
reopenSession: async () => {
|
||||
reopenCalls += 1;
|
||||
markReopenStarted();
|
||||
await reopenReleased;
|
||||
},
|
||||
} as any,
|
||||
readiness: {
|
||||
ensure: async () => { readinessCalls += 1; return { ok: true }; },
|
||||
} as any,
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
|
||||
const first = app.inject({ method: "POST", url: "/sessions/s1/resume" });
|
||||
await reopenStarted;
|
||||
const second = app.inject({ method: "POST", url: "/sessions/s1/resume" });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
releaseReopen();
|
||||
const [firstResponse, secondResponse] = await Promise.all([first, second]);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(firstResponse.json()).toEqual({ id: "s1", alreadyActive: false });
|
||||
expect(secondResponse.json()).toEqual({ id: "s1", alreadyActive: true });
|
||||
expect({ showCalls, readinessCalls, reopenCalls, createCalls, clearCalls }).toEqual({
|
||||
showCalls: 1,
|
||||
readinessCalls: 1,
|
||||
reopenCalls: 1,
|
||||
createCalls: 1,
|
||||
clearCalls: 1,
|
||||
});
|
||||
});
|
||||
|
||||
test("a replaced runtime cannot publish or fail the newly resumed session", async () => {
|
||||
const controlledBridge = (initialState: string) => {
|
||||
let state = initialState;
|
||||
let listener: ((event: any) => void) | undefined;
|
||||
return {
|
||||
turnState: () => state,
|
||||
setState: (next: string) => { state = next; },
|
||||
onClientEvent: (next: (event: any) => void) => { listener = next; },
|
||||
emitClientEvent: (event: any) => listener?.(event),
|
||||
pendingWidget: () => null,
|
||||
};
|
||||
};
|
||||
const oldBridge = controlledBridge("running");
|
||||
const newBridge = controlledBridge("running");
|
||||
const runtimes = [{ bridge: oldBridge }, { bridge: newBridge }];
|
||||
let current: any;
|
||||
let failed = 0;
|
||||
const published: any[] = [];
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
mgr: {
|
||||
get: () => current,
|
||||
createFor: () => { current = runtimes.shift(); return current; },
|
||||
configure: async () => {},
|
||||
start: () => {},
|
||||
teardown: () => { current = undefined; },
|
||||
} as any,
|
||||
hub: {
|
||||
clear: () => {},
|
||||
publish: (_id: string, event: string, data: object) => {
|
||||
published.push({ event, data });
|
||||
return published.length;
|
||||
},
|
||||
} as any,
|
||||
thtRunner: {
|
||||
sessionNew: async () => ({ id: "s1" }),
|
||||
searchPack: async () => {},
|
||||
sessionShow: async () => ({ status: "open", archived: false }),
|
||||
reopenSession: async () => {},
|
||||
failSession: async () => { failed += 1; },
|
||||
} as any,
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
oldBridge.setState("idle");
|
||||
|
||||
const response = await app.inject({ method: "POST", url: "/sessions/s1/resume" });
|
||||
published.length = 0;
|
||||
// The manager removes an unexpectedly exited active runtime before its bridge publishes the
|
||||
// public failure sequence. Binding identity, rather than mgr.get(), must still admit it.
|
||||
current = undefined;
|
||||
oldBridge.emitClientEvent({ type: "system_event", event: "session_failed" });
|
||||
newBridge.emitClientEvent({ type: "info", text: "new runtime" });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(response.json()).toEqual({ id: "s1", alreadyActive: false });
|
||||
expect(failed).toBe(0);
|
||||
expect(published).toEqual([{
|
||||
event: "info",
|
||||
data: { type: "info", text: "new runtime" },
|
||||
}]);
|
||||
});
|
||||
|
||||
test("a deleted runtime cannot repopulate or fail the forgotten session", async () => {
|
||||
let listener: ((event: any) => void) | undefined;
|
||||
const bridge = {
|
||||
turnState: () => "running",
|
||||
onClientEvent: (next: (event: any) => void) => { listener = next; },
|
||||
emitClientEvent: (event: any) => listener?.(event),
|
||||
};
|
||||
const runtime = { bridge };
|
||||
let current: any;
|
||||
let failed = 0;
|
||||
const published: any[] = [];
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
mgr: {
|
||||
get: () => current,
|
||||
createFor: () => { current = runtime; return runtime; },
|
||||
configure: async () => {},
|
||||
start: () => {},
|
||||
teardown: () => { current = undefined; },
|
||||
} as any,
|
||||
hub: {
|
||||
publish: (_id: string, event: string, data: object) => {
|
||||
published.push({ event, data });
|
||||
return published.length;
|
||||
},
|
||||
forget: () => {},
|
||||
} as any,
|
||||
thtRunner: {
|
||||
sessionNew: async () => ({ id: "s1" }),
|
||||
searchPack: async () => {},
|
||||
deleteSession: async () => {},
|
||||
failSession: async () => { failed += 1; },
|
||||
} as any,
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
const response = await app.inject({ method: "DELETE", url: "/sessions/s1" });
|
||||
published.length = 0;
|
||||
bridge.emitClientEvent({ type: "system_event", event: "session_failed" });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(response.statusCode).toBe(204);
|
||||
expect(failed).toBe(0);
|
||||
expect(published).toEqual([]);
|
||||
});
|
||||
|
||||
test("an unexpectedly exited runtime publishes its terminal sequence then releases its binding", async () => {
|
||||
let listener: ((event: any) => void) | undefined;
|
||||
const bridge = {
|
||||
onClientEvent: (next: (event: any) => void) => { listener = next; },
|
||||
emitClientEvent: (event: any) => listener?.(event),
|
||||
};
|
||||
const runtime = { bridge };
|
||||
let current: any;
|
||||
let failed = 0;
|
||||
const published: any[] = [];
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
mgr: {
|
||||
get: () => current,
|
||||
createFor: () => { current = runtime; return runtime; },
|
||||
configure: async () => {},
|
||||
start: () => {},
|
||||
teardown: () => { current = undefined; },
|
||||
} as any,
|
||||
hub: {
|
||||
publish: (_id: string, event: string, data: object) => {
|
||||
published.push({ event, data });
|
||||
return published.length;
|
||||
},
|
||||
} as any,
|
||||
thtRunner: {
|
||||
sessionNew: async () => ({ id: "s1" }),
|
||||
searchPack: async () => {},
|
||||
failSession: async () => { failed += 1; },
|
||||
} as any,
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
published.length = 0;
|
||||
|
||||
// PiProcessManager deletes an unexpectedly exited runtime before emitting this sequence.
|
||||
current = undefined;
|
||||
bridge.emitClientEvent({ type: "info", level: "error", text: "public failure" });
|
||||
bridge.emitClientEvent({ type: "system_event", event: "session_failed" });
|
||||
bridge.emitClientEvent({ type: "system_event", event: "agent_end" });
|
||||
bridge.emitClientEvent({ type: "text_delta", text: "too late" });
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(failed).toBe(1);
|
||||
expect(published).toEqual([
|
||||
{ event: "info", data: { type: "info", level: "error", text: "public failure" } },
|
||||
{ event: "system_event", data: { type: "system_event", event: "session_failed" } },
|
||||
{ event: "system_event", data: { type: "system_event", event: "agent_end" } },
|
||||
]);
|
||||
});
|
||||
|
||||
test("POST resume tears down a created runtime when bridge binding fails", async () => {
|
||||
const delivered: string[] = [];
|
||||
const actualHub = new SseHub();
|
||||
let clearCalls = 0;
|
||||
let teardownCalls = 0;
|
||||
let current: any = { bridge: { turnState: () => "failed" } };
|
||||
const hub = {
|
||||
subscribe: actualHub.subscribe.bind(actualHub),
|
||||
publish: actualHub.publish.bind(actualHub),
|
||||
clear: (id: string) => { clearCalls += 1; actualHub.clear(id); },
|
||||
} as any;
|
||||
hub.subscribe("s1", (_event: string, data: any) => delivered.push(data.text));
|
||||
hub.publish("s1", "info", { text: "before" });
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
mgr: {
|
||||
get: () => current,
|
||||
teardown: () => { teardownCalls += 1; current = undefined; },
|
||||
createFor: () => {
|
||||
current = {
|
||||
bridge: {
|
||||
onClientEvent: () => { throw new Error("BIND_SENTINEL"); },
|
||||
},
|
||||
};
|
||||
return current;
|
||||
},
|
||||
} as any,
|
||||
hub,
|
||||
thtRunner: {
|
||||
sessionShow: async () => ({ status: "open", archived: false }),
|
||||
reopenSession: async () => {},
|
||||
} as any,
|
||||
readiness: { ensure: async () => ({ ok: true }) } as any,
|
||||
getSettings: () => ({ workspace: "local" }) as any,
|
||||
});
|
||||
|
||||
const response = await app.inject({ method: "POST", url: "/sessions/s1/resume" });
|
||||
hub.publish("s1", "info", { text: "post-failure probe" });
|
||||
|
||||
expect(response.statusCode).toBe(503);
|
||||
expect(response.body).not.toContain("BIND_SENTINEL");
|
||||
expect(teardownCalls).toBe(2);
|
||||
expect(current).toBeUndefined();
|
||||
expect(clearCalls).toBe(0);
|
||||
expect(delivered).toEqual(["before", "post-failure probe"]);
|
||||
});
|
||||
|
||||
test("POST /sessions/:id/response inoltra al bridge (no error)", async () => {
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: {
|
||||
@@ -332,12 +691,31 @@ test("DELETE /sessions/:id tears down the runtime before deleting on disk", asyn
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: { deleteSession: async (id: string) => { order.push(`del:${id}`); } } as any,
|
||||
mgr: { teardown: (id: string) => { order.push(`teardown:${id}`); } } as any,
|
||||
hub: { forget: (id: string) => { order.push(`forget:${id}`); } } as any,
|
||||
getSettings: () => ({ workspace: "w" }) as any,
|
||||
spawnFn: () => nodeSpawn("node", [FAKE, SCRIPT]) as any,
|
||||
});
|
||||
const res = await app.inject({ method: "DELETE", url: "/sessions/s1" });
|
||||
expect(res.statusCode).toBe(204);
|
||||
expect(order).toEqual(["teardown:s1", "del:s1"]);
|
||||
expect(order).toEqual(["teardown:s1", "del:s1", "forget:s1"]);
|
||||
});
|
||||
|
||||
test("POST /sessions/:id/close clears transient hub state without forgetting its id", async () => {
|
||||
const calls: string[] = [];
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
thtRunner: { closeSession: async () => {} } as any,
|
||||
mgr: { teardown: () => {} } as any,
|
||||
hub: {
|
||||
clear: (id: string) => { calls.push(`clear:${id}`); },
|
||||
forget: (id: string) => { calls.push(`forget:${id}`); },
|
||||
} as any,
|
||||
getSettings: () => ({ workspace: "w" }) as any,
|
||||
});
|
||||
|
||||
const response = await app.inject({ method: "POST", url: "/sessions/s1/close" });
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(calls).toEqual(["clear:s1"]);
|
||||
});
|
||||
|
||||
test("GET /sessions/:id/documents returns the runner output", async () => {
|
||||
@@ -495,6 +873,7 @@ test("POST /sessions bootstrap failure emits only a fixed recovery message", asy
|
||||
"connect https://secret.invalid/bootstrap?token=DO_NOT_LEAK using /srv/private/model-key";
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
|
||||
mgr: {
|
||||
get: () => undefined,
|
||||
createFor: () => runtime,
|
||||
configure: async () => { throw new Error(rawFailure); },
|
||||
start: () => {},
|
||||
|
||||
@@ -40,6 +40,21 @@ test("clear drops stale subscribers and buffers but preserves the per-session id
|
||||
expect(resumed).toEqual([{ event: "info", data: { text: "after" }, id: 2 }]);
|
||||
});
|
||||
|
||||
test("forget drops subscribers, buffers, and the per-session id sequence", () => {
|
||||
const hub = new SseHub();
|
||||
const stale: any[] = [];
|
||||
hub.subscribe("s1", (event, data, id) => stale.push({ event, data, id }));
|
||||
expect(hub.publish("s1", "info", { text: "before" })).toBe(1);
|
||||
|
||||
hub.forget("s1");
|
||||
expect(hub.publish("s1", "info", { text: "after" })).toBe(1);
|
||||
const fresh: any[] = [];
|
||||
hub.subscribe("s1", (event, data, id) => fresh.push({ event, data, id }));
|
||||
|
||||
expect(stale).toEqual([{ event: "info", data: { text: "before" }, id: 1 }]);
|
||||
expect(fresh).toEqual([{ event: "info", data: { text: "after" }, id: 1 }]);
|
||||
});
|
||||
|
||||
test("a buffered pending gate is emitted exactly once and matched by descriptor id", () => {
|
||||
const hub = new SseHub();
|
||||
const descriptor = { id: "gate-1", widget: "select", title: "Original" };
|
||||
|
||||
Reference in New Issue
Block a user