fix: serialize session lifecycle transitions

This commit is contained in:
User
2026-07-15 01:37:42 +02:00
parent 08b1f4909e
commit 6747f6f8a0
10 changed files with 1402 additions and 101 deletions
+11 -4
View File
@@ -174,9 +174,16 @@ export class PiProcessManager {
teardown(id: string): void {
const rt = this.runtimes.get(id);
if (rt) {
rt.child.kill();
this.runtimes.delete(id);
}
if (rt) this.teardownIfCurrent(id, rt);
}
/** Remove only the runtime identity the caller observed. */
teardownIfCurrent(id: string, expected: SessionRuntime): boolean {
if (this.runtimes.get(id) !== expected) return false;
// Delete before signalling the child so its asynchronous exit cannot be mistaken for a
// crash, and so a replacement installed by a later lifecycle operation is never targeted.
this.runtimes.delete(id);
expected.child.kill();
return true;
}
}
+78 -28
View File
@@ -27,24 +27,27 @@ export function sessionRoutes(
app: FastifyInstance,
d: { mgr: PiProcessManager; tht: ThtRunner; hub: SseHub; getSettings: () => Settings; readiness: ReadinessManager },
) {
const resumeTails = new Map<string, Promise<void>>();
const lifecycleTails = new Map<string, Promise<void>>();
const boundRuntimes = new Map<
string,
ReturnType<PiProcessManager["createFor"]>
>();
const failurePersistenceClaimed = new WeakSet<
ReturnType<PiProcessManager["createFor"]>
>();
const withResumeLock = async <T>(id: string, work: () => Promise<T>): Promise<T> => {
const previous = resumeTails.get(id) ?? Promise.resolve();
const withSessionLifecycle = async <T>(id: string, work: () => Promise<T>): Promise<T> => {
const previous = lifecycleTails.get(id) ?? Promise.resolve();
let release!: () => void;
const gate = new Promise<void>((resolve) => { release = resolve; });
const tail = previous.then(() => gate);
resumeTails.set(id, tail);
lifecycleTails.set(id, tail);
await previous;
try {
return await work();
} finally {
release();
if (resumeTails.get(id) === tail) resumeTails.delete(id);
if (lifecycleTails.get(id) === tail) lifecycleTails.delete(id);
}
};
@@ -61,7 +64,17 @@ export function sessionRoutes(
// remains bound after an unexpected exit so its public failure events still reach SSE.
if (boundRuntimes.get(id) !== rt) return;
if (e.type === "system_event" && e.event === "session_failed") {
void d.tht.failSession(id, d.getSettings().workspace).catch(() => undefined);
if (!failurePersistenceClaimed.has(rt)) {
failurePersistenceClaimed.add(rt);
void withSessionLifecycle(id, async () => {
// agent_end can release the old binding before this queued work acquires the lock.
// Undefined means no replacement; a different identity means Resume won and the
// old failure must not touch its manifest.
const bound = boundRuntimes.get(id);
if (bound !== undefined && bound !== rt) return;
await d.tht.failSession(id, d.getSettings().workspace).catch(() => undefined);
}).catch(() => undefined);
}
}
d.hub.publish(id, e.type, e);
if (
@@ -91,14 +104,25 @@ export function sessionRoutes(
try {
if (retrieval) info(id, "Preparing retrieval context");
await Promise.all([configure, retrieval]);
if (d.mgr.get(id) !== rt) return;
info(id, "Starting model");
if (d.mgr.get(id) !== rt) return;
start();
} catch {
d.mgr.teardown(id);
void d.tht.failSession(id, d.getSettings().workspace).catch(() => undefined);
rt.bridge.emitClientEvent({ type: "info", level: "error", text: BOOTSTRAP_FAILURE_MESSAGE });
rt.bridge.emitClientEvent({ type: "system_event", event: "session_failed" });
rt.bridge.emitClientEvent({ type: "system_event", event: "agent_end" });
void withSessionLifecycle(id, async () => {
// A bootstrap continuation can settle after Close/Delete or after a replacement was
// installed. Claim only the runtime identity that actually failed; holding the same
// lifecycle lock through persistence prevents a Resume from becoming that failure's
// accidental target.
if (d.mgr.get(id) !== rt || !d.mgr.teardownIfCurrent(id, rt)) return;
if (!failurePersistenceClaimed.has(rt)) {
failurePersistenceClaimed.add(rt);
await d.tht.failSession(id, d.getSettings().workspace).catch(() => undefined);
}
rt.bridge.emitClientEvent({ type: "info", level: "error", text: BOOTSTRAP_FAILURE_MESSAGE });
rt.bridge.emitClientEvent({ type: "system_event", event: "session_failed" });
rt.bridge.emitClientEvent({ type: "system_event", event: "agent_end" });
});
}
})();
};
@@ -158,7 +182,7 @@ export function sessionRoutes(
});
app.post("/sessions/:id/resume", async (req, reply) => {
const id = (req.params as any).id;
return withResumeLock(id, async () => {
return withSessionLifecycle(id, async () => {
// This check belongs inside the per-session lock: a preceding cold Resume may have
// installed a running runtime while this request was waiting.
const existing = d.mgr.get(id);
@@ -192,18 +216,31 @@ export function sessionRoutes(
return reply.code(503).send({ error: RESUME_FAILURE_MESSAGE });
}
// Re-check immediately before the replacement commit. A lifecycle operation that ran
// before this request acquired the lock may have changed or removed the runtime.
const current = d.mgr.get(id);
if (current) {
const state = current.bridge.turnState();
if (state === "running" || state === "waiting") {
return reply.code(200).send({ id, alreadyActive: true });
}
}
let rt: ReturnType<PiProcessManager["createFor"]> | undefined;
try {
if (existing) {
boundRuntimes.delete(id);
d.mgr.teardown(id);
if (current) {
if (boundRuntimes.get(id) === current) boundRuntimes.delete(id);
d.mgr.teardownIfCurrent(id, current);
}
rt = d.mgr.createFor(id, options);
bindRuntime(id, rt);
} catch {
// A created-but-unbound runtime is not usable. The old hub remains attached because
// clear() has not happened yet.
if (rt) d.mgr.teardown(id);
if (rt) {
if (boundRuntimes.get(id) === rt) boundRuntimes.delete(id);
d.mgr.teardownIfCurrent(id, rt);
}
return reply.code(503).send({ error: RESUME_FAILURE_MESSAGE });
}
@@ -217,14 +254,19 @@ export function sessionRoutes(
});
app.post("/sessions/:id/close", async (req) => {
const id = (req.params as { id: string }).id;
try {
await d.tht.closeSession(id, d.getSettings().workspace);
} finally {
return withSessionLifecycle(id, async () => {
// Invalidate the live generation before persistence can yield. Otherwise its deferred
// bootstrap may start Pi while Close is already in progress.
const current = d.mgr.get(id);
boundRuntimes.delete(id);
d.mgr.teardown(id);
d.hub.clear(id);
}
return { closed: true };
if (current) d.mgr.teardownIfCurrent(id, current);
try {
await d.tht.closeSession(id, d.getSettings().workspace);
} finally {
d.hub.clear(id);
}
return { closed: true };
});
});
app.get("/sessions/:id/events", (req, reply) => {
const id = (req.params as any).id;
@@ -252,6 +294,11 @@ export function sessionRoutes(
const off = d.hub.subscribe(id, send, {
afterId,
pending: rt?.bridge.pendingWidget() ?? null,
// clear()/forget() end every old transport so native EventSource reconnects with its
// Last-Event-ID instead of remaining attached to a subscriber callback that no longer exists.
close: () => {
if (!reply.raw.writableEnded) reply.raw.end();
},
});
req.raw.on("close", off);
});
@@ -273,11 +320,14 @@ export function sessionRoutes(
});
app.delete("/sessions/:id", async (req, reply) => {
const id = (req.params as any).id;
boundRuntimes.delete(id);
d.mgr.teardown(id); // drop any live runtime before deleting on disk
await d.tht.deleteSession(id, d.getSettings().workspace);
d.hub.forget(id);
return reply.code(204).send();
return withSessionLifecycle(id, async () => {
const current = d.mgr.get(id);
boundRuntimes.delete(id);
if (current) d.mgr.teardownIfCurrent(id, current);
await d.tht.deleteSession(id, d.getSettings().workspace);
d.hub.forget(id);
return reply.code(204).send();
});
});
app.get("/sessions/:id/documents", async (req) => d.tht.documents((req.params as any).id));
}
+21 -4
View File
@@ -1,5 +1,11 @@
type Send = (event: string, data: object, id: number) => void;
interface Subscriber {
send: Send;
close: () => void;
closed: boolean;
}
/**
* Per-session ring buffer of recent events.
*
@@ -17,6 +23,7 @@ interface BufferedEvent {
interface SubscribeOptions {
afterId?: number;
pending?: object | null;
close?: () => void;
}
function descriptorId(value: unknown): string | null {
@@ -32,7 +39,7 @@ function bufferedGateId(item: BufferedEvent): string | null {
}
export class SseHub {
private subs = new Map<string, Set<Send>>();
private subs = new Map<string, Set<Subscriber>>();
private buffers = new Map<string, BufferedEvent[]>();
private lastIds = new Map<string, number>();
@@ -42,7 +49,8 @@ export class SseHub {
options: SubscribeOptions = {},
): () => void {
if (!this.subs.has(sessionId)) this.subs.set(sessionId, new Set());
this.subs.get(sessionId)!.add(send);
const subscriber = { send, close: options.close ?? (() => undefined), closed: false };
this.subs.get(sessionId)!.add(subscriber);
const afterId = Number.isSafeInteger(options.afterId) && (options.afterId ?? 0) >= 0
? options.afterId ?? 0
@@ -63,12 +71,14 @@ export class SseHub {
send(item.event, item.data, item.id);
}
return () => this.subs.get(sessionId)?.delete(send);
return () => this.subs.get(sessionId)?.delete(subscriber);
}
publish(sessionId: string, event: string, data: object): number {
const item = this.buffer(sessionId, event, data);
for (const send of this.subs.get(sessionId) ?? []) send(event, data, item.id);
for (const subscriber of this.subs.get(sessionId) ?? []) {
subscriber.send(event, data, item.id);
}
return item.id;
}
@@ -88,6 +98,13 @@ export class SseHub {
/** Clear buffered/runtime bindings while retaining session event-id monotonicity. */
clear(sessionId: string): void {
// Snapshot because ending an HTTP response can synchronously/asynchronously unsubscribe it.
// Mark before invoking callbacks so even a re-entrant clear cannot close a response twice.
for (const subscriber of [...(this.subs.get(sessionId) ?? [])]) {
if (subscriber.closed) continue;
subscriber.closed = true;
try { subscriber.close(); } catch { /* disconnect every remaining subscriber */ }
}
this.buffers.delete(sessionId);
this.subs.delete(sessionId);
}
+21
View File
@@ -66,6 +66,27 @@ test("l'exit del VECCHIO child non elimina il nuovo runtime (exit identity-check
mgr.teardown("respawn-id");
});
test("identity-checked teardown cannot kill a replacement runtime", () => {
const cfg = loadConfig({ THT_HARNESS_DIR: "../harness" });
const firstChild = recordingChild();
const secondChild = recordingChild();
firstChild.kill = vi.fn();
secondChild.kill = vi.fn();
const children = [firstChild, secondChild];
const mgr = new PiProcessManager(cfg, { spawnFn: () => children.shift() as any });
const first = mgr.createFor("replace-id", {});
mgr.teardown("replace-id");
const second = mgr.createFor("replace-id", {});
expect(mgr.teardownIfCurrent("replace-id", first)).toBe(false);
expect(mgr.get("replace-id")).toBe(second);
expect(secondChild.kill).not.toHaveBeenCalled();
expect(mgr.teardownIfCurrent("replace-id", second)).toBe(true);
expect(mgr.get("replace-id")).toBeUndefined();
expect(secondChild.kill).toHaveBeenCalledOnce();
});
test("oltre maxPiProcesses solleva errore", async () => {
const cfg = { ...loadConfig({}), maxPiProcesses: 1 };
const mgr = new PiProcessManager(cfg, { spawnFn: () => spawn("node", [FAKE, SCRIPT]) as any });
+579 -24
View File
@@ -18,6 +18,16 @@ function mutApp(thtRunner: any) {
});
}
function deferred<T = void>() {
let resolve!: (value: T | PromiseLike<T>) => void;
let reject!: (reason?: unknown) => void;
const promise = new Promise<T>((onResolve, onReject) => {
resolve = onResolve;
reject = onReject;
});
return { promise, resolve, reject };
}
test("POST /sessions usa i settings (workspace/provider/model/thinking) e crea+avvia", async () => {
const modelKey = path.join(os.tmpdir(), `thoth-model-key-${process.pid}`);
writeFileSync(modelKey, "test-model-key", { mode: 0o600 });
@@ -52,11 +62,17 @@ test("POST /sessions configura Pi con il thinking globale selezionato", async ()
let configured: any;
const bridge = { onClientEvent: () => {}, emitClientEvent: () => {} };
const runtime = { bridge } as any;
let current: any;
const mgr = {
createFor: () => runtime,
get: () => current,
createFor: () => { current = runtime; return runtime; },
configure: async (_rt: any, options: any) => { configured = options; },
start: () => {},
teardown: () => {},
teardownIfCurrent: (_id: string, expected: any) => {
if (current !== expected) return false;
current = undefined;
return true;
},
} as any;
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
mgr,
@@ -78,12 +94,17 @@ test("POST /sessions/:id/resume configura Pi con il thinking persistito", async
let configured: any;
const bridge = { onClientEvent: () => {}, emitClientEvent: () => {} };
const runtime = { bridge } as any;
let current: any;
const mgr = {
get: () => undefined,
createFor: () => runtime,
get: () => current,
createFor: () => { current = runtime; return runtime; },
configure: async (_rt: any, options: any) => { configured = options; },
start: () => {},
teardown: () => {},
teardownIfCurrent: (_id: string, expected: any) => {
if (current !== expected) return false;
current = undefined;
return true;
},
} as any;
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
mgr,
@@ -111,12 +132,17 @@ test("POST /sessions/:id/resume usa il thinking globale se manca nel manifest",
let configured: any;
const bridge = { onClientEvent: () => {}, emitClientEvent: () => {} };
const runtime = { bridge } as any;
let current: any;
const mgr = {
get: () => undefined,
createFor: () => runtime,
get: () => current,
createFor: () => { current = runtime; return runtime; },
configure: async (_rt: any, options: any) => { configured = options; },
start: () => {},
teardown: () => {},
teardownIfCurrent: (_id: string, expected: any) => {
if (current !== expected) return false;
current = undefined;
return true;
},
} as any;
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
mgr,
@@ -162,11 +188,17 @@ test.each(["idle", "failed"])(
const order: string[] = [];
const oldRuntime = { bridge: { turnState: () => state } } as any;
const newRuntime = { bridge: { onClientEvent: () => {}, emitClientEvent: () => {} } } as any;
let current: any = oldRuntime;
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
mgr: {
get: () => oldRuntime,
teardown: (id: string) => order.push(`teardown:${id}`),
createFor: () => { order.push("create"); return newRuntime; },
get: () => current,
teardownIfCurrent: (id: string, expected: any) => {
if (current !== expected) return false;
order.push(`teardown:${id}`);
current = undefined;
return true;
},
createFor: () => { order.push("create"); current = newRuntime; return newRuntime; },
configure: async () => {},
start: () => order.push("start"),
} as any,
@@ -195,12 +227,14 @@ test("POST resume without a runtime clears stale SSE state before cold start", a
const order: string[] = [];
let createOptions: any;
const newRuntime = { bridge: { onClientEvent: () => {}, emitClientEvent: () => {} } } as any;
let current: any;
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
mgr: {
get: () => undefined,
get: () => current,
createFor: (_id: string, options: any) => {
createOptions = options;
order.push("create");
current = newRuntime;
return newRuntime;
},
configure: async () => {},
@@ -327,10 +361,15 @@ test("POST resume sanitizes runtime creation failure and preserves the old hub a
} as any;
hub.subscribe("s1", (_event: string, data: any) => delivered.push(data.text));
hub.publish("s1", "info", { text: "before" });
let current: any = { bridge: { turnState: () => "failed" } };
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
mgr: {
get: () => ({ bridge: { turnState: () => "failed" } }),
teardown: () => {},
get: () => current,
teardownIfCurrent: (_id: string, expected: any) => {
if (current !== expected) return false;
current = undefined;
return true;
},
createFor: () => { createCalls += 1; throw new Error("CREATE_SENTINEL"); },
} as any,
hub,
@@ -424,6 +463,490 @@ test("concurrent cold Resume requests serialize and create one runtime", async (
});
});
test.each([
["close", "resolve"],
["close", "reject"],
["delete", "resolve"],
["delete", "reject"],
] as const)(
"a bootstrap that %s left behind cannot %s against a replacement runtime",
async (lifecycle, outcome) => {
const oldConfigure = deferred<void>();
const published: any[] = [];
const starts: string[] = [];
let failed = 0;
let configureCalls = 0;
let current: any;
const runtime = (name: string) => {
let listener: ((event: any) => void) | undefined;
return {
name,
bridge: {
turnState: () => "running",
onClientEvent: (next: (event: any) => void) => { listener = next; },
emitClientEvent: (event: any) => listener?.(event),
},
};
};
const oldRuntime = runtime("old");
const replacement = runtime("replacement");
const runtimes = [oldRuntime, replacement];
const mgr = {
get: () => current,
createFor: () => {
current = runtimes.shift();
return current;
},
configure: async () => {
configureCalls += 1;
if (configureCalls === 1) await oldConfigure.promise;
},
start: (_id: string, expected: any) => {
if (current !== expected) throw new Error("stale start");
starts.push(expected.name);
},
teardown: () => { current = undefined; },
teardownIfCurrent: (_id: string, expected: any) => {
if (current !== expected) return false;
current = undefined;
return true;
},
};
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
mgr: mgr as any,
hub: {
publish: (_id: string, event: string, data: object) => {
published.push({ event, data });
return published.length;
},
clear: () => {},
forget: () => {},
} as any,
thtRunner: {
sessionNew: async () => ({ id: "s1" }),
searchPack: async () => {},
sessionShow: async () => ({ status: "open", archived: false }),
reopenSession: async () => {},
closeSession: async () => {},
deleteSession: async () => {},
failSession: async () => { failed += 1; },
} as any,
readiness: { ensure: async () => ({ ok: true }) } as any,
getSettings: () => ({ workspace: "local" }) as any,
});
await app.inject({ method: "POST", url: "/sessions", payload: { question: "old" } });
if (lifecycle === "close") {
await app.inject({ method: "POST", url: "/sessions/s1/close" });
await app.inject({ method: "POST", url: "/sessions/s1/resume" });
} else {
await app.inject({ method: "DELETE", url: "/sessions/s1" });
await app.inject({ method: "POST", url: "/sessions", payload: { question: "replacement" } });
}
await new Promise((resolve) => setImmediate(resolve));
expect(current).toBe(replacement);
published.length = 0;
starts.length = 0;
failed = 0;
if (outcome === "resolve") oldConfigure.resolve();
else oldConfigure.reject(new Error("old bootstrap failed"));
await new Promise((resolve) => setImmediate(resolve));
expect(current).toBe(replacement);
expect(starts).toEqual([]);
expect(failed).toBe(0);
expect(published).toEqual([]);
},
);
test.each(["resolve", "reject"] as const)(
"a deleted session stays forgotten when its stale bootstrap later %s",
async (outcome) => {
const oldConfigure = deferred<void>();
const published: any[] = [];
let failed = 0;
let current: any;
let forgotten = false;
let listener: ((event: any) => void) | undefined;
const runtime = {
bridge: {
onClientEvent: (next: (event: any) => void) => { listener = next; },
emitClientEvent: (event: any) => listener?.(event),
},
};
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
mgr: {
get: () => current,
createFor: () => { current = runtime; return runtime; },
configure: async () => oldConfigure.promise,
start: () => { throw new Error("deleted runtime must not start"); },
teardown: () => { current = undefined; },
teardownIfCurrent: (_id: string, expected: any) => {
if (current !== expected) return false;
current = undefined;
return true;
},
} as any,
hub: {
publish: (_id: string, event: string, data: object) => {
published.push({ event, data });
return published.length;
},
forget: () => { forgotten = true; },
} as any,
thtRunner: {
sessionNew: async () => ({ id: "s1" }),
searchPack: async () => {},
deleteSession: async () => {},
failSession: async () => { failed += 1; },
} as any,
readiness: { ensure: async () => ({ ok: true }) } as any,
getSettings: () => ({ workspace: "local" }) as any,
});
await app.inject({ method: "POST", url: "/sessions", payload: { question: "old" } });
await app.inject({ method: "DELETE", url: "/sessions/s1" });
published.length = 0;
if (outcome === "resolve") oldConfigure.resolve();
else oldConfigure.reject(new Error("old bootstrap failed"));
await new Promise((resolve) => setImmediate(resolve));
expect(current).toBeUndefined();
expect(forgotten).toBe(true);
expect(failed).toBe(0);
expect(published).toEqual([]);
},
);
test.each(["close", "delete"] as const)(
"Resume followed by %s leaves no runtime or post-removal bootstrap events",
async (lifecycle) => {
const reopen = deferred<void>();
const configure = deferred<void>();
let markReopenStarted!: () => void;
const reopenStarted = new Promise<void>((resolve) => { markReopenStarted = resolve; });
let current: any;
let removalSettled = false;
const published: any[] = [];
const removalEvents: string[] = [];
const replacement = {
bridge: {
turnState: () => "running",
onClientEvent: () => {},
emitClientEvent: () => {},
},
};
const teardown = (expected?: any) => {
if (expected && current !== expected) return false;
current = undefined;
return true;
};
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
mgr: {
get: () => current,
createFor: () => { current = replacement; return replacement; },
configure: async () => configure.promise,
start: () => { throw new Error("removed runtime must not start"); },
teardown: () => { teardown(); },
teardownIfCurrent: (_id: string, expected: any) => teardown(expected),
} as any,
hub: {
publish: (_id: string, event: string, data: object) => {
published.push({ event, data });
return published.length;
},
clear: () => { removalEvents.push("clear"); },
forget: () => { removalEvents.push("forget"); },
} as any,
thtRunner: {
sessionShow: async () => ({ status: "open", archived: false }),
reopenSession: async () => {
markReopenStarted();
await reopen.promise;
},
closeSession: async () => { removalEvents.push("close"); },
deleteSession: async () => { removalEvents.push("delete"); },
failSession: async () => {},
} as any,
readiness: { ensure: async () => ({ ok: true }) } as any,
getSettings: () => ({ workspace: "local" }) as any,
});
const resumeResponse = app.inject({ method: "POST", url: "/sessions/s1/resume" });
await reopenStarted;
const removalResponse = app.inject({
method: lifecycle === "close" ? "POST" : "DELETE",
url: `/sessions/s1/${lifecycle === "close" ? "close" : ""}`.replace(/\/$/, ""),
}).then((response) => {
removalSettled = true;
return response;
});
await new Promise((resolve) => setImmediate(resolve));
expect(removalSettled).toBe(false);
reopen.resolve();
expect((await resumeResponse).json()).toEqual({ id: "s1", alreadyActive: false });
await removalResponse;
published.length = 0;
configure.resolve();
await new Promise((resolve) => setImmediate(resolve));
expect(current).toBeUndefined();
expect(published).toEqual([]);
expect(removalEvents.at(-1)).toBe(lifecycle === "close" ? "clear" : "forget");
},
);
test("Close followed by Resume installs a fresh runtime only after Close finishes", async () => {
const close = deferred<void>();
let markCloseStarted!: () => void;
const closeStarted = new Promise<void>((resolve) => { markCloseStarted = resolve; });
const oldRuntime = { bridge: { turnState: () => "running" } };
const replacement = {
bridge: {
turnState: () => "running",
onClientEvent: () => {},
emitClientEvent: () => {},
},
};
let current: any = oldRuntime;
let resumeSettled = false;
const teardown = (expected?: any) => {
if (expected && current !== expected) return false;
current = undefined;
return true;
};
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
mgr: {
get: () => current,
createFor: () => { current = replacement; return replacement; },
configure: async () => {},
start: () => {},
teardown: () => { teardown(); },
teardownIfCurrent: (_id: string, expected: any) => teardown(expected),
} as any,
hub: { publish: () => 1, clear: () => {} } as any,
thtRunner: {
closeSession: async () => { markCloseStarted(); await close.promise; },
sessionShow: async () => ({ status: "open", archived: false }),
reopenSession: async () => {},
} as any,
readiness: { ensure: async () => ({ ok: true }) } as any,
getSettings: () => ({ workspace: "local" }) as any,
});
const closeResponse = app.inject({ method: "POST", url: "/sessions/s1/close" });
await closeStarted;
const resumeResponse = app.inject({ method: "POST", url: "/sessions/s1/resume" })
.then((response) => { resumeSettled = true; return response; });
await new Promise((resolve) => setImmediate(resolve));
expect(resumeSettled).toBe(false);
close.resolve();
await closeResponse;
const resumed = await resumeResponse;
await new Promise((resolve) => setImmediate(resolve));
expect(resumed.json()).toEqual({ id: "s1", alreadyActive: false });
expect(current).toBe(replacement);
});
test("Close suppresses a bootstrap that settles while close persistence is pending", async () => {
const configure = deferred<void>();
const close = deferred<void>();
let markCloseStarted!: () => void;
const closeStarted = new Promise<void>((resolve) => { markCloseStarted = resolve; });
const published: any[] = [];
const starts: any[] = [];
let current: any;
let listener: ((event: any) => void) | undefined;
const runtime = {
bridge: {
onClientEvent: (next: (event: any) => void) => { listener = next; },
emitClientEvent: (event: any) => listener?.(event),
},
};
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
mgr: {
get: () => current,
createFor: () => { current = runtime; return runtime; },
configure: async () => configure.promise,
start: (_id: string, expected: any) => { starts.push(expected); },
teardownIfCurrent: (_id: string, expected: any) => {
if (current !== expected) return false;
current = undefined;
return true;
},
} as any,
hub: {
publish: (_id: string, event: string, data: object) => {
published.push({ event, data });
return published.length;
},
clear: () => {},
} as any,
thtRunner: {
sessionNew: async () => ({ id: "s1" }),
searchPack: async () => {},
closeSession: async () => { markCloseStarted(); await close.promise; },
failSession: async () => {},
} as any,
readiness: { ensure: async () => ({ ok: true }) } as any,
getSettings: () => ({ workspace: "local" }) as any,
});
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
const closeResponse = app.inject({ method: "POST", url: "/sessions/s1/close" });
await closeStarted;
published.length = 0;
configure.resolve();
await new Promise((resolve) => setImmediate(resolve));
expect(starts).toEqual([]);
expect(published).toEqual([]);
close.resolve();
await closeResponse;
expect(current).toBeUndefined();
});
test("bootstrap failure persists once and keeps Resume serialized behind that persistence", async () => {
const oldConfigure = deferred<void>();
const failurePersistence = deferred<void>();
let markFailureStarted!: () => void;
const failureStarted = new Promise<void>((resolve) => { markFailureStarted = resolve; });
const oldRuntime = {
bridge: {
onClientEvent: undefined as ((next: (event: any) => void) => void) | undefined,
emitClientEvent: undefined as ((event: any) => void) | undefined,
},
} as any;
const replacement = {
bridge: {
turnState: () => "running",
onClientEvent: () => {},
emitClientEvent: () => {},
},
};
let oldListener: ((event: any) => void) | undefined;
oldRuntime.bridge.onClientEvent = (next: (event: any) => void) => { oldListener = next; };
oldRuntime.bridge.emitClientEvent = (event: any) => oldListener?.(event);
const runtimes = [oldRuntime, replacement];
let current: any;
let configureCalls = 0;
let failCalls = 0;
let resumeSettled = false;
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
mgr: {
get: () => current,
createFor: () => { current = runtimes.shift(); return current; },
configure: async () => {
configureCalls += 1;
if (configureCalls === 1) await oldConfigure.promise;
},
start: () => {},
teardownIfCurrent: (_id: string, expected: any) => {
if (current !== expected) return false;
current = undefined;
return true;
},
} as any,
hub: { publish: () => 1, clear: () => {} } as any,
thtRunner: {
sessionNew: async () => ({ id: "s1" }),
searchPack: async () => {},
sessionShow: async () => ({ status: "open", archived: false }),
reopenSession: async () => {},
failSession: async () => {
failCalls += 1;
if (failCalls === 1) {
markFailureStarted();
await failurePersistence.promise;
}
},
} as any,
readiness: { ensure: async () => ({ ok: true }) } as any,
getSettings: () => ({ workspace: "local" }) as any,
});
await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
oldConfigure.reject(new Error("configure failed"));
await failureStarted;
const resumeResponse = app.inject({ method: "POST", url: "/sessions/s1/resume" })
.then((response) => { resumeSettled = true; return response; });
await new Promise((resolve) => setImmediate(resolve));
expect(resumeSettled).toBe(false);
failurePersistence.resolve();
expect((await resumeResponse).json()).toEqual({ id: "s1", alreadyActive: false });
await new Promise((resolve) => setImmediate(resolve));
expect(failCalls).toBe(1);
expect(current).toBe(replacement);
});
test("Delete followed by Resume cannot resurrect the deleted session", async () => {
const deletion = deferred<void>();
let markDeleteStarted!: () => void;
const deleteStarted = new Promise<void>((resolve) => { markDeleteStarted = resolve; });
const oldRuntime = { bridge: { turnState: () => "running" } };
let current: any = oldRuntime;
let deleted = false;
let resumeSettled = false;
let createCalls = 0;
const teardown = (expected?: any) => {
if (expected && current !== expected) return false;
current = undefined;
return true;
};
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
mgr: {
get: () => current,
createFor: () => {
createCalls += 1;
current = { bridge: { onClientEvent: () => {}, emitClientEvent: () => {} } };
return current;
},
configure: async () => {},
start: () => {},
teardown: () => { teardown(); },
teardownIfCurrent: (_id: string, expected: any) => teardown(expected),
} as any,
hub: { publish: () => 1, clear: () => {}, forget: () => {} } as any,
thtRunner: {
deleteSession: async () => {
markDeleteStarted();
await deletion.promise;
deleted = true;
},
sessionShow: async () => {
if (deleted) throw new Error("session deleted");
return { status: "open", archived: false };
},
reopenSession: async () => {},
} as any,
readiness: { ensure: async () => ({ ok: true }) } as any,
getSettings: () => ({ workspace: "local" }) as any,
});
const deleteResponse = app.inject({ method: "DELETE", url: "/sessions/s1" });
await deleteStarted;
const resumeResponse = app.inject({ method: "POST", url: "/sessions/s1/resume" })
.then((response) => { resumeSettled = true; return response; });
await new Promise((resolve) => setImmediate(resolve));
expect(resumeSettled).toBe(false);
deletion.resolve();
await deleteResponse;
const resumed = await resumeResponse;
expect(resumed.statusCode).toBe(500);
expect(current).toBeUndefined();
expect(createCalls).toBe(0);
});
test("a replaced runtime cannot publish or fail the newly resumed session", async () => {
const controlledBridge = (initialState: string) => {
let state = initialState;
@@ -448,7 +971,11 @@ test("a replaced runtime cannot publish or fail the newly resumed session", asyn
createFor: () => { current = runtimes.shift(); return current; },
configure: async () => {},
start: () => {},
teardown: () => { current = undefined; },
teardownIfCurrent: (_id: string, expected: any) => {
if (current !== expected) return false;
current = undefined;
return true;
},
} as any,
hub: {
clear: () => {},
@@ -505,7 +1032,11 @@ test("a deleted runtime cannot repopulate or fail the forgotten session", async
createFor: () => { current = runtime; return runtime; },
configure: async () => {},
start: () => {},
teardown: () => { current = undefined; },
teardownIfCurrent: (_id: string, expected: any) => {
if (current !== expected) return false;
current = undefined;
return true;
},
} as any,
hub: {
publish: (_id: string, event: string, data: object) => {
@@ -604,7 +1135,12 @@ test("POST resume tears down a created runtime when bridge binding fails", async
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
mgr: {
get: () => current,
teardown: () => { teardownCalls += 1; current = undefined; },
teardownIfCurrent: (_id: string, expected: any) => {
if (current !== expected) return false;
teardownCalls += 1;
current = undefined;
return true;
},
createFor: () => {
current = {
bridge: {
@@ -688,9 +1224,17 @@ test("DELETE /sessions/:id calls deleteSession", async () => {
test("DELETE /sessions/:id tears down the runtime before deleting on disk", async () => {
const order: string[] = [];
const runtime = {} as any;
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
thtRunner: { deleteSession: async (id: string) => { order.push(`del:${id}`); } } as any,
mgr: { teardown: (id: string) => { order.push(`teardown:${id}`); } } as any,
mgr: {
get: () => runtime,
teardownIfCurrent: (id: string, expected: any) => {
expect(expected).toBe(runtime);
order.push(`teardown:${id}`);
return true;
},
} as any,
hub: { forget: (id: string) => { order.push(`forget:${id}`); } } as any,
getSettings: () => ({ workspace: "w" }) as any,
spawnFn: () => nodeSpawn("node", [FAKE, SCRIPT]) as any,
@@ -704,7 +1248,7 @@ test("POST /sessions/:id/close clears transient hub state without forgetting its
const calls: string[] = [];
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
thtRunner: { closeSession: async () => {} } as any,
mgr: { teardown: () => {} } as any,
mgr: { get: () => undefined } as any,
hub: {
clear: (id: string) => { calls.push(`clear:${id}`); },
forget: (id: string) => { calls.push(`forget:${id}`); },
@@ -833,14 +1377,20 @@ test("POST /sessions returns after bridge attachment but starts only after retri
emitClientEvent: () => {},
};
const runtime = { bridge } as any;
let current: any;
const mgr = {
createFor: () => runtime,
get: () => current,
createFor: () => { current = runtime; return runtime; },
configure: async () => {},
start: () => {
expect(bridgeAttached).toBe(true);
started = true;
},
teardown: () => {},
teardownIfCurrent: (_id: string, expected: any) => {
if (current !== expected) return false;
current = undefined;
return true;
},
} as any;
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
mgr,
@@ -871,13 +1421,18 @@ test("POST /sessions bootstrap failure emits only a fixed recovery message", asy
const runtime = { bridge } as any;
const rawFailure =
"connect https://secret.invalid/bootstrap?token=DO_NOT_LEAK using /srv/private/model-key";
let current: any;
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
mgr: {
get: () => undefined,
createFor: () => runtime,
get: () => current,
createFor: () => { current = runtime; return runtime; },
configure: async () => { throw new Error(rawFailure); },
start: () => {},
teardown: () => {},
teardownIfCurrent: (_id: string, expected: any) => {
if (current !== expected) return false;
current = undefined;
return true;
},
} as any,
hub: {
publish: (_id: string, event: string, data: any) => published.push({ event, data }),
+54 -9
View File
@@ -21,14 +21,34 @@ test("publish assigns monotonic ids and a subscriber replays only ids newer than
]);
});
test("clear drops stale subscribers and buffers but preserves the per-session id sequence", () => {
test("clear closes every stale subscriber once and replays post-clear events from the cursor", () => {
const hub = new SseHub();
const stale: any[] = [];
hub.subscribe("s1", (event, data, id) => stale.push({ event, data, id }));
const firstStale: any[] = [];
const secondStale: any[] = [];
const closeCalls = [0, 0];
let offFirst: () => void = () => undefined;
let offSecond: () => void = () => undefined;
offFirst = hub.subscribe(
"s1",
(event, data, id) => firstStale.push({ event, data, id }),
{ close: () => { closeCalls[0] += 1; offFirst(); } },
);
offSecond = hub.subscribe(
"s1",
(event, data, id) => secondStale.push({ event, data, id }),
{ close: () => { closeCalls[1] += 1; offSecond(); } },
);
expect(hub.publish("s1", "info", { text: "before" })).toBe(1);
hub.clear("s1");
offFirst();
offSecond();
expect(closeCalls).toEqual([1, 1]);
expect(hub.publish("s1", "info", { text: "after" })).toBe(2);
expect(hub.publish("s1", "ui_request", {
type: "ui_request",
ui_request: { id: "gate-1", widget: "select" },
})).toBe(3);
const resumed: any[] = [];
hub.subscribe(
"s1",
@@ -36,22 +56,47 @@ test("clear drops stale subscribers and buffers but preserves the per-session id
{ afterId: 1 },
);
expect(stale).toEqual([{ event: "info", data: { text: "before" }, id: 1 }]);
expect(resumed).toEqual([{ event: "info", data: { text: "after" }, id: 2 }]);
const before = [{ event: "info", data: { text: "before" }, id: 1 }];
expect(firstStale).toEqual(before);
expect(secondStale).toEqual(before);
expect(resumed).toEqual([
{ event: "info", data: { text: "after" }, id: 2 },
{
event: "ui_request",
data: { type: "ui_request", ui_request: { id: "gate-1", widget: "select" } },
id: 3,
},
]);
});
test("forget drops subscribers, buffers, and the per-session id sequence", () => {
test("forget closes every subscriber once and resets the per-session id sequence", () => {
const hub = new SseHub();
const stale: any[] = [];
hub.subscribe("s1", (event, data, id) => stale.push({ event, data, id }));
const firstStale: any[] = [];
const secondStale: any[] = [];
const closeCalls = [0, 0];
let offFirst: () => void = () => undefined;
let offSecond: () => void = () => undefined;
offFirst = hub.subscribe(
"s1",
(event, data, id) => firstStale.push({ event, data, id }),
{ close: () => { closeCalls[0] += 1; offFirst(); } },
);
offSecond = hub.subscribe(
"s1",
(event, data, id) => secondStale.push({ event, data, id }),
{ close: () => { closeCalls[1] += 1; offSecond(); } },
);
expect(hub.publish("s1", "info", { text: "before" })).toBe(1);
hub.forget("s1");
expect(closeCalls).toEqual([1, 1]);
expect(hub.publish("s1", "info", { text: "after" })).toBe(1);
const fresh: any[] = [];
hub.subscribe("s1", (event, data, id) => fresh.push({ event, data, id }));
expect(stale).toEqual([{ event: "info", data: { text: "before" }, id: 1 }]);
const before = [{ event: "info", data: { text: "before" }, id: 1 }];
expect(firstStale).toEqual(before);
expect(secondStale).toEqual(before);
expect(fresh).toEqual([{ event: "info", data: { text: "after" }, id: 1 }]);
});
+62
View File
@@ -55,6 +55,17 @@ async function captureReplay(options: { query?: string; lastEventId?: string }):
}
}
async function expectStreamEnd(reader: ReadableStreamDefaultReader<Uint8Array>): Promise<void> {
const result = await Promise.race([
reader.read(),
new Promise<{ timeout: true }>((resolve) =>
setTimeout(() => resolve({ timeout: true }), 1_000),
),
]);
expect(result).not.toEqual({ timeout: true });
expect("done" in result && result.done).toBe(true);
}
test("SSE emits ids and honors the native Last-Event-ID replay cursor", async () => {
const body = await captureReplay({ lastEventId: "1" });
@@ -80,3 +91,54 @@ test("SSE uses the newer valid cursor when header and query are both present", a
expect(body).not.toContain('"two"');
expect(body).toContain("id: 3\nevent: info\n");
});
test("clear ends every live SSE response and a cursor reconnect replays post-clear events", async () => {
const hub = new SseHub();
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/tmp/h" }), {
hub,
thtRunner: {} as any,
});
await app.listen({ port: 0, host: "127.0.0.1" });
const port = (app.server.address() as { port: number }).port;
const controllers = [new AbortController(), new AbortController(), new AbortController()];
try {
const responses = await Promise.all(controllers.slice(0, 2).map((controller) => fetch(
`http://127.0.0.1:${port}/sessions/s1/events`,
{ signal: controller.signal },
)));
const readers = responses.map((response) => response.body!.getReader());
expect(hub.publish("s1", "info", { type: "info", text: "before" })).toBe(1);
const initial = await Promise.all(readers.map((reader) =>
readUntil(reader, (text) => text.includes('"before"'))));
expect(initial.every((body) => body.includes("id: 1\nevent: info\n"))).toBe(true);
hub.clear("s1");
await Promise.all(readers.map(expectStreamEnd));
expect(hub.publish("s1", "info", { type: "info", text: "after" })).toBe(2);
expect(hub.publish("s1", "ui_request", {
type: "ui_request",
ui_request: { id: "gate-1", widget: "select" },
})).toBe(3);
const reconnected = await fetch(
`http://127.0.0.1:${port}/sessions/s1/events`,
{
headers: { "Last-Event-ID": "1" },
signal: controllers[2].signal,
},
);
const reconnectReader = reconnected.body!.getReader();
const replay = await readUntil(reconnectReader, (text) => text.includes('"gate-1"'));
expect(replay).toContain("id: 2\nevent: info\n");
expect(replay).toContain('data: {"type":"info","text":"after"}');
expect(replay).toContain("id: 3\nevent: ui_request\n");
expect(replay).toContain('"ui_request":{"id":"gate-1","widget":"select"}');
await reconnectReader.cancel();
} finally {
for (const controller of controllers) controller.abort();
await app.close();
}
});