fix: make session resume atomic across restarts

2026-07-15 00:42:35 +02:00
parent 2b4797f133
commit 08b1f4909e
12 changed files with 1082 additions and 107 deletions
@@ -5,6 +5,7 @@ import os from "node:os";
 import { chmodSync, unlinkSync, writeFileSync } from "node:fs";
 import { buildApp } from "../src/app.js";
 import { loadConfig } from "../src/config.js";
+import { SseHub } from "../src/sse/sse-hub.js";

 const FAKE = path.resolve("../harness/tests/fake_pi/fake_pi_rpc.mjs");
 const SCRIPT = path.resolve("../harness/tests/fake_pi/scripts/f1_disambiguation.json");
@@ -186,7 +187,7 @@ test.each(["idle", "failed"])(
    const response = await app.inject({ method: "POST", url: "/sessions/s1/resume" });
    await new Promise((resolve) => setImmediate(resolve));
    expect(response.json()).toEqual({ id: "s1", alreadyActive: false });
-    expect(order).toEqual(["teardown:s1", "clear:s1", "reopen", "create", "start"]);
+    expect(order).toEqual(["reopen", "teardown:s1", "create", "clear:s1", "start"]);
  },
 );

@@ -224,7 +225,7 @@ test("POST resume without a runtime clears stale SSE state before cold start", a
  await new Promise((resolve) => setImmediate(resolve));

  expect(response.json()).toEqual({ id: "crashed", alreadyActive: false });
-  expect(order).toEqual(["clear:crashed", "reopen", "create", "start"]);
+  expect(order).toEqual(["reopen", "create", "clear:crashed", "start"]);
  expect(createOptions).toMatchObject({
    provider: "local-qwen", model: "qwen3.6-35b-a3b", thinking: "low", mode: "resume",
  });
@@ -275,6 +276,364 @@ test("POST resume keeps a failed runtime stream attached when readiness fails",
  expect(cleared).toBe(false);
 });

+test("POST resume sanitizes reopen failure and preserves the old hub attachment", async () => {
+  const delivered: string[] = [];
+  const actualHub = new SseHub();
+  let clearCalls = 0;
+  let tornDown = false;
+  const hub = {
+    subscribe: actualHub.subscribe.bind(actualHub),
+    publish: actualHub.publish.bind(actualHub),
+    clear: (id: string) => { clearCalls += 1; actualHub.clear(id); },
+  } as any;
+  hub.subscribe("s1", (_event: string, data: any) => delivered.push(data.text));
+  hub.publish("s1", "info", { text: "before" });
+  const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
+    mgr: {
+      get: () => ({ bridge: { turnState: () => "idle" } }),
+      teardown: () => { tornDown = true; },
+    } as any,
+    hub,
+    thtRunner: {
+      sessionShow: async () => ({ status: "open", archived: false }),
+      reopenSession: async () => { throw new Error("REOPEN_SENTINEL"); },
+    } as any,
+    readiness: { ensure: async () => ({ ok: true }) } as any,
+    getSettings: () => ({ workspace: "local" }) as any,
+  });
+
+  const response = await app.inject({ method: "POST", url: "/sessions/s1/resume" });
+  hub.publish("s1", "info", { text: "post-failure probe" });
+
+  expect(response.statusCode).toBe(503);
+  expect(response.json()).toEqual({
+    error: "Session could not be resumed. Check configuration and connectivity, then try again.",
+  });
+  expect(response.body).not.toContain("REOPEN_SENTINEL");
+  expect(clearCalls).toBe(0);
+  expect(tornDown).toBe(false);
+  expect(delivered).toEqual(["before", "post-failure probe"]);
+});
+
+test("POST resume sanitizes runtime creation failure and preserves the old hub attachment", async () => {
+  const delivered: string[] = [];
+  const actualHub = new SseHub();
+  let clearCalls = 0;
+  let createCalls = 0;
+  const hub = {
+    subscribe: actualHub.subscribe.bind(actualHub),
+    publish: actualHub.publish.bind(actualHub),
+    clear: (id: string) => { clearCalls += 1; actualHub.clear(id); },
+  } as any;
+  hub.subscribe("s1", (_event: string, data: any) => delivered.push(data.text));
+  hub.publish("s1", "info", { text: "before" });
+  const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
+    mgr: {
+      get: () => ({ bridge: { turnState: () => "failed" } }),
+      teardown: () => {},
+      createFor: () => { createCalls += 1; throw new Error("CREATE_SENTINEL"); },
+    } as any,
+    hub,
+    thtRunner: {
+      sessionShow: async () => ({ status: "open", archived: false }),
+      reopenSession: async () => {},
+    } as any,
+    readiness: { ensure: async () => ({ ok: true }) } as any,
+    getSettings: () => ({ workspace: "local" }) as any,
+  });
+
+  const response = await app.inject({ method: "POST", url: "/sessions/s1/resume" });
+  hub.publish("s1", "info", { text: "post-failure probe" });
+
+  expect(response.statusCode).toBe(503);
+  expect(response.json()).toEqual({
+    error: "Session could not be resumed. Check configuration and connectivity, then try again.",
+  });
+  expect(response.body).not.toContain("CREATE_SENTINEL");
+  expect(createCalls).toBe(1);
+  expect(clearCalls).toBe(0);
+  expect(delivered).toEqual(["before", "post-failure probe"]);
+});
+
+test("concurrent cold Resume requests serialize and create one runtime", async () => {
+  let runtime: any;
+  let showCalls = 0;
+  let readinessCalls = 0;
+  let reopenCalls = 0;
+  let createCalls = 0;
+  let clearCalls = 0;
+  let releaseReopen!: () => void;
+  let markReopenStarted!: () => void;
+  const reopenStarted = new Promise<void>((resolve) => { markReopenStarted = resolve; });
+  const reopenReleased = new Promise<void>((resolve) => { releaseReopen = resolve; });
+  const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
+    mgr: {
+      get: () => runtime,
+      createFor: () => {
+        createCalls += 1;
+        runtime = {
+          bridge: {
+            turnState: () => "running",
+            onClientEvent: () => {},
+            emitClientEvent: () => {},
+          },
+        };
+        return runtime;
+      },
+      configure: async () => {},
+      start: () => {},
+      teardown: () => {},
+    } as any,
+    hub: {
+      clear: () => { clearCalls += 1; },
+      publish: () => {},
+    } as any,
+    thtRunner: {
+      sessionShow: async () => {
+        showCalls += 1;
+        return { status: "open", archived: false };
+      },
+      reopenSession: async () => {
+        reopenCalls += 1;
+        markReopenStarted();
+        await reopenReleased;
+      },
+    } as any,
+    readiness: {
+      ensure: async () => { readinessCalls += 1; return { ok: true }; },
+    } as any,
+    getSettings: () => ({ workspace: "local" }) as any,
+  });
+
+  const first = app.inject({ method: "POST", url: "/sessions/s1/resume" });
+  await reopenStarted;
+  const second = app.inject({ method: "POST", url: "/sessions/s1/resume" });
+  await new Promise((resolve) => setImmediate(resolve));
+  releaseReopen();
+  const [firstResponse, secondResponse] = await Promise.all([first, second]);
+  await new Promise((resolve) => setImmediate(resolve));
+
+  expect(firstResponse.json()).toEqual({ id: "s1", alreadyActive: false });
+  expect(secondResponse.json()).toEqual({ id: "s1", alreadyActive: true });
+  expect({ showCalls, readinessCalls, reopenCalls, createCalls, clearCalls }).toEqual({
+    showCalls: 1,
+    readinessCalls: 1,
+    reopenCalls: 1,
+    createCalls: 1,
+    clearCalls: 1,
+  });
+});
+
+test("a replaced runtime cannot publish or fail the newly resumed session", async () => {
+  const controlledBridge = (initialState: string) => {
+    let state = initialState;
+    let listener: ((event: any) => void) | undefined;
+    return {
+      turnState: () => state,
+      setState: (next: string) => { state = next; },
+      onClientEvent: (next: (event: any) => void) => { listener = next; },
+      emitClientEvent: (event: any) => listener?.(event),
+      pendingWidget: () => null,
+    };
+  };
+  const oldBridge = controlledBridge("running");
+  const newBridge = controlledBridge("running");
+  const runtimes = [{ bridge: oldBridge }, { bridge: newBridge }];
+  let current: any;
+  let failed = 0;
+  const published: any[] = [];
+  const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
+    mgr: {
+      get: () => current,
+      createFor: () => { current = runtimes.shift(); return current; },
+      configure: async () => {},
+      start: () => {},
+      teardown: () => { current = undefined; },
+    } as any,
+    hub: {
+      clear: () => {},
+      publish: (_id: string, event: string, data: object) => {
+        published.push({ event, data });
+        return published.length;
+      },
+    } as any,
+    thtRunner: {
+      sessionNew: async () => ({ id: "s1" }),
+      searchPack: async () => {},
+      sessionShow: async () => ({ status: "open", archived: false }),
+      reopenSession: async () => {},
+      failSession: async () => { failed += 1; },
+    } as any,
+    readiness: { ensure: async () => ({ ok: true }) } as any,
+    getSettings: () => ({ workspace: "local" }) as any,
+  });
+  await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
+  await new Promise((resolve) => setImmediate(resolve));
+  oldBridge.setState("idle");
+
+  const response = await app.inject({ method: "POST", url: "/sessions/s1/resume" });
+  published.length = 0;
+  // The manager removes an unexpectedly exited active runtime before its bridge publishes the
+  // public failure sequence. Binding identity, rather than mgr.get(), must still admit it.
+  current = undefined;
+  oldBridge.emitClientEvent({ type: "system_event", event: "session_failed" });
+  newBridge.emitClientEvent({ type: "info", text: "new runtime" });
+  await new Promise((resolve) => setImmediate(resolve));
+
+  expect(response.json()).toEqual({ id: "s1", alreadyActive: false });
+  expect(failed).toBe(0);
+  expect(published).toEqual([{
+    event: "info",
+    data: { type: "info", text: "new runtime" },
+  }]);
+});
+
+test("a deleted runtime cannot repopulate or fail the forgotten session", async () => {
+  let listener: ((event: any) => void) | undefined;
+  const bridge = {
+    turnState: () => "running",
+    onClientEvent: (next: (event: any) => void) => { listener = next; },
+    emitClientEvent: (event: any) => listener?.(event),
+  };
+  const runtime = { bridge };
+  let current: any;
+  let failed = 0;
+  const published: any[] = [];
+  const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
+    mgr: {
+      get: () => current,
+      createFor: () => { current = runtime; return runtime; },
+      configure: async () => {},
+      start: () => {},
+      teardown: () => { current = undefined; },
+    } as any,
+    hub: {
+      publish: (_id: string, event: string, data: object) => {
+        published.push({ event, data });
+        return published.length;
+      },
+      forget: () => {},
+    } as any,
+    thtRunner: {
+      sessionNew: async () => ({ id: "s1" }),
+      searchPack: async () => {},
+      deleteSession: async () => {},
+      failSession: async () => { failed += 1; },
+    } as any,
+    readiness: { ensure: async () => ({ ok: true }) } as any,
+    getSettings: () => ({ workspace: "local" }) as any,
+  });
+  await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
+  await new Promise((resolve) => setImmediate(resolve));
+
+  const response = await app.inject({ method: "DELETE", url: "/sessions/s1" });
+  published.length = 0;
+  bridge.emitClientEvent({ type: "system_event", event: "session_failed" });
+  await new Promise((resolve) => setImmediate(resolve));
+
+  expect(response.statusCode).toBe(204);
+  expect(failed).toBe(0);
+  expect(published).toEqual([]);
+});
+
+test("an unexpectedly exited runtime publishes its terminal sequence then releases its binding", async () => {
+  let listener: ((event: any) => void) | undefined;
+  const bridge = {
+    onClientEvent: (next: (event: any) => void) => { listener = next; },
+    emitClientEvent: (event: any) => listener?.(event),
+  };
+  const runtime = { bridge };
+  let current: any;
+  let failed = 0;
+  const published: any[] = [];
+  const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
+    mgr: {
+      get: () => current,
+      createFor: () => { current = runtime; return runtime; },
+      configure: async () => {},
+      start: () => {},
+      teardown: () => { current = undefined; },
+    } as any,
+    hub: {
+      publish: (_id: string, event: string, data: object) => {
+        published.push({ event, data });
+        return published.length;
+      },
+    } as any,
+    thtRunner: {
+      sessionNew: async () => ({ id: "s1" }),
+      searchPack: async () => {},
+      failSession: async () => { failed += 1; },
+    } as any,
+    readiness: { ensure: async () => ({ ok: true }) } as any,
+    getSettings: () => ({ workspace: "local" }) as any,
+  });
+  await app.inject({ method: "POST", url: "/sessions", payload: { question: "q" } });
+  await new Promise((resolve) => setImmediate(resolve));
+  published.length = 0;
+
+  // PiProcessManager deletes an unexpectedly exited runtime before emitting this sequence.
+  current = undefined;
+  bridge.emitClientEvent({ type: "info", level: "error", text: "public failure" });
+  bridge.emitClientEvent({ type: "system_event", event: "session_failed" });
+  bridge.emitClientEvent({ type: "system_event", event: "agent_end" });
+  bridge.emitClientEvent({ type: "text_delta", text: "too late" });
+  await new Promise((resolve) => setImmediate(resolve));
+
+  expect(failed).toBe(1);
+  expect(published).toEqual([
+    { event: "info", data: { type: "info", level: "error", text: "public failure" } },
+    { event: "system_event", data: { type: "system_event", event: "session_failed" } },
+    { event: "system_event", data: { type: "system_event", event: "agent_end" } },
+  ]);
+});
+
+test("POST resume tears down a created runtime when bridge binding fails", async () => {
+  const delivered: string[] = [];
+  const actualHub = new SseHub();
+  let clearCalls = 0;
+  let teardownCalls = 0;
+  let current: any = { bridge: { turnState: () => "failed" } };
+  const hub = {
+    subscribe: actualHub.subscribe.bind(actualHub),
+    publish: actualHub.publish.bind(actualHub),
+    clear: (id: string) => { clearCalls += 1; actualHub.clear(id); },
+  } as any;
+  hub.subscribe("s1", (_event: string, data: any) => delivered.push(data.text));
+  hub.publish("s1", "info", { text: "before" });
+  const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
+    mgr: {
+      get: () => current,
+      teardown: () => { teardownCalls += 1; current = undefined; },
+      createFor: () => {
+        current = {
+          bridge: {
+            onClientEvent: () => { throw new Error("BIND_SENTINEL"); },
+          },
+        };
+        return current;
+      },
+    } as any,
+    hub,
+    thtRunner: {
+      sessionShow: async () => ({ status: "open", archived: false }),
+      reopenSession: async () => {},
+    } as any,
+    readiness: { ensure: async () => ({ ok: true }) } as any,
+    getSettings: () => ({ workspace: "local" }) as any,
+  });
+
+  const response = await app.inject({ method: "POST", url: "/sessions/s1/resume" });
+  hub.publish("s1", "info", { text: "post-failure probe" });
+
+  expect(response.statusCode).toBe(503);
+  expect(response.body).not.toContain("BIND_SENTINEL");
+  expect(teardownCalls).toBe(2);
+  expect(current).toBeUndefined();
+  expect(clearCalls).toBe(0);
+  expect(delivered).toEqual(["before", "post-failure probe"]);
+});
+
 test("POST /sessions/:id/response inoltra al bridge (no error)", async () => {
  const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
    thtRunner: {
@@ -332,12 +691,31 @@ test("DELETE /sessions/:id tears down the runtime before deleting on disk", asyn
  const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
    thtRunner: { deleteSession: async (id: string) => { order.push(`del:${id}`); } } as any,
    mgr: { teardown: (id: string) => { order.push(`teardown:${id}`); } } as any,
+    hub: { forget: (id: string) => { order.push(`forget:${id}`); } } as any,
    getSettings: () => ({ workspace: "w" }) as any,
    spawnFn: () => nodeSpawn("node", [FAKE, SCRIPT]) as any,
  });
  const res = await app.inject({ method: "DELETE", url: "/sessions/s1" });
  expect(res.statusCode).toBe(204);
-  expect(order).toEqual(["teardown:s1", "del:s1"]);
+  expect(order).toEqual(["teardown:s1", "del:s1", "forget:s1"]);
+});
+
+test("POST /sessions/:id/close clears transient hub state without forgetting its id", async () => {
+  const calls: string[] = [];
+  const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
+    thtRunner: { closeSession: async () => {} } as any,
+    mgr: { teardown: () => {} } as any,
+    hub: {
+      clear: (id: string) => { calls.push(`clear:${id}`); },
+      forget: (id: string) => { calls.push(`forget:${id}`); },
+    } as any,
+    getSettings: () => ({ workspace: "w" }) as any,
+  });
+
+  const response = await app.inject({ method: "POST", url: "/sessions/s1/close" });
+
+  expect(response.statusCode).toBe(200);
+  expect(calls).toEqual(["clear:s1"]);
 });

 test("GET /sessions/:id/documents returns the runner output", async () => {
@@ -495,6 +873,7 @@ test("POST /sessions bootstrap failure emits only a fixed recovery message", asy
    "connect https://secret.invalid/bootstrap?token=DO_NOT_LEAK using /srv/private/model-key";
  const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
    mgr: {
+      get: () => undefined,
      createFor: () => runtime,
      configure: async () => { throw new Error(rawFailure); },
      start: () => {},