fix(backend): generation-aware SSE event ids — stale cursors can no longer eat events

Audit finding 3.1 (high, 3/3 reviewer consensus). Event ids restart at 1
when the backend restarts; a browser auto-reconnect carrying the old
numeric Last-Event-ID was honored whenever the new process had already
emitted that many events, silently suppressing fresh events (same ids,
different content). The previous guard only caught cursor > lastId.

Wire ids are now "<generation>:<seq>" (generation = per-hub instance
token; seq = the existing per-session monotonic counter). The hub parses
raw header/query candidates itself: other-generation and legacy bare-
number cursors are stale → replay from the beginning; same-generation
cursors keep the newest-valid-wins behavior. EventSource treats ids as
opaque, so no frontend change.

Finding 3.2 (eviction) resolved by NOT evicting: close keeps the seq
counter on purpose (sessions reopen; monotonicity is what makes old
cursors detectable) — documented at the call site; buffers are emptied by
clear() and ring-bounded at 200.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

backend/test/sse-hub.test.ts

@@ -1,6 +1,12 @@
 import { test, expect } from "vitest";
 import { SseHub } from "../src/sse/sse-hub.js";
 
+// Wire ids are "<generation>:<seq>": seq is the per-session monotonic counter,
+// generation identifies the hub instance (process). Tests derive both from
+// published ids instead of hardcoding the random generation.
+const seqOf = (id: string) => Number(id.split(":")[1]);
+const genOf = (id: string) => id.split(":")[0];
+
 test("publish assigns monotonic ids and a subscriber replays only ids newer than its cursor", () => {
   const hub = new SseHub();
   const firstId = hub.publish("s1", "text_delta", { text: "one" });
@@ -10,14 +16,15 @@ test("publish assigns monotonic ids and a subscriber replays only ids newer than
   hub.subscribe(
     "s1",
     (event, data, id) => sent.push({ event, data, id }),
-    { afterId: firstId },
+    { after: [firstId] },
   );
   const thirdId = hub.publish("s1", "text_delta", { text: "three" });
 
-  expect([firstId, secondId, thirdId]).toEqual([1, 2, 3]);
+  expect([firstId, secondId, thirdId].map(seqOf)).toEqual([1, 2, 3]);
+  expect(new Set([firstId, secondId, thirdId].map(genOf)).size).toBe(1);
   expect(sent).toEqual([
-    { event: "text_delta", data: { text: "two" }, id: 2 },
-    { event: "text_delta", data: { text: "three" }, id: 3 },
+    { event: "text_delta", data: { text: "two" }, id: secondId },
+    { event: "text_delta", data: { text: "three" }, id: thirdId },
   ]);
 });
 
@@ -38,33 +45,36 @@ test("clear closes every stale subscriber once and replays post-clear events fro
     (event, data, id) => secondStale.push({ event, data, id }),
     { close: () => { closeCalls[1] += 1; offSecond(); } },
   );
-  expect(hub.publish("s1", "info", { text: "before" })).toBe(1);
+  const beforeId = hub.publish("s1", "info", { text: "before" });
+  expect(seqOf(beforeId)).toBe(1);
 
   hub.clear("s1");
   offFirst();
   offSecond();
   expect(closeCalls).toEqual([1, 1]);
-  expect(hub.publish("s1", "info", { text: "after" })).toBe(2);
-  expect(hub.publish("s1", "ui_request", {
+  const afterId = hub.publish("s1", "info", { text: "after" });
+  expect(seqOf(afterId)).toBe(2);
+  const gateId = hub.publish("s1", "ui_request", {
     type: "ui_request",
     ui_request: { id: "gate-1", widget: "select" },
-  })).toBe(3);
+  });
+  expect(seqOf(gateId)).toBe(3);
   const resumed: any[] = [];
   hub.subscribe(
     "s1",
     (event, data, id) => resumed.push({ event, data, id }),
-    { afterId: 1 },
+    { after: [beforeId] },
   );
 
-  const before = [{ event: "info", data: { text: "before" }, id: 1 }];
+  const before = [{ event: "info", data: { text: "before" }, id: beforeId }];
   expect(firstStale).toEqual(before);
   expect(secondStale).toEqual(before);
   expect(resumed).toEqual([
-    { event: "info", data: { text: "after" }, id: 2 },
+    { event: "info", data: { text: "after" }, id: afterId },
     {
       event: "ui_request",
       data: { type: "ui_request", ui_request: { id: "gate-1", widget: "select" } },
-      id: 3,
+      id: gateId,
     },
   ]);
 });
@@ -86,95 +96,118 @@ test("forget closes every subscriber once and resets the per-session id sequence
     (event, data, id) => secondStale.push({ event, data, id }),
     { close: () => { closeCalls[1] += 1; offSecond(); } },
   );
-  expect(hub.publish("s1", "info", { text: "before" })).toBe(1);
+  const beforeId = hub.publish("s1", "info", { text: "before" });
+  expect(seqOf(beforeId)).toBe(1);
 
   hub.forget("s1");
   expect(closeCalls).toEqual([1, 1]);
-  expect(hub.publish("s1", "info", { text: "after" })).toBe(1);
+  const afterId = hub.publish("s1", "info", { text: "after" });
+  expect(seqOf(afterId)).toBe(1);
   const fresh: any[] = [];
   hub.subscribe("s1", (event, data, id) => fresh.push({ event, data, id }));
 
-  const before = [{ event: "info", data: { text: "before" }, id: 1 }];
+  const before = [{ event: "info", data: { text: "before" }, id: beforeId }];
   expect(firstStale).toEqual(before);
   expect(secondStale).toEqual(before);
-  expect(fresh).toEqual([{ event: "info", data: { text: "after" }, id: 1 }]);
+  expect(fresh).toEqual([{ event: "info", data: { text: "after" }, id: afterId }]);
 });
 
 test("a buffered pending gate is emitted exactly once and matched by descriptor id", () => {
   const hub = new SseHub();
   const descriptor = { id: "gate-1", widget: "select", title: "Original" };
-  hub.publish("s1", "ui_request", { type: "ui_request", ui_request: descriptor });
+  const gateId = hub.publish("s1", "ui_request", { type: "ui_request", ui_request: descriptor });
   const replayed: any[] = [];
 
   hub.subscribe(
     "s1",
     (event, data, id) => replayed.push({ event, data, id }),
-    { afterId: 0, pending: { ...descriptor, title: "Runtime copy" } },
+    { after: [], pending: { ...descriptor, title: "Runtime copy" } },
   );
 
   expect(replayed).toEqual([{
     event: "ui_request",
     data: { type: "ui_request", ui_request: descriptor },
-    id: 1,
+    id: gateId,
   }]);
 
   const alreadySeen: any[] = [];
   hub.subscribe(
     "s1",
     (event, data, id) => alreadySeen.push({ event, data, id }),
-    { afterId: 1, pending: descriptor },
+    { after: [gateId], pending: descriptor },
   );
   expect(alreadySeen).toEqual([]);
 });
 
 test("an unbuffered pending gate receives one fresh buffered id", () => {
   const hub = new SseHub();
-  hub.publish("s1", "info", { text: "ready" });
+  const readyId = hub.publish("s1", "info", { text: "ready" });
   const first: any[] = [];
 
   hub.subscribe(
     "s1",
     (event, data, id) => first.push({ event, data, id }),
-    { afterId: 1, pending: { id: "gate-1", widget: "select" } },
+    { after: [readyId], pending: { id: "gate-1", widget: "select" } },
   );
   expect(first).toEqual([{
     event: "ui_request",
     data: { type: "ui_request", ui_request: { id: "gate-1", widget: "select" } },
-    id: 2,
+    id: `${genOf(readyId)}:2`,
   }]);
 
   const reconnect: any[] = [];
   hub.subscribe(
     "s1",
     (event, data, id) => reconnect.push({ event, data, id }),
-    { afterId: 1, pending: { id: "gate-1", widget: "select" } },
+    { after: [readyId], pending: { id: "gate-1", widget: "select" } },
   );
   expect(reconnect).toEqual(first);
 });
 
-test("a cursor newer than this hub generation replays low ids and a buffered gate once", () => {
-  const hub = new SseHub();
-  const descriptor = { id: "gate-fresh", widget: "select", title: "Choose" };
-  hub.publish("s1", "info", { type: "info", text: "fresh generation" });
-  hub.publish("s1", "ui_request", { type: "ui_request", ui_request: descriptor });
+test("a cursor from a previous hub generation is stale and replays from the beginning", () => {
+  // Simulates a backend restart: the browser auto-reconnects with the Last-Event-ID it
+  // got from the OLD process. The new hub must ignore it even when its own seqs have
+  // already reached (or passed) that number — same ids, different content.
+  const oldHub = new SseHub();
+  oldHub.publish("s1", "info", { text: "old-1" });
+  const oldCursor = oldHub.publish("s1", "info", { text: "old-2" });
+
+  const newHub = new SseHub();
+  const freshFirst = newHub.publish("s1", "info", { text: "new-1" });
+  const freshSecond = newHub.publish("s1", "info", { text: "new-2" });
   const replayed: any[] = [];
 
-  hub.subscribe(
+  newHub.subscribe(
     "s1",
     (event, data, id) => replayed.push({ event, data, id }),
-    { afterId: 900, pending: { ...descriptor } },
+    { after: [oldCursor] },
   );
 
   expect(replayed).toEqual([
-    {
-      event: "info",
-      data: { type: "info", text: "fresh generation" },
-      id: 1,
-    },
-    {
-      event: "ui_request",
-      data: { type: "ui_request", ui_request: descriptor },
-      id: 2,
-    },
+    { event: "info", data: { text: "new-1" }, id: freshFirst },
+    { event: "info", data: { text: "new-2" }, id: freshSecond },
   ]);
 });
+
+test("legacy bare-number and too-new same-generation cursors replay from the beginning", () => {
+  const hub = new SseHub();
+  const descriptor = { id: "gate-fresh", widget: "select", title: "Choose" };
+  const firstId = hub.publish("s1", "info", { type: "info", text: "fresh generation" });
+  const gateId = hub.publish("s1", "ui_request", { type: "ui_request", ui_request: descriptor });
+  const expected = [
+    { event: "info", data: { type: "info", text: "fresh generation" }, id: firstId },
+    { event: "ui_request", data: { type: "ui_request", ui_request: descriptor }, id: gateId },
+  ];
+
+  const legacy: any[] = [];
+  hub.subscribe("s1", (event, data, id) => legacy.push({ event, data, id }), { after: ["2"] });
+  expect(legacy).toEqual(expected);
+
+  const tooNew: any[] = [];
+  hub.subscribe(
+    "s1",
+    (event, data, id) => tooNew.push({ event, data, id }),
+    { after: [`${genOf(firstId)}:900`], pending: { ...descriptor } },
+  );
+  expect(tooNew).toEqual(expected);
+});

backend/test/sse-route.test.ts

@@ -3,6 +3,8 @@ import { buildApp } from "../src/app.js";
 import { loadConfig } from "../src/config.js";
 import { SseHub } from "../src/sse/sse-hub.js";
 
+const seqOf = (id: string) => Number(id.split(":")[1]);
+
 async function readUntil(
   reader: ReadableStreamDefaultReader<Uint8Array>,
   predicate: (text: string) => boolean,
@@ -24,11 +26,15 @@ async function readUntil(
   return text;
 }
 
-async function captureReplay(options: { query?: string; lastEventId?: string }): Promise<string> {
+async function captureReplay(
+  options: { query?: (ids: string[]) => string; lastEventId?: (ids: string[]) => string },
+): Promise<{ body: string; ids: string[] }> {
   const hub = new SseHub();
-  hub.publish("s1", "info", { type: "info", text: "one" });
-  hub.publish("s1", "info", { type: "info", text: "two" });
-  hub.publish("s1", "info", { type: "info", text: "three" });
+  const ids = [
+    hub.publish("s1", "info", { type: "info", text: "one" }),
+    hub.publish("s1", "info", { type: "info", text: "two" }),
+    hub.publish("s1", "info", { type: "info", text: "three" }),
+  ];
   const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/tmp/h" }), {
     hub,
     thtRunner: {} as any,
@@ -38,17 +44,19 @@ async function captureReplay(options: { query?: string; lastEventId?: string }):
   const controller = new AbortController();
 
   try {
+    const lastEventId = options.lastEventId?.(ids);
+    const query = options.query ? `?lastEventId=${encodeURIComponent(options.query(ids))}` : "";
     const response = await fetch(
-      `http://127.0.0.1:${port}/sessions/s1/events${options.query ?? ""}`,
+      `http://127.0.0.1:${port}/sessions/s1/events${query}`,
       {
-        headers: options.lastEventId ? { "Last-Event-ID": options.lastEventId } : undefined,
+        headers: lastEventId ? { "Last-Event-ID": lastEventId } : undefined,
         signal: controller.signal,
       },
     );
     const reader = response.body!.getReader();
     const body = await readUntil(reader, (text) => text.includes('"three"'));
     await reader.cancel();
-    return body;
+    return { body, ids };
   } finally {
     controller.abort();
     await app.close();
@@ -67,36 +75,45 @@ async function expectStreamEnd(reader: ReadableStreamDefaultReader<Uint8Array>):
 }
 
 test("SSE emits ids and honors the native Last-Event-ID replay cursor", async () => {
-  const body = await captureReplay({ lastEventId: "1" });
+  const { body, ids } = await captureReplay({ lastEventId: (published) => published[0] });
 
   expect(body).not.toContain('"one"');
-  expect(body).toContain("id: 2\nevent: info\n");
+  expect(body).toContain(`id: ${ids[1]}\nevent: info\n`);
   expect(body).toContain('data: {"type":"info","text":"two"}');
-  expect(body).toContain("id: 3\nevent: info\n");
+  expect(body).toContain(`id: ${ids[2]}\nevent: info\n`);
   expect(body).toContain('data: {"type":"info","text":"three"}');
 });
 
 test("SSE honors the manual lastEventId query cursor", async () => {
-  const body = await captureReplay({ query: "?lastEventId=2" });
+  const { body, ids } = await captureReplay({ query: (published) => published[1] });
 
   expect(body).not.toContain('"one"');
   expect(body).not.toContain('"two"');
-  expect(body).toContain("id: 3\nevent: info\n");
+  expect(body).toContain(`id: ${ids[2]}\nevent: info\n`);
   expect(body).toContain('data: {"type":"info","text":"three"}');
 });
 
 test("SSE uses the newer valid cursor when header and query are both present", async () => {
-  const body = await captureReplay({ query: "?lastEventId=2", lastEventId: "1" });
+  const { body } = await captureReplay({
+    query: (published) => published[1],
+    lastEventId: (published) => published[0],
+  });
 
   expect(body).not.toContain('"two"');
-  expect(body).toContain("id: 3\nevent: info\n");
+  expect(body).toContain('"three"');
 });
 
-test("SSE resets a cursor from an older process and emits a buffered pending gate once", async () => {
+test("SSE resets a cursor from an older backend generation and emits a buffered pending gate once", async () => {
+  // The cursor comes from a PREVIOUS process: same session, ids restarted. It must be
+  // treated as stale (replay from the beginning), not honored against the new ids.
+  const oldGeneration = new SseHub();
+  oldGeneration.publish("s1", "info", { type: "info", text: "old" });
+  const staleCursor = oldGeneration.publish("s1", "info", { type: "info", text: "older" });
+
   const hub = new SseHub();
   const descriptor = { id: "gate-fresh", widget: "select", title: "Choose" };
-  hub.publish("s1", "info", { type: "info", text: "fresh generation" });
-  hub.publish("s1", "ui_request", { type: "ui_request", ui_request: descriptor });
+  const infoId = hub.publish("s1", "info", { type: "info", text: "fresh generation" });
+  const gateId = hub.publish("s1", "ui_request", { type: "ui_request", ui_request: descriptor });
   const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/tmp/h" }), {
     hub,
     thtRunner: {} as any,
@@ -110,17 +127,17 @@ test("SSE resets a cursor from an older process and emits a buffered pending gat
 
   try {
     const response = await fetch(`http://127.0.0.1:${port}/sessions/s1/events`, {
-      headers: { "Last-Event-ID": "900" },
+      headers: { "Last-Event-ID": staleCursor },
       signal: controller.signal,
     });
     const reader = response.body!.getReader();
     const body = await readUntil(reader, (text) => text.includes('"gate-fresh"'));
     await reader.cancel();
 
-    expect(body).toContain("id: 1\nevent: info\n");
+    expect(body).toContain(`id: ${infoId}\nevent: info\n`);
     expect(body).toContain('data: {"type":"info","text":"fresh generation"}');
     expect(body.match(/event: ui_request/g)).toHaveLength(1);
-    expect(body).toContain("id: 2\nevent: ui_request\n");
+    expect(body).toContain(`id: ${gateId}\nevent: ui_request\n`);
     expect(body).toContain('"ui_request":{"id":"gate-fresh","widget":"select","title":"Choose"}');
   } finally {
     controller.abort();
@@ -145,32 +162,35 @@ test("clear ends every live SSE response and a cursor reconnect replays post-cle
     )));
     const readers = responses.map((response) => response.body!.getReader());
 
-    expect(hub.publish("s1", "info", { type: "info", text: "before" })).toBe(1);
+    const beforeId = hub.publish("s1", "info", { type: "info", text: "before" });
+    expect(seqOf(beforeId)).toBe(1);
     const initial = await Promise.all(readers.map((reader) =>
       readUntil(reader, (text) => text.includes('"before"'))));
-    expect(initial.every((body) => body.includes("id: 1\nevent: info\n"))).toBe(true);
+    expect(initial.every((body) => body.includes(`id: ${beforeId}\nevent: info\n`))).toBe(true);
 
     hub.clear("s1");
     await Promise.all(readers.map(expectStreamEnd));
 
-    expect(hub.publish("s1", "info", { type: "info", text: "after" })).toBe(2);
-    expect(hub.publish("s1", "ui_request", {
+    const afterId = hub.publish("s1", "info", { type: "info", text: "after" });
+    expect(seqOf(afterId)).toBe(2);
+    const gateId = hub.publish("s1", "ui_request", {
       type: "ui_request",
       ui_request: { id: "gate-1", widget: "select" },
-    })).toBe(3);
+    });
+    expect(seqOf(gateId)).toBe(3);
 
     const reconnected = await fetch(
       `http://127.0.0.1:${port}/sessions/s1/events`,
       {
-        headers: { "Last-Event-ID": "1" },
+        headers: { "Last-Event-ID": beforeId },
         signal: controllers[2].signal,
       },
     );
     const reconnectReader = reconnected.body!.getReader();
     const replay = await readUntil(reconnectReader, (text) => text.includes('"gate-1"'));
-    expect(replay).toContain("id: 2\nevent: info\n");
+    expect(replay).toContain(`id: ${afterId}\nevent: info\n`);
     expect(replay).toContain('data: {"type":"info","text":"after"}');
-    expect(replay).toContain("id: 3\nevent: ui_request\n");
+    expect(replay).toContain(`id: ${gateId}\nevent: ui_request\n`);
     expect(replay).toContain('"ui_request":{"id":"gate-1","widget":"select"}');
     await reconnectReader.cancel();
   } finally {