fix: make session resume atomic across restarts
This commit is contained in:
+109
-36
@@ -10,6 +10,8 @@ const BOOTSTRAP_FAILURE_MESSAGE =
|
||||
"Session startup failed. Check configuration and connectivity, then Resume the session.";
|
||||
const READINESS_FAILURE_MESSAGE =
|
||||
"Session services are not ready. Check configuration and connectivity, then try again.";
|
||||
const RESUME_FAILURE_MESSAGE =
|
||||
"Session could not be resumed. Check configuration and connectivity, then try again.";
|
||||
|
||||
function eventCursor(...values: unknown[]): number {
|
||||
let cursor = 0;
|
||||
@@ -25,16 +27,58 @@ export function sessionRoutes(
|
||||
app: FastifyInstance,
|
||||
d: { mgr: PiProcessManager; tht: ThtRunner; hub: SseHub; getSettings: () => Settings; readiness: ReadinessManager },
|
||||
) {
|
||||
const resumeTails = new Map<string, Promise<void>>();
|
||||
const boundRuntimes = new Map<
|
||||
string,
|
||||
ReturnType<PiProcessManager["createFor"]>
|
||||
>();
|
||||
|
||||
const withResumeLock = async <T>(id: string, work: () => Promise<T>): Promise<T> => {
|
||||
const previous = resumeTails.get(id) ?? Promise.resolve();
|
||||
let release!: () => void;
|
||||
const gate = new Promise<void>((resolve) => { release = resolve; });
|
||||
const tail = previous.then(() => gate);
|
||||
resumeTails.set(id, tail);
|
||||
await previous;
|
||||
try {
|
||||
return await work();
|
||||
} finally {
|
||||
release();
|
||||
if (resumeTails.get(id) === tail) resumeTails.delete(id);
|
||||
}
|
||||
};
|
||||
|
||||
const info = (id: string, text: string, level = "info") =>
|
||||
d.hub.publish(id, "info", { type: "info", level, text });
|
||||
|
||||
const bindRuntime = (id: string, rt: ReturnType<PiProcessManager["createFor"]>) =>
|
||||
rt.bridge.onClientEvent((e) => {
|
||||
if (e.type === "system_event" && e.event === "session_failed") {
|
||||
void d.tht.failSession(id, d.getSettings().workspace).catch(() => undefined);
|
||||
}
|
||||
d.hub.publish(id, e.type, e);
|
||||
});
|
||||
const bindRuntime = (id: string, rt: ReturnType<PiProcessManager["createFor"]>) => {
|
||||
const previous = boundRuntimes.get(id);
|
||||
boundRuntimes.set(id, rt);
|
||||
try {
|
||||
rt.bridge.onClientEvent((e) => {
|
||||
// Child termination is asynchronous. Ignore queued events from a runtime once a newer
|
||||
// identity is bound or the session is explicitly closed/deleted. The active identity
|
||||
// remains bound after an unexpected exit so its public failure events still reach SSE.
|
||||
if (boundRuntimes.get(id) !== rt) return;
|
||||
if (e.type === "system_event" && e.event === "session_failed") {
|
||||
void d.tht.failSession(id, d.getSettings().workspace).catch(() => undefined);
|
||||
}
|
||||
d.hub.publish(id, e.type, e);
|
||||
if (
|
||||
e.type === "system_event"
|
||||
&& e.event === "agent_end"
|
||||
&& d.mgr.get(id) !== rt
|
||||
&& boundRuntimes.get(id) === rt
|
||||
) {
|
||||
boundRuntimes.delete(id);
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
if (previous) boundRuntimes.set(id, previous);
|
||||
else if (boundRuntimes.get(id) === rt) boundRuntimes.delete(id);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
const bootstrap = (
|
||||
id: string,
|
||||
@@ -114,42 +158,69 @@ export function sessionRoutes(
|
||||
});
|
||||
app.post("/sessions/:id/resume", async (req, reply) => {
|
||||
const id = (req.params as any).id;
|
||||
const existing = d.mgr.get(id);
|
||||
if (existing) {
|
||||
const state = existing.bridge.turnState();
|
||||
if (state === "running" || state === "waiting") {
|
||||
return reply.code(200).send({ id, alreadyActive: true });
|
||||
return withResumeLock(id, async () => {
|
||||
// This check belongs inside the per-session lock: a preceding cold Resume may have
|
||||
// installed a running runtime while this request was waiting.
|
||||
const existing = d.mgr.get(id);
|
||||
if (existing) {
|
||||
const state = existing.bridge.turnState();
|
||||
if (state === "running" || state === "waiting") {
|
||||
return reply.code(200).send({ id, alreadyActive: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
const manifest = (await d.tht.sessionShow(id, d.getSettings().workspace)) as { status?: string; archived?: boolean } | null;
|
||||
if (manifest?.status === "finalized" || manifest?.archived) {
|
||||
return reply.code(409).send({ error: "sessione in sola lettura (finalizzata o archiviata)" });
|
||||
}
|
||||
const settings = d.getSettings();
|
||||
const ensure = await d.readiness.ensure(settings.workspace ?? "");
|
||||
if (!ensure.ok) return reply.code(503).send({ error: READINESS_FAILURE_MESSAGE });
|
||||
const saved = manifest as { provider?: string; model?: string; thinking?: string } | null;
|
||||
const options = {
|
||||
provider: saved?.provider,
|
||||
model: saved?.model,
|
||||
thinking: saved?.thinking ?? settings.thinking,
|
||||
author: getUser(req).id,
|
||||
mode: "resume" as const,
|
||||
};
|
||||
if (existing) d.mgr.teardown(id);
|
||||
d.hub.clear(id);
|
||||
await d.tht.reopenSession(id, settings.workspace);
|
||||
const rt = d.mgr.createFor(id, options);
|
||||
bindRuntime(id, rt);
|
||||
info(id, "Resuming session");
|
||||
bootstrap(id, rt, d.mgr.configure(rt, options), null, () => d.mgr.start(id, rt, options));
|
||||
return reply.code(200).send({ id, alreadyActive: false });
|
||||
const manifest = (await d.tht.sessionShow(id, d.getSettings().workspace)) as { status?: string; archived?: boolean } | null;
|
||||
if (manifest?.status === "finalized" || manifest?.archived) {
|
||||
return reply.code(409).send({ error: "sessione in sola lettura (finalizzata o archiviata)" });
|
||||
}
|
||||
const settings = d.getSettings();
|
||||
const ensure = await d.readiness.ensure(settings.workspace ?? "");
|
||||
if (!ensure.ok) return reply.code(503).send({ error: READINESS_FAILURE_MESSAGE });
|
||||
const saved = manifest as { provider?: string; model?: string; thinking?: string } | null;
|
||||
const options = {
|
||||
provider: saved?.provider,
|
||||
model: saved?.model,
|
||||
thinking: saved?.thinking ?? settings.thinking,
|
||||
author: getUser(req).id,
|
||||
mode: "resume" as const,
|
||||
};
|
||||
|
||||
// Reopening is validation, not the transport commit point. Keep the old hub intact if
|
||||
// persistence cannot be reopened.
|
||||
try {
|
||||
await d.tht.reopenSession(id, settings.workspace);
|
||||
} catch {
|
||||
return reply.code(503).send({ error: RESUME_FAILURE_MESSAGE });
|
||||
}
|
||||
|
||||
let rt: ReturnType<PiProcessManager["createFor"]> | undefined;
|
||||
try {
|
||||
if (existing) {
|
||||
boundRuntimes.delete(id);
|
||||
d.mgr.teardown(id);
|
||||
}
|
||||
rt = d.mgr.createFor(id, options);
|
||||
bindRuntime(id, rt);
|
||||
} catch {
|
||||
// A created-but-unbound runtime is not usable. The old hub remains attached because
|
||||
// clear() has not happened yet.
|
||||
if (rt) d.mgr.teardown(id);
|
||||
return reply.code(503).send({ error: RESUME_FAILURE_MESSAGE });
|
||||
}
|
||||
|
||||
// Commit the replacement only after reopen + runtime creation/binding succeeded, and
|
||||
// immediately before the first event produced by the new Resume.
|
||||
d.hub.clear(id);
|
||||
info(id, "Resuming session");
|
||||
bootstrap(id, rt, d.mgr.configure(rt, options), null, () => d.mgr.start(id, rt, options));
|
||||
return reply.code(200).send({ id, alreadyActive: false });
|
||||
});
|
||||
});
|
||||
app.post("/sessions/:id/close", async (req) => {
|
||||
const id = (req.params as { id: string }).id;
|
||||
try {
|
||||
await d.tht.closeSession(id, d.getSettings().workspace);
|
||||
} finally {
|
||||
boundRuntimes.delete(id);
|
||||
d.mgr.teardown(id);
|
||||
d.hub.clear(id);
|
||||
}
|
||||
@@ -202,8 +273,10 @@ export function sessionRoutes(
|
||||
});
|
||||
app.delete("/sessions/:id", async (req, reply) => {
|
||||
const id = (req.params as any).id;
|
||||
boundRuntimes.delete(id);
|
||||
d.mgr.teardown(id); // drop any live runtime before deleting on disk
|
||||
await d.tht.deleteSession(id, d.getSettings().workspace);
|
||||
d.hub.forget(id);
|
||||
return reply.code(204).send();
|
||||
});
|
||||
app.get("/sessions/:id/documents", async (req) => d.tht.documents((req.params as any).id));
|
||||
|
||||
Reference in New Issue
Block a user