fix: harden Pi lifecycle recovery

backend/src/app.ts

@@ -87,16 +87,22 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
     return effectiveSettings(config, loadSettings(config));
   };
 
+  const maintenanceBarrier = deps?.maintenanceBarrier ?? new MaintenanceBarrier(config.maintenanceFile);
   const authenticate = authPreHandler(config.authMode);
   app.addHook("preHandler", async (req, reply) => {
     // Process readiness is intentionally unauthenticated for local container/proxy probes.
     if (req.url === "/health" || req.url === "/health/dwh") return;
+    if (isMaintenanceControl(req.url)) {
+      if (!isLoopback(req.ip)) {
+        return reply.code(403).send({ error: "loopback maintenance control required" });
+      }
+      return;
+    }
     return authenticate(req, reply);
   });
   app.get("/health", async () => ({ status: "ok" }));
   app.get("/health/dwh", async () => tht.dbPing());
   app.get("/me", async (req) => getPrincipal(req));
-  const maintenanceBarrier = deps?.maintenanceBarrier ?? new MaintenanceBarrier();
   sessionRoutes(app, {
     mgr, tht: tht as ThtRunner, hub, getSettings, readiness, listModels, workspaceRegistry,
     dwhPrecheck: config.dwhPrecheck,
@@ -105,17 +111,14 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
     maintenanceBarrier,
   });
   app.post("/internal/maintenance/activate", async (req, reply) => {
-    if (!isLoopback(req.ip) || req.principal?.subject !== "thothctl-maintenance") return reply.code(403).send({ error: "loopback maintenance control required" });
     await maintenanceBarrier.activate();
     return maintenanceBarrier.status();
   });
   app.post("/internal/maintenance/deactivate", async (req, reply) => {
-    if (!isLoopback(req.ip) || req.principal?.subject !== "thothctl-maintenance") return reply.code(403).send({ error: "loopback maintenance control required" });
     maintenanceBarrier.deactivate();
     return maintenanceBarrier.status();
   });
   app.get("/internal/maintenance/status", async (req, reply) => {
-    if (!isLoopback(req.ip) || req.principal?.subject !== "thothctl-maintenance") return reply.code(403).send({ error: "loopback maintenance control required" });
     return maintenanceBarrier.status();
   });
   sqlRoutes(app, { tht: tht as ThtRunner, getSettings });
@@ -127,3 +130,6 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
 }
 
 function isLoopback(ip: string): boolean { return ip === "127.0.0.1" || ip === "::1" || ip === "::ffff:127.0.0.1"; }
+function isMaintenanceControl(url: string): boolean {
+  return /^\/internal\/maintenance\/(?:activate|deactivate|status)(?:\?|$)/.test(url);
+}

backend/src/config.ts

@@ -196,6 +196,7 @@ export function loadConfig(env: Record<string, string | undefined>): AppConfig {
     maxImportBytes: positiveImportLimit(env.THT_WORKSPACE_MAX_IMPORT_BYTES, 10 * 1024 * 1024),
     maxImportEntries: positiveImportLimit(env.THT_WORKSPACE_MAX_IMPORT_ENTRIES, 32),
   };
+  const settingsFile = env.SETTINGS_FILE ?? "data/settings.json";
   return {
     host: env.HOST ?? "127.0.0.1",
     port: Number(env.PORT ?? 8787),
@@ -206,8 +207,8 @@ export function loadConfig(env: Record<string, string | undefined>): AppConfig {
     sessionStorage,
     defaults: { provider: env.PI_PROVIDER, model: env.PI_MODEL, thinking: env.PI_THINKING },
     maxPiProcesses: Number(env.MAX_PI_PROCESSES ?? 4),
-    settingsFile: env.SETTINGS_FILE ?? "data/settings.json",
-    maintenanceFile: env.THT_MAINTENANCE_FILE ?? "data/maintenance.json",
+    settingsFile,
+    maintenanceFile: env.THT_MAINTENANCE_FILE ?? path.join(path.dirname(settingsFile), "maintenance.json"),
     dataRoot: env.THT_DATA_ROOT,
     ollamaEnsureTimeoutMs: Number(env.OLLAMA_ENSURE_TIMEOUT_MS ?? 60000),
     secretsFile,

backend/src/runtime/maintenance-gate.ts

@@ -1,9 +1,25 @@
-/** An in-process admission barrier. A lease spans the complete create/resume decision. */
+import {
+  closeSync,
+  existsSync,
+  fsyncSync,
+  mkdirSync,
+  openSync,
+  renameSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
+import { dirname } from "node:path";
+
+/** A durable admission barrier. A lease spans the complete create/resume decision. */
 export class MaintenanceBarrier {
-  private active = false;
+  private active: boolean;
   private admissions = 0;
   private waiters: (() => void)[] = [];
 
+  constructor(private readonly markerFile?: string) {
+    this.active = markerFile === undefined ? false : existsSync(markerFile);
+  }
+
   acquire(): (() => void) | undefined {
     if (this.active) return undefined;
     this.admissions += 1;
@@ -17,11 +33,48 @@ export class MaintenanceBarrier {
   }
 
   async activate(): Promise<void> {
+    this.persistMarker();
     this.active = true;
     if (this.admissions === 0) return;
     await new Promise<void>((resolve) => this.waiters.push(resolve));
   }
 
-  deactivate(): void { this.active = false; }
+  deactivate(): void {
+    this.removeMarker();
+    this.active = false;
+  }
   status(): { active: boolean; admissions: number } { return { active: this.active, admissions: this.admissions }; }
+
+  private persistMarker(): void {
+    if (!this.markerFile) return;
+    const directory = dirname(this.markerFile);
+    mkdirSync(directory, { recursive: true });
+    const temporary = `${this.markerFile}.tmp-${process.pid}-${Date.now()}`;
+    const fd = openSync(temporary, "wx", 0o600);
+    try {
+      writeFileSync(fd, '{"version":1,"active":true}\n', "utf8");
+      fsyncSync(fd);
+    } finally {
+      closeSync(fd);
+    }
+    try {
+      renameSync(temporary, this.markerFile);
+      syncDirectory(directory);
+    } catch (error) {
+      try { unlinkSync(temporary); } catch { /* already renamed or best-effort cleanup */ }
+      throw error;
+    }
+  }
+
+  private removeMarker(): void {
+    if (!this.markerFile || !existsSync(this.markerFile)) return;
+    unlinkSync(this.markerFile);
+    syncDirectory(dirname(this.markerFile));
+  }
+}
+
+function syncDirectory(directory: string): void {
+  if (process.platform === "win32") return;
+  const fd = openSync(directory, "r");
+  try { fsyncSync(fd); } finally { closeSync(fd); }
 }

backend/src/settings/settings-store.ts

@@ -1,4 +1,13 @@
-import { closeSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import {
+  closeSync,
+  fsyncSync,
+  mkdirSync,
+  openSync,
+  readFileSync,
+  renameSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
 import { dirname } from "node:path";
 import type { AppConfig } from "../config.js";
 
@@ -9,6 +18,10 @@ export interface Settings {
   thinking?: string;
 }
 
+export interface SettingsDurability {
+  syncDirectory(directory: string): void;
+}
+
 /**
  * Settings files are installation defaults only. Personal workspace/model/thinking choices
  * belong to the browser and must never be written back here by request handlers.
@@ -27,23 +40,65 @@ export function loadSettings(cfg: AppConfig): Settings {
 }
 
 /** Persist settings (pretty JSON). Creates the parent directory if needed. */
-export function saveSettings(cfg: AppConfig, s: Settings): Settings {
+export function saveSettings(
+  cfg: AppConfig,
+  s: Settings,
+  durability: SettingsDurability = defaultDurability,
+): Settings {
   mkdirSync(dirname(cfg.settingsFile), { recursive: true });
   const directory = dirname(cfg.settingsFile);
-  const temporary = `${cfg.settingsFile}.tmp-${process.pid}-${Date.now()}`;
-  const fd = openSync(temporary, "wx", 0o600);
+  let previous: Buffer | undefined;
   try {
-    writeFileSync(fd, JSON.stringify(s, null, 2) + "\n", "utf8");
-    fsyncSync(fd);
-  } finally {
-    closeSync(fd);
+    previous = readFileSync(cfg.settingsFile);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
   }
-  renameSync(temporary, cfg.settingsFile);
-  // The core image runs Linux. Keep the directory acknowledgement explicit there; Windows
-  // filesystem replacement semantics are delegated to the host-side Go durable writer.
-  if (process.platform !== "win32") {
-    const dirFd = openSync(directory, "r");
-    try { fsyncSync(dirFd); } finally { closeSync(dirFd); }
+
+  replaceSettingsFile(cfg.settingsFile, Buffer.from(JSON.stringify(s, null, 2) + "\n", "utf8"));
+  try {
+    durability.syncDirectory(directory);
+  } catch (durabilityError) {
+    try {
+      if (previous === undefined) unlinkSync(cfg.settingsFile);
+      else replaceSettingsFile(cfg.settingsFile, previous);
+      durability.syncDirectory(directory);
+    } catch {
+      throw new Error("settings durability failed and previous settings could not be restored", {
+        cause: durabilityError,
+      });
+    }
+    throw durabilityError;
   }
   return s;
 }
+
+let temporarySequence = 0;
+
+function replaceSettingsFile(path: string, contents: Buffer): void {
+  const temporary = `${path}.tmp-${process.pid}-${Date.now()}-${temporarySequence++}`;
+  const fd = openSync(temporary, "wx", 0o600);
+  try {
+    writeFileSync(fd, contents);
+    fsyncSync(fd);
+  } catch (error) {
+    try { closeSync(fd); } catch { /* preserve the write error */ }
+    try { unlinkSync(temporary); } catch { /* best effort */ }
+    throw error;
+  }
+  closeSync(fd);
+  try {
+    renameSync(temporary, path);
+  } catch (error) {
+    try { unlinkSync(temporary); } catch { /* best effort */ }
+    throw error;
+  }
+}
+
+const defaultDurability: SettingsDurability = {
+  syncDirectory(directory: string): void {
+    // The core image runs Linux. Windows durability is owned by the host-side Go executable.
+    if (process.platform === "win32") return;
+    const dirFd = openSync(directory, "r");
+    try { fsyncSync(dirFd); } finally { closeSync(dirFd); }
+  },
+};

backend/test/config.test.ts

@@ -17,6 +17,7 @@ test("loadConfig accepts container listening and runtime paths", () => {
     thtBin: "/opt/venv/bin/tht",
     piBin: "/usr/local/bin/pi",
     settingsFile: "/data/settings/settings.json",
+    maintenanceFile: "/data/settings/maintenance.json",
     dataRoot: "/data",
   });
 });
@@ -29,6 +30,7 @@ test("loadConfig keeps local development defaults", () => {
     thtBin: "tht",
     piBin: "pi",
     settingsFile: "data/settings.json",
+    maintenanceFile: "data/maintenance.json",
     workspaceRegistry: {
       root: "/data/workspace-registry",
       branch: "main",

backend/test/fixtures/sessions-scope-all.json

@@ -0,0 +1,14 @@
+[
+  {
+    "id": "open-session",
+    "status": "open",
+    "archived": false,
+    "active": false
+  },
+  {
+    "id": "finished-session",
+    "status": "finalized",
+    "archived": false,
+    "active": false
+  }
+]

backend/test/maintenance-gate.test.ts

@@ -1,4 +1,7 @@
 import { test, expect } from "vitest";
+import { existsSync, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { MaintenanceBarrier } from "../src/runtime/maintenance-gate.js";
 
 test("activation waits for an in-flight admission lease and rejects later admissions", async () => {
@@ -17,3 +20,28 @@ test("activation waits for an in-flight admission lease and rejects later admiss
   gate.deactivate();
   expect(gate.acquire()).toBeTypeOf("function");
 });
+
+test("durable activation survives recreation and deactivation removes the marker first", async () => {
+  const directory = mkdtempSync(join(tmpdir(), "tht-maintenance-gate-"));
+  const marker = join(directory, "maintenance.json");
+  try {
+    const first = new MaintenanceBarrier(marker);
+    const release = first.acquire();
+    expect(release).toBeTypeOf("function");
+    const activation = first.activate();
+    expect(existsSync(marker)).toBe(true);
+    expect(first.acquire()).toBeUndefined();
+    expect(first.status()).toEqual({ active: true, admissions: 1 });
+    release?.();
+    await activation;
+    expect(first.status()).toEqual({ active: true, admissions: 0 });
+
+    const recreated = new MaintenanceBarrier(marker);
+    expect(recreated.status()).toEqual({ active: true, admissions: 0 });
+    recreated.deactivate();
+    expect(existsSync(marker)).toBe(false);
+    expect(recreated.status()).toEqual({ active: false, admissions: 0 });
+  } finally {
+    rmSync(directory, { recursive: true, force: true });
+  }
+});

backend/test/routes-sessions.test.ts

@@ -2,7 +2,7 @@ import { test, expect, vi } from "vitest";
 import { spawn as nodeSpawn } from "node:child_process";
 import path from "node:path";
 import os from "node:os";
-import { chmodSync, unlinkSync, writeFileSync, mkdtempSync, rmSync } from "node:fs";
+import { chmodSync, readFileSync, unlinkSync, writeFileSync, mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { buildApp as buildRealApp } from "../src/app.js";
 import { loadConfig } from "../src/config.js";
@@ -97,7 +97,7 @@ test("maintenance rejects new and resumed session admission without interrupting
   expect(resume.json()).toEqual(create.json());
 });
 
-test("a server-profile marker does not weaken the in-process maintenance gate", async () => {
+test("a durable maintenance marker initializes admission closed after backend recreation", async () => {
   const dir = mkdtempSync(path.join(tmpdir(), "tht-maintenance-"));
   try {
     const marker = path.join(dir, "maintenance.json");
@@ -108,12 +108,71 @@ test("a server-profile marker does not weaken the in-process maintenance gate",
     const response = await app.inject({
       method: "POST", url: "/sessions", headers: aliceHeaders, payload: { question: "q" },
     });
-    expect(response.statusCode).not.toBe(503);
+    expect(response.statusCode).toBe(503);
+    expect(response.json()).toMatchObject({ code: "maintenance" });
   } finally {
     rmSync(dir, { recursive: true, force: true });
   }
 });
 
+test.each(["none", "upstream"] as const)(
+  "maintenance control is loopback-only and independent of %s authentication",
+  async (authMode) => {
+    const dir = mkdtempSync(path.join(tmpdir(), "tht-maintenance-control-"));
+    const marker = path.join(dir, "maintenance.json");
+    try {
+      const app = buildApp(loadConfig({
+        AUTH_MODE: authMode,
+        THT_HARNESS_DIR: "../harness",
+        THT_MAINTENANCE_FILE: marker,
+      }), { thtRunner: {} as any });
+
+      const activated = await app.inject({ method: "POST", url: "/internal/maintenance/activate" });
+      expect(activated.statusCode).toBe(200);
+      expect(activated.json()).toEqual({ active: true, admissions: 0 });
+
+      const spoofedProxy = await app.inject({
+        method: "POST",
+        url: "/internal/maintenance/deactivate",
+        remoteAddress: "172.30.0.9",
+        headers: {
+          "x-thoth-principal-subject": "thothctl-maintenance",
+          "x-thoth-is-admin": "1",
+        },
+      });
+      expect(spoofedProxy.statusCode).toBe(403);
+
+      const status = await app.inject({ method: "GET", url: "/internal/maintenance/status" });
+      expect(status.json()).toEqual({ active: true, admissions: 0 });
+      const deactivated = await app.inject({ method: "POST", url: "/internal/maintenance/deactivate" });
+      expect(deactivated.json()).toEqual({ active: false, admissions: 0 });
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  },
+);
+
+test("admin all-sessions response matches the authenticated lifecycle wire fixture", async () => {
+  const fixture = JSON.parse(readFileSync(
+    path.join(import.meta.dirname, "fixtures", "sessions-scope-all.json"),
+    "utf8",
+  ));
+  const app = buildApp(loadConfig({ AUTH_MODE: "upstream", THT_HARNESS_DIR: "../harness" }), {
+    thtRunner: {
+      withPrincipal: () => ({ sessionList: async () => fixture.map(({ active: _active, ...row }: any) => row) }),
+    } as any,
+    mgr: { get: () => undefined } as any,
+    workspaceRegistry: defaultWorkspaceRegistry as any,
+  });
+  const response = await app.inject({
+    method: "GET",
+    url: "/sessions?scope=all",
+    headers: { ...aliceHeaders, "x-thoth-is-admin": "1" },
+  });
+  expect(response.statusCode).toBe(200);
+  expect(response.json()).toEqual(fixture);
+});
+
 test("session routes conceal foreign or missing sessions and deny SSE before it subscribes", async () => {
   let subscribed = false;
   const app = buildApp(loadConfig({ AUTH_MODE: "upstream", THT_HARNESS_DIR: "../harness" }), {

backend/test/settings-store.test.ts

@@ -1,5 +1,5 @@
 import { test, expect } from "vitest";
-import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { closeSync, fsyncSync, mkdtempSync, openSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { loadSettings, saveSettings } from "../src/settings/settings-store.js";
@@ -45,3 +45,28 @@ test("loadConfig sets settingsFile from SETTINGS_FILE, default data/settings.jso
   expect(loadConfig({}).settingsFile).toBe("data/settings.json");
   expect(loadConfig({ SETTINGS_FILE: "/x/y.json" }).settingsFile).toBe("/x/y.json");
 });
+
+test("saveSettings restores the previous file when post-rename directory durability fails", () => {
+  const dir = mkdtempSync(join(tmpdir(), "tht-set-transaction-"));
+  try {
+    const cfg = cfgWith(join(dir, "settings.json"));
+    saveSettings(cfg, { provider: "old", model: "old-model", thinking: "low" });
+    let syncs = 0;
+    expect(() => saveSettings(
+      cfg,
+      { provider: "new", model: "new-model", thinking: "high" },
+      {
+        syncDirectory(directory: string) {
+          syncs += 1;
+          if (syncs === 1) throw new Error("injected directory fsync failure");
+          const fd = openSync(directory, "r");
+          try { fsyncSync(fd); } finally { closeSync(fd); }
+        },
+      },
+    )).toThrow(/directory fsync failure/);
+    expect(loadSettings(cfg)).toEqual({ provider: "old", model: "old-model", thinking: "low" });
+    expect(syncs).toBeGreaterThanOrEqual(2);
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});