fix: harden Pi lifecycle recovery

backend/src/app.ts

@@ -87,16 +87,22 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
     return effectiveSettings(config, loadSettings(config));
   };
 
+  const maintenanceBarrier = deps?.maintenanceBarrier ?? new MaintenanceBarrier(config.maintenanceFile);
   const authenticate = authPreHandler(config.authMode);
   app.addHook("preHandler", async (req, reply) => {
     // Process readiness is intentionally unauthenticated for local container/proxy probes.
     if (req.url === "/health" || req.url === "/health/dwh") return;
+    if (isMaintenanceControl(req.url)) {
+      if (!isLoopback(req.ip)) {
+        return reply.code(403).send({ error: "loopback maintenance control required" });
+      }
+      return;
+    }
     return authenticate(req, reply);
   });
   app.get("/health", async () => ({ status: "ok" }));
   app.get("/health/dwh", async () => tht.dbPing());
   app.get("/me", async (req) => getPrincipal(req));
-  const maintenanceBarrier = deps?.maintenanceBarrier ?? new MaintenanceBarrier();
   sessionRoutes(app, {
     mgr, tht: tht as ThtRunner, hub, getSettings, readiness, listModels, workspaceRegistry,
     dwhPrecheck: config.dwhPrecheck,
@@ -105,17 +111,14 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
     maintenanceBarrier,
   });
   app.post("/internal/maintenance/activate", async (req, reply) => {
-    if (!isLoopback(req.ip) || req.principal?.subject !== "thothctl-maintenance") return reply.code(403).send({ error: "loopback maintenance control required" });
     await maintenanceBarrier.activate();
     return maintenanceBarrier.status();
   });
   app.post("/internal/maintenance/deactivate", async (req, reply) => {
-    if (!isLoopback(req.ip) || req.principal?.subject !== "thothctl-maintenance") return reply.code(403).send({ error: "loopback maintenance control required" });
     maintenanceBarrier.deactivate();
     return maintenanceBarrier.status();
   });
   app.get("/internal/maintenance/status", async (req, reply) => {
-    if (!isLoopback(req.ip) || req.principal?.subject !== "thothctl-maintenance") return reply.code(403).send({ error: "loopback maintenance control required" });
     return maintenanceBarrier.status();
   });
   sqlRoutes(app, { tht: tht as ThtRunner, getSettings });
@@ -127,3 +130,6 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
 }
 
 function isLoopback(ip: string): boolean { return ip === "127.0.0.1" || ip === "::1" || ip === "::ffff:127.0.0.1"; }
+function isMaintenanceControl(url: string): boolean {
+  return /^\/internal\/maintenance\/(?:activate|deactivate|status)(?:\?|$)/.test(url);
+}

backend/src/config.ts

@@ -196,6 +196,7 @@ export function loadConfig(env: Record<string, string | undefined>): AppConfig {
     maxImportBytes: positiveImportLimit(env.THT_WORKSPACE_MAX_IMPORT_BYTES, 10 * 1024 * 1024),
     maxImportEntries: positiveImportLimit(env.THT_WORKSPACE_MAX_IMPORT_ENTRIES, 32),
   };
+  const settingsFile = env.SETTINGS_FILE ?? "data/settings.json";
   return {
     host: env.HOST ?? "127.0.0.1",
     port: Number(env.PORT ?? 8787),
@@ -206,8 +207,8 @@ export function loadConfig(env: Record<string, string | undefined>): AppConfig {
     sessionStorage,
     defaults: { provider: env.PI_PROVIDER, model: env.PI_MODEL, thinking: env.PI_THINKING },
     maxPiProcesses: Number(env.MAX_PI_PROCESSES ?? 4),
-    settingsFile: env.SETTINGS_FILE ?? "data/settings.json",
-    maintenanceFile: env.THT_MAINTENANCE_FILE ?? "data/maintenance.json",
+    settingsFile,
+    maintenanceFile: env.THT_MAINTENANCE_FILE ?? path.join(path.dirname(settingsFile), "maintenance.json"),
     dataRoot: env.THT_DATA_ROOT,
     ollamaEnsureTimeoutMs: Number(env.OLLAMA_ENSURE_TIMEOUT_MS ?? 60000),
     secretsFile,

backend/src/runtime/maintenance-gate.ts

@@ -1,9 +1,25 @@
-/** An in-process admission barrier. A lease spans the complete create/resume decision. */
+import {
+  closeSync,
+  existsSync,
+  fsyncSync,
+  mkdirSync,
+  openSync,
+  renameSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
+import { dirname } from "node:path";
+
+/** A durable admission barrier. A lease spans the complete create/resume decision. */
 export class MaintenanceBarrier {
-  private active = false;
+  private active: boolean;
   private admissions = 0;
   private waiters: (() => void)[] = [];
 
+  constructor(private readonly markerFile?: string) {
+    this.active = markerFile === undefined ? false : existsSync(markerFile);
+  }
+
   acquire(): (() => void) | undefined {
     if (this.active) return undefined;
     this.admissions += 1;
@@ -17,11 +33,48 @@ export class MaintenanceBarrier {
   }
 
   async activate(): Promise<void> {
+    this.persistMarker();
     this.active = true;
     if (this.admissions === 0) return;
     await new Promise<void>((resolve) => this.waiters.push(resolve));
   }
 
-  deactivate(): void { this.active = false; }
+  deactivate(): void {
+    this.removeMarker();
+    this.active = false;
+  }
   status(): { active: boolean; admissions: number } { return { active: this.active, admissions: this.admissions }; }
+
+  private persistMarker(): void {
+    if (!this.markerFile) return;
+    const directory = dirname(this.markerFile);
+    mkdirSync(directory, { recursive: true });
+    const temporary = `${this.markerFile}.tmp-${process.pid}-${Date.now()}`;
+    const fd = openSync(temporary, "wx", 0o600);
+    try {
+      writeFileSync(fd, '{"version":1,"active":true}\n', "utf8");
+      fsyncSync(fd);
+    } finally {
+      closeSync(fd);
+    }
+    try {
+      renameSync(temporary, this.markerFile);
+      syncDirectory(directory);
+    } catch (error) {
+      try { unlinkSync(temporary); } catch { /* already renamed or best-effort cleanup */ }
+      throw error;
+    }
+  }
+
+  private removeMarker(): void {
+    if (!this.markerFile || !existsSync(this.markerFile)) return;
+    unlinkSync(this.markerFile);
+    syncDirectory(dirname(this.markerFile));
+  }
+}
+
+function syncDirectory(directory: string): void {
+  if (process.platform === "win32") return;
+  const fd = openSync(directory, "r");
+  try { fsyncSync(fd); } finally { closeSync(fd); }
 }

backend/src/settings/settings-store.ts

@@ -1,4 +1,13 @@
-import { closeSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import {
+  closeSync,
+  fsyncSync,
+  mkdirSync,
+  openSync,
+  readFileSync,
+  renameSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
 import { dirname } from "node:path";
 import type { AppConfig } from "../config.js";
 
@@ -9,6 +18,10 @@ export interface Settings {
   thinking?: string;
 }
 
+export interface SettingsDurability {
+  syncDirectory(directory: string): void;
+}
+
 /**
  * Settings files are installation defaults only. Personal workspace/model/thinking choices
  * belong to the browser and must never be written back here by request handlers.
@@ -27,23 +40,65 @@ export function loadSettings(cfg: AppConfig): Settings {
 }
 
 /** Persist settings (pretty JSON). Creates the parent directory if needed. */
-export function saveSettings(cfg: AppConfig, s: Settings): Settings {
+export function saveSettings(
+  cfg: AppConfig,
+  s: Settings,
+  durability: SettingsDurability = defaultDurability,
+): Settings {
   mkdirSync(dirname(cfg.settingsFile), { recursive: true });
   const directory = dirname(cfg.settingsFile);
-  const temporary = `${cfg.settingsFile}.tmp-${process.pid}-${Date.now()}`;
-  const fd = openSync(temporary, "wx", 0o600);
+  let previous: Buffer | undefined;
   try {
-    writeFileSync(fd, JSON.stringify(s, null, 2) + "\n", "utf8");
-    fsyncSync(fd);
-  } finally {
-    closeSync(fd);
+    previous = readFileSync(cfg.settingsFile);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
   }
-  renameSync(temporary, cfg.settingsFile);
-  // The core image runs Linux. Keep the directory acknowledgement explicit there; Windows
-  // filesystem replacement semantics are delegated to the host-side Go durable writer.
-  if (process.platform !== "win32") {
-    const dirFd = openSync(directory, "r");
-    try { fsyncSync(dirFd); } finally { closeSync(dirFd); }
+
+  replaceSettingsFile(cfg.settingsFile, Buffer.from(JSON.stringify(s, null, 2) + "\n", "utf8"));
+  try {
+    durability.syncDirectory(directory);
+  } catch (durabilityError) {
+    try {
+      if (previous === undefined) unlinkSync(cfg.settingsFile);
+      else replaceSettingsFile(cfg.settingsFile, previous);
+      durability.syncDirectory(directory);
+    } catch {
+      throw new Error("settings durability failed and previous settings could not be restored", {
+        cause: durabilityError,
+      });
+    }
+    throw durabilityError;
   }
   return s;
 }
+
+let temporarySequence = 0;
+
+function replaceSettingsFile(path: string, contents: Buffer): void {
+  const temporary = `${path}.tmp-${process.pid}-${Date.now()}-${temporarySequence++}`;
+  const fd = openSync(temporary, "wx", 0o600);
+  try {
+    writeFileSync(fd, contents);
+    fsyncSync(fd);
+  } catch (error) {
+    try { closeSync(fd); } catch { /* preserve the write error */ }
+    try { unlinkSync(temporary); } catch { /* best effort */ }
+    throw error;
+  }
+  closeSync(fd);
+  try {
+    renameSync(temporary, path);
+  } catch (error) {
+    try { unlinkSync(temporary); } catch { /* best effort */ }
+    throw error;
+  }
+}
+
+const defaultDurability: SettingsDurability = {
+  syncDirectory(directory: string): void {
+    // The core image runs Linux. Windows durability is owned by the host-side Go executable.
+    if (process.platform === "win32") return;
+    const dirFd = openSync(directory, "r");
+    try { fsyncSync(dirFd); } finally { closeSync(dirFd); }
+  },
+};