mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix: harden embedded Pi lifecycle recovery

Browse Source
This commit is contained in:
marcopan
2026-08-04 23:14:47 +02:00
parent a368889838
commit 935bb1db0e
15 changed files with 572 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/package-lock.json
Generated
+6 -6
View File
@@ -1455,9 +1455,9 @@
}
},
"node_modules/fast-uri": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.2.tgz",
"integrity": "sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==",
"version": "3.1.5",
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz",
"integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==",
"funding": [
{
"type": "github",
@@ -1529,9 +1529,9 @@
}
},
"node_modules/find-my-way": {
"version": "9.6.0",
"resolved": "https://registry.npmjs.org/find-my-way/-/find-my-way-9.6.0.tgz",
"integrity": "sha512-Zf4Xve4RymLl7NgaavNebZ01joJ8MfVerOG43wy7SHLO+r+K0C6d/SE0BiR7AV5V1VOCFlOP7ecdo+I4qmiHrQ==",
"version": "9.7.0",
"resolved": "https://registry.npmjs.org/find-my-way/-/find-my-way-9.7.0.tgz",
"integrity": "sha512-f2JHn75x2JlwUwLenZypgczR7YWMb/uO9BvUXtus+JMgkbIkLADd38cI4EiV+OQqrGo1Zlq6V8wnqMJ8e62wUQ==",
"license": "MIT",
"dependencies": {
"fast-deep-equal": "^3.1.3",
backend/src/app.ts
+2
View File
@@ -117,6 +117,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
} catch {
return reply.code(500).send({
...maintenanceBarrier.status(),
code: "maintenance_durability_failed",
error: "maintenance activation durability was not acknowledged",
});
}
@@ -128,6 +129,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
} catch {
return reply.code(500).send({
...maintenanceBarrier.status(),
code: "maintenance_durability_failed",
error: "maintenance deactivation durability was not acknowledged",
});
}
backend/src/runtime/maintenance-gate.ts
+26 -9
View File
@@ -11,6 +11,8 @@ import {
import { dirname } from "node:path";
export interface MaintenanceDurability {
writeFile?(descriptor: number, contents: string): void;
syncFile?(descriptor: number): void;
syncDirectory(directory: string): void;
}
@@ -19,6 +21,7 @@ export class MaintenanceBarrier {
private active: boolean;
private admissions = 0;
private waiters: (() => void)[] = [];
private recoveryRequired = false;
constructor(
private readonly markerFile?: string,
@@ -44,11 +47,14 @@ export class MaintenanceBarrier {
let persistError: unknown;
if (this.markerFile === undefined) {
this.active = true;
this.recoveryRequired = false;
} else {
try {
this.persistMarker();
this.recoveryRequired = false;
} catch (error) {
persistError = error;
this.recoveryRequired = true;
} finally {
this.reconcileActive();
}
@@ -63,17 +69,24 @@ export class MaintenanceBarrier {
deactivate(): void {
if (this.markerFile === undefined) {
this.active = false;
this.recoveryRequired = false;
return;
}
try {
this.removeMarker();
this.recoveryRequired = false;
} catch (error) {
try { this.persistMarker(); } catch { /* marker existence is reconciled below */ }
this.recoveryRequired = true;
throw error;
} finally {
this.reconcileActive();
}
}
status(): { active: boolean; admissions: number } {
status(): { active: boolean; admissions: number; recoveryRequired?: true } {
this.reconcileActive();
return { active: this.active, admissions: this.admissions };
const status = { active: this.active, admissions: this.admissions };
return this.recoveryRequired ? { ...status, recoveryRequired: true } : status;
}
private reconcileActive(): void {
@@ -86,24 +99,28 @@ export class MaintenanceBarrier {
mkdirSync(directory, { recursive: true });
const temporary = `${this.markerFile}.tmp-${process.pid}-${Date.now()}`;
const fd = openSync(temporary, "wx", 0o600);
let closed = false;
try {
writeFileSync(fd, '{"version":1,"active":true}\n', "utf8");
fsyncSync(fd);
} finally {
const contents = '{"version":1,"active":true}\n';
if (this.durability.writeFile) this.durability.writeFile(fd, contents);
else writeFileSync(fd, contents, "utf8");
if (this.durability.syncFile) this.durability.syncFile(fd);
else fsyncSync(fd);
closeSync(fd);
}
try {
closed = true;
renameSync(temporary, this.markerFile);
this.durability.syncDirectory(directory);
} catch (error) {
if (!closed) try { closeSync(fd); } catch { /* preserve the original failure */ }
try { unlinkSync(temporary); } catch { /* already renamed or best-effort cleanup */ }
throw error;
}
}
private removeMarker(): void {
if (!this.markerFile || !existsSync(this.markerFile)) return;
unlinkSync(this.markerFile);
if (!this.markerFile) return;
if (existsSync(this.markerFile)) unlinkSync(this.markerFile);
else if (!this.recoveryRequired) return;
this.durability.syncDirectory(dirname(this.markerFile));
}
}
backend/test/maintenance-gate.test.ts
+32 -5
View File
@@ -1,5 +1,5 @@
import { test, expect } from "vitest";
import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { existsSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { MaintenanceBarrier } from "../src/runtime/maintenance-gate.js";
@@ -55,7 +55,7 @@ test("activation reconciles active state when directory fsync fails after marker
});
await expect(gate.activate()).rejects.toThrow(/post-rename fsync failure/);
expect(existsSync(marker)).toBe(true);
expect(gate.status()).toEqual({ active: true, admissions: 0 });
expect(gate.status()).toEqual({ active: true, admissions: 0, recoveryRequired: true });
expect(gate.acquire()).toBeUndefined();
const recovered = new MaintenanceBarrier(marker);
@@ -66,6 +66,33 @@ test("activation reconciles active state when directory fsync fails after marker
}
});
test.each(["write", "fsync"] as const)(
"activation cleans its marker temp file after a pre-rename %s failure",
async (failure) => {
const directory = mkdtempSync(join(tmpdir(), "tht-maintenance-temp-cleanup-"));
const marker = join(directory, "maintenance.json");
try {
const gate = new MaintenanceBarrier(marker, {
writeFile(descriptor, contents) {
if (failure === "write") throw new Error("injected marker write failure");
writeFileSync(descriptor, contents, "utf8");
},
syncFile() {
if (failure === "fsync") throw new Error("injected marker fsync failure");
},
syncDirectory() {},
});
await expect(gate.activate()).rejects.toThrow(`injected marker ${failure} failure`);
expect(existsSync(marker)).toBe(false);
expect(readdirSync(directory).filter((entry) => entry.includes(".tmp-"))).toEqual([]);
expect(gate.status()).toEqual({ active: false, admissions: 0, recoveryRequired: true });
} finally {
rmSync(directory, { recursive: true, force: true });
}
},
);
test("deactivation reconciles inactive state when directory fsync fails after marker removal", async () => {
const directory = mkdtempSync(join(tmpdir(), "tht-maintenance-deactivate-fsync-"));
const marker = join(directory, "maintenance.json");
@@ -79,9 +106,9 @@ test("deactivation reconciles inactive state when directory fsync fails after ma
await gate.activate();
failSync = true;
expect(() => gate.deactivate()).toThrow(/post-remove fsync failure/);
expect(existsSync(marker)).toBe(false);
expect(gate.status()).toEqual({ active: false, admissions: 0 });
expect(gate.acquire()).toBeTypeOf("function");
expect(existsSync(marker)).toBe(true);
expect(gate.status()).toEqual({ active: true, admissions: 0, recoveryRequired: true });
expect(gate.acquire()).toBeUndefined();
} finally {
rmSync(directory, { recursive: true, force: true });
}
backend/test/routes-sessions.test.ts
+13 -3
View File
@@ -170,15 +170,25 @@ test("maintenance endpoints report marker-derived state after post-rename and po
const activated = await app.inject({ method: "POST", url: "/internal/maintenance/activate" });
expect(activated.statusCode).toBe(500);
expect(activated.json()).toMatchObject({ active: true, admissions: 0 });
expect(activated.json()).toMatchObject({
active: true,
admissions: 0,
recoveryRequired: true,
code: "maintenance_durability_failed",
});
failSync = false;
expect((await app.inject({ method: "GET", url: "/internal/maintenance/status" })).json())
.toEqual({ active: true, admissions: 0 });
.toEqual({ active: true, admissions: 0, recoveryRequired: true });
failSync = true;
const deactivated = await app.inject({ method: "POST", url: "/internal/maintenance/deactivate" });
expect(deactivated.statusCode).toBe(500);
expect(deactivated.json()).toMatchObject({ active: false, admissions: 0 });
expect(deactivated.json()).toMatchObject({
active: true,
admissions: 0,
recoveryRequired: true,
code: "maintenance_durability_failed",
});
} finally {
rmSync(dir, { recursive: true, force: true });
}