test(auth): gate local and OIDC authentication release
This commit is contained in:
@@ -74,6 +74,38 @@ jobs:
|
|||||||
npx vitest run
|
npx vitest run
|
||||||
npx tsc -b
|
npx tsc -b
|
||||||
|
|
||||||
|
authentication-browser:
|
||||||
|
name: Hermetic authentication browser gate
|
||||||
|
needs: deterministic
|
||||||
|
runs-on: ubuntu-24.04
|
||||||
|
timeout-minutes: 30
|
||||||
|
steps:
|
||||||
|
- name: Check out source
|
||||||
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
- name: Set up Node.js
|
||||||
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||||
|
with:
|
||||||
|
node-version: "24.16.0"
|
||||||
|
package-manager-cache: false
|
||||||
|
- name: Set up Go
|
||||||
|
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||||
|
with:
|
||||||
|
go-version: "1.26.5"
|
||||||
|
cache-dependency-path: tools/tht/go.sum
|
||||||
|
- name: Install backend dependencies
|
||||||
|
working-directory: backend
|
||||||
|
run: npm ci
|
||||||
|
- name: Install frontend dependencies
|
||||||
|
working-directory: frontend
|
||||||
|
run: npm ci
|
||||||
|
- name: Install Chromium for Playwright
|
||||||
|
working-directory: frontend
|
||||||
|
run: npx playwright install --with-deps chromium
|
||||||
|
- name: Run authentication browser smoke
|
||||||
|
run: bash scripts/authentication-smoke.sh
|
||||||
|
|
||||||
linux-docker:
|
linux-docker:
|
||||||
name: Linux Docker deployment and rollback
|
name: Linux Docker deployment and rollback
|
||||||
runs-on: ubuntu-24.04
|
runs-on: ubuntu-24.04
|
||||||
|
|||||||
+25
-17
@@ -7,26 +7,34 @@
|
|||||||
> ThothII per il repository (app + CLI `thothctl`), (3) come usare l'applicazione ThothII di base
|
> ThothII per il repository (app + CLI `thothctl`), (3) come usare l'applicazione ThothII di base
|
||||||
> (sessioni, domande, gate). Il documento userà parole semplici ed esempi; i dettagli tecnici
|
> (sessioni, domande, gate). Il documento userà parole semplici ed esempi; i dettagli tecnici
|
||||||
> resteranno nei contratti esistenti. Esempio pratico completo: Policlinico San Donato.
|
> resteranno nei contratti esistenti. Esempio pratico completo: Policlinico San Donato.
|
||||||
> Last updated: 2026-08-18 (Task 14 documentation implemented; strict documentation and
|
> Last updated: 2026-08-18 (Task 15 automated evidence recorded; the authentication feature is
|
||||||
> authentication release gates remain blocking or pending as listed below).
|
> not complete or release-accepted while the required FAIL/PENDING gates listed below remain).
|
||||||
> Point a fresh session here ("read PROJECT_STATE.md") before substantial work.
|
> Point a fresh session here ("read PROJECT_STATE.md") before substantial work.
|
||||||
|
|
||||||
### Task 14 authentication documentation — implementation status (2026-08-18)
|
### Task 15 authentication release — automated evidence, release incomplete (2026-08-18)
|
||||||
|
|
||||||
- Documentation now describes local Argon2id users, ordinary and remembered session expiry,
|
- Task 13 carry-ins are closed with focused TDD: restore acquires the lifecycle lock before
|
||||||
revision invalidation, generic OIDC direct groups claims, exact group-role mapping, Authentik
|
target-dependent preflight, stages/revalidates archive bytes and hashes under that lock, and uses
|
||||||
group-view-only catalog checks, tht auth/tht doctor ordering, diagnostics, CSRF, and restore
|
an opaque installation-bound transaction capability. Focused mutation/concurrency/lock-leak
|
||||||
reauthentication.
|
tests and the full Go race/build gate pass.
|
||||||
- The Task 14 authentication-documentation smoke passes. The MkDocs `--strict` baseline remains a
|
- PASS on the pinned Node `v24.16.0`: backend 75 files / 1081 tests plus typecheck/build; frontend
|
||||||
blocking release gate: it exits 1 with 69 warnings, the same warning count before and after Task
|
61 files / 444 tests plus typecheck/build and 6 Playwright tests; hermetic loopback OIDC browser
|
||||||
14. It is not a PASS or release evidence.
|
smoke; harness 921 passed / 4 L2 deselected; authentication-documentation smoke; unified Docker
|
||||||
- Browser OIDC callback E2E, native Windows behavioral execution, PSD/manual test identities, and
|
smoke with scoped cleanup; installer shell test; relevant Compose/security contracts. The default
|
||||||
external L2 remain pending Task 15/release gates. Task 14 documentation is implemented, not
|
host Node is `v25.6.1`; it is not the release contract and no tracked `v24.19.0` pin exists.
|
||||||
release-accepted.
|
- Task15 fixture regressions in preprocess, server Pi-state, and Pi-auth Compose contracts were
|
||||||
- Task 13 has two parked restore-lock preconditions that remain mandatory before certification:
|
repaired and their focused checks pass. Static Windows amd64 cross-build and test compilation
|
||||||
lock before target-dependent preflight with archive bytes/hashes staged and revalidated inside the
|
pass for 18 Go packages.
|
||||||
lock immediately before extraction; and an opaque installation-bound transaction capability or
|
- FAIL baseline evidence: Ruff reports 192 errors; MkDocs `--strict` exits 1 with 69 warnings;
|
||||||
closure replacing convention-only lock-held helpers.
|
the canonical-install and workspace-install-doc checks report existing manual wording mismatches;
|
||||||
|
the Pi user-auth Compose check reaches an existing model allow-list mismatch; and the broad
|
||||||
|
deployment-coupling scanner sees preserved ignored private deployment material.
|
||||||
|
- PENDING evidence: native Windows execution (no KVM, `sshpass`, Windows container, or PowerShell);
|
||||||
|
real PSD/manual acceptance (no real test identity/access); L2 (no configured local secret layout);
|
||||||
|
and isolated provider-readiness Docker smoke (a host port is already occupied by an unrelated
|
||||||
|
resource). These are not PASS claims.
|
||||||
|
- **Release state: NOT COMPLETE.** Do not mark authentication release-complete until every required
|
||||||
|
gate is rerun in an eligible environment and is PASS.
|
||||||
|
|
||||||
### P3 effective configuration and `.tht-dwh` — implementation complete, automated PASS, manual PASS (2026-08-13)
|
### P3 effective configuration and `.tht-dwh` — implementation complete, automated PASS, manual PASS (2026-08-13)
|
||||||
|
|
||||||
|
|||||||
+328
@@ -0,0 +1,328 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* Hermetic loopback OIDC/AuthentiK-shaped provider for browser smoke tests.
|
||||||
|
*
|
||||||
|
* It deliberately has no network dependency and binds only to 127.0.0.1. Its
|
||||||
|
* ephemeral TLS and signing keys are test-scoped; callers receive the CA path
|
||||||
|
* needed to trust the provider from a spawned backend process.
|
||||||
|
*/
|
||||||
|
import { spawnSync } from "node:child_process";
|
||||||
|
import { createHash, generateKeyPairSync, randomBytes, sign as signRsa } from "node:crypto";
|
||||||
|
import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||||
|
import { createServer } from "node:https";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join, resolve } from "node:path";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
|
||||||
|
const LOOPBACK_HOST = "127.0.0.1";
|
||||||
|
const ISSUER_PATH = "/application/o/thothii";
|
||||||
|
const MAX_BODY_BYTES = 32 * 1024;
|
||||||
|
const VALID_IDENTITIES = new Set([
|
||||||
|
"ordinary",
|
||||||
|
"admin",
|
||||||
|
"missing-groups",
|
||||||
|
"malformed-groups",
|
||||||
|
"unmapped",
|
||||||
|
"expired",
|
||||||
|
]);
|
||||||
|
|
||||||
|
const identityClaims = Object.freeze({
|
||||||
|
ordinary: { subject: "fixture-ordinary", displayName: "Fixture ordinary", groups: ["fixture-users"] },
|
||||||
|
admin: { subject: "fixture-admin", displayName: "Fixture administrator", groups: ["fixture-admin"] },
|
||||||
|
"missing-groups": { subject: "fixture-missing-groups", displayName: "Fixture missing groups" },
|
||||||
|
"malformed-groups": { subject: "fixture-malformed-groups", displayName: "Fixture malformed groups", groups: ["fixture-users", 7] },
|
||||||
|
unmapped: { subject: "fixture-unmapped", displayName: "Fixture unmapped", groups: ["fixture-unmapped"] },
|
||||||
|
expired: { subject: "fixture-expired", displayName: "Fixture expired", groups: ["fixture-users"], expired: true },
|
||||||
|
});
|
||||||
|
|
||||||
|
function safeError(code) {
|
||||||
|
return new Error(code);
|
||||||
|
}
|
||||||
|
|
||||||
|
function ensurePrivateDirectory(directory) {
|
||||||
|
mkdirSync(directory, { recursive: true, mode: 0o700 });
|
||||||
|
chmodSync(directory, 0o700);
|
||||||
|
}
|
||||||
|
|
||||||
|
function createCertificate(directory) {
|
||||||
|
const keyFile = join(directory, "provider-key.pem");
|
||||||
|
const certificateFile = join(directory, "provider-ca.pem");
|
||||||
|
const result = spawnSync("openssl", [
|
||||||
|
"req", "-x509", "-newkey", "rsa:2048", "-sha256", "-nodes",
|
||||||
|
"-keyout", keyFile,
|
||||||
|
"-out", certificateFile,
|
||||||
|
"-subj", "/CN=127.0.0.1",
|
||||||
|
"-addext", "subjectAltName=IP:127.0.0.1",
|
||||||
|
"-days", "1",
|
||||||
|
], { stdio: "ignore" });
|
||||||
|
if (result.status !== 0) throw safeError("oidc_fixture_certificate_generation_failed");
|
||||||
|
chmodSync(keyFile, 0o600);
|
||||||
|
chmodSync(certificateFile, 0o600);
|
||||||
|
return { key: readFileSync(keyFile), cert: readFileSync(certificateFile), certificateFile };
|
||||||
|
}
|
||||||
|
|
||||||
|
function sendJson(reply, status, value) {
|
||||||
|
reply.writeHead(status, {
|
||||||
|
"cache-control": "no-store",
|
||||||
|
"content-type": "application/json; charset=utf-8",
|
||||||
|
});
|
||||||
|
reply.end(JSON.stringify(value));
|
||||||
|
}
|
||||||
|
|
||||||
|
function redirect(reply, location) {
|
||||||
|
reply.writeHead(302, { "cache-control": "no-store", location });
|
||||||
|
reply.end();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function requestBody(request) {
|
||||||
|
let size = 0;
|
||||||
|
const chunks = [];
|
||||||
|
for await (const chunk of request) {
|
||||||
|
size += chunk.length;
|
||||||
|
if (size > MAX_BODY_BYTES) throw safeError("oidc_fixture_request_too_large");
|
||||||
|
chunks.push(chunk);
|
||||||
|
}
|
||||||
|
return Buffer.concat(chunks).toString("utf8");
|
||||||
|
}
|
||||||
|
|
||||||
|
function jwt(privateKey, issuer, audience, nonce, identity) {
|
||||||
|
const claims = identityClaims[identity];
|
||||||
|
const now = Math.floor(Date.now() / 1_000);
|
||||||
|
const payload = {
|
||||||
|
iss: issuer,
|
||||||
|
sub: claims.subject,
|
||||||
|
aud: audience,
|
||||||
|
exp: now + (claims.expired ? -30 : 60),
|
||||||
|
iat: now - 1,
|
||||||
|
nonce,
|
||||||
|
name: claims.displayName,
|
||||||
|
...(Object.hasOwn(claims, "groups") ? { groups: claims.groups } : {}),
|
||||||
|
};
|
||||||
|
const header = { alg: "RS256", typ: "JWT", kid: "fixture-rs256" };
|
||||||
|
const protectedPart = Buffer.from(JSON.stringify(header)).toString("base64url");
|
||||||
|
const payloadPart = Buffer.from(JSON.stringify(payload)).toString("base64url");
|
||||||
|
const signingInput = `${protectedPart}.${payloadPart}`;
|
||||||
|
const signature = signRsa("RSA-SHA256", Buffer.from(signingInput), privateKey).toString("base64url");
|
||||||
|
return `${signingInput}.${signature}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function authorizationIdentity(identity) {
|
||||||
|
if (!VALID_IDENTITIES.has(identity)) throw safeError("oidc_fixture_identity_invalid");
|
||||||
|
return identity;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start an HTTPS test provider. The returned telemetry intentionally excludes
|
||||||
|
* transient authorization codes, browser state, nonces, and token material.
|
||||||
|
*/
|
||||||
|
export async function startFakeOidcProvider(options = {}) {
|
||||||
|
const host = options.host ?? LOOPBACK_HOST;
|
||||||
|
if (host !== LOOPBACK_HOST) throw safeError("oidc_fixture_loopback_required");
|
||||||
|
const ownsDirectory = options.directory === undefined;
|
||||||
|
const directory = options.directory ?? mkdtempSync(join(tmpdir(), "thothii-oidc-fixture-"));
|
||||||
|
ensurePrivateDirectory(directory);
|
||||||
|
const certificate = createCertificate(directory);
|
||||||
|
const { privateKey, publicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 });
|
||||||
|
const publicJwk = publicKey.export({ format: "jwk" });
|
||||||
|
const jwks = {
|
||||||
|
keys: [{ ...publicJwk, alg: "RS256", kid: "fixture-rs256", use: "sig" }],
|
||||||
|
};
|
||||||
|
const authorizations = new Map();
|
||||||
|
const deviceCodes = new Map();
|
||||||
|
let activeIdentity = authorizationIdentity(options.identity ?? "ordinary");
|
||||||
|
let lastAuthorization = undefined;
|
||||||
|
let issuer = undefined;
|
||||||
|
let baseUrl = undefined;
|
||||||
|
|
||||||
|
const server = createServer({ key: certificate.key, cert: certificate.cert }, async (request, reply) => {
|
||||||
|
try {
|
||||||
|
if (!issuer || !baseUrl || !request.url) {
|
||||||
|
sendJson(reply, 503, { error: "temporarily_unavailable" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const url = new URL(request.url, baseUrl);
|
||||||
|
const path = url.pathname;
|
||||||
|
const discoveryPath = `${ISSUER_PATH}/.well-known/openid-configuration`;
|
||||||
|
const rfc8414Path = `/.well-known/openid-configuration${ISSUER_PATH}`;
|
||||||
|
if (request.method === "GET" && (path === discoveryPath || path === rfc8414Path)) {
|
||||||
|
sendJson(reply, 200, {
|
||||||
|
issuer,
|
||||||
|
authorization_endpoint: `${issuer}authorize`,
|
||||||
|
token_endpoint: `${issuer}token`,
|
||||||
|
jwks_uri: `${issuer}jwks`,
|
||||||
|
device_authorization_endpoint: `${issuer}device_authorization`,
|
||||||
|
response_types_supported: ["code"],
|
||||||
|
subject_types_supported: ["public"],
|
||||||
|
grant_types_supported: ["authorization_code", "urn:ietf:params:oauth:grant-type:device_code"],
|
||||||
|
token_endpoint_auth_methods_supported: ["client_secret_basic", "client_secret_post"],
|
||||||
|
code_challenge_methods_supported: ["S256"],
|
||||||
|
id_token_signing_alg_values_supported: ["RS256"],
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (request.method === "GET" && path === `${ISSUER_PATH}/jwks`) {
|
||||||
|
sendJson(reply, 200, jwks);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (request.method === "GET" && path === `${ISSUER_PATH}/authorize`) {
|
||||||
|
const redirectUri = url.searchParams.get("redirect_uri");
|
||||||
|
const clientId = url.searchParams.get("client_id");
|
||||||
|
const state = url.searchParams.get("state");
|
||||||
|
const nonce = url.searchParams.get("nonce");
|
||||||
|
const challenge = url.searchParams.get("code_challenge");
|
||||||
|
if (url.searchParams.get("response_type") !== "code" || !redirectUri || !clientId || !state || !nonce
|
||||||
|
|| !challenge || url.searchParams.get("code_challenge_method") !== "S256") {
|
||||||
|
sendJson(reply, 400, { error: "invalid_request" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let callback;
|
||||||
|
try {
|
||||||
|
callback = new URL(redirectUri);
|
||||||
|
if (callback.protocol !== "http:" || callback.hostname !== LOOPBACK_HOST || callback.username || callback.password) {
|
||||||
|
throw safeError("oidc_fixture_redirect_invalid");
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
sendJson(reply, 400, { error: "invalid_request" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const code = randomBytes(32).toString("base64url");
|
||||||
|
authorizations.set(code, {
|
||||||
|
challenge,
|
||||||
|
clientId,
|
||||||
|
identity: activeIdentity,
|
||||||
|
nonce,
|
||||||
|
used: false,
|
||||||
|
});
|
||||||
|
lastAuthorization = { identity: activeIdentity, codeChallengeMethod: "S256", pkceVerified: false };
|
||||||
|
callback.searchParams.set("code", code);
|
||||||
|
callback.searchParams.set("state", state);
|
||||||
|
redirect(reply, callback.href);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (request.method === "POST" && path === `${ISSUER_PATH}/device_authorization`) {
|
||||||
|
const values = new URLSearchParams(await requestBody(request));
|
||||||
|
const clientId = values.get("client_id");
|
||||||
|
if (!clientId) {
|
||||||
|
sendJson(reply, 400, { error: "invalid_request" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const deviceCode = randomBytes(32).toString("base64url");
|
||||||
|
const userCode = "FIXTURE-CODE";
|
||||||
|
deviceCodes.set(deviceCode, { clientId, identity: activeIdentity, nonce: "device", used: false });
|
||||||
|
sendJson(reply, 200, {
|
||||||
|
device_code: deviceCode,
|
||||||
|
user_code: userCode,
|
||||||
|
verification_uri: `${issuer}device`,
|
||||||
|
verification_uri_complete: `${issuer}device?user_code=${userCode}`,
|
||||||
|
expires_in: 60,
|
||||||
|
interval: 1,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (request.method === "POST" && path === `${ISSUER_PATH}/token`) {
|
||||||
|
const values = new URLSearchParams(await requestBody(request));
|
||||||
|
const grantType = values.get("grant_type");
|
||||||
|
let record;
|
||||||
|
if (grantType === "authorization_code") {
|
||||||
|
const code = values.get("code") ?? "";
|
||||||
|
record = authorizations.get(code);
|
||||||
|
const verifier = values.get("code_verifier") ?? "";
|
||||||
|
const verified = record !== undefined && !record.used
|
||||||
|
&& createHash("sha256").update(verifier).digest("base64url") === record.challenge;
|
||||||
|
if (!verified) {
|
||||||
|
sendJson(reply, 400, { error: "invalid_grant" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
record.used = true;
|
||||||
|
if (lastAuthorization) lastAuthorization = { ...lastAuthorization, pkceVerified: true };
|
||||||
|
} else if (grantType === "urn:ietf:params:oauth:grant-type:device_code") {
|
||||||
|
const deviceCode = values.get("device_code") ?? "";
|
||||||
|
record = deviceCodes.get(deviceCode);
|
||||||
|
if (record === undefined || record.used) {
|
||||||
|
sendJson(reply, 400, { error: "invalid_grant" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
record.used = true;
|
||||||
|
} else {
|
||||||
|
sendJson(reply, 400, { error: "unsupported_grant_type" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
sendJson(reply, 200, {
|
||||||
|
access_token: randomBytes(32).toString("base64url"),
|
||||||
|
token_type: "Bearer",
|
||||||
|
expires_in: 60,
|
||||||
|
id_token: jwt(privateKey, issuer, record.clientId, record.nonce, record.identity),
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (request.method === "GET" && path === "/api/v3/core/groups/") {
|
||||||
|
if (!request.headers.authorization?.startsWith("Bearer ")) {
|
||||||
|
sendJson(reply, 401, { detail: "authentication required" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const name = url.searchParams.get("name") ?? "";
|
||||||
|
const present = name === "fixture-users" || name === "fixture-admin";
|
||||||
|
sendJson(reply, 200, {
|
||||||
|
pagination: { next: null },
|
||||||
|
results: present ? [{ name }] : [],
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
sendJson(reply, 404, { error: "not_found" });
|
||||||
|
} catch {
|
||||||
|
if (!reply.headersSent) sendJson(reply, 400, { error: "invalid_request" });
|
||||||
|
else reply.end();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
await new Promise((resolveListen, rejectListen) => {
|
||||||
|
const onError = (error) => rejectListen(error);
|
||||||
|
server.once("error", onError);
|
||||||
|
server.listen({ host, port: options.port ?? 0 }, () => {
|
||||||
|
server.off("error", onError);
|
||||||
|
resolveListen();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
server.close();
|
||||||
|
if (ownsDirectory) rmSync(directory, { recursive: true, force: true });
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
const address = server.address();
|
||||||
|
if (!address || typeof address === "string") {
|
||||||
|
await new Promise((resolveClose) => server.close(resolveClose));
|
||||||
|
if (ownsDirectory) rmSync(directory, { recursive: true, force: true });
|
||||||
|
throw safeError("oidc_fixture_listen_failed");
|
||||||
|
}
|
||||||
|
baseUrl = `https://${LOOPBACK_HOST}:${address.port}`;
|
||||||
|
issuer = `${baseUrl}${ISSUER_PATH}/`;
|
||||||
|
|
||||||
|
return {
|
||||||
|
baseUrl,
|
||||||
|
issuer,
|
||||||
|
caFile: certificate.certificateFile,
|
||||||
|
setIdentity(identity) {
|
||||||
|
activeIdentity = authorizationIdentity(identity);
|
||||||
|
},
|
||||||
|
lastAuthorization() {
|
||||||
|
return lastAuthorization === undefined ? undefined : { ...lastAuthorization };
|
||||||
|
},
|
||||||
|
async close() {
|
||||||
|
await new Promise((resolveClose) => server.close(resolveClose));
|
||||||
|
if (ownsDirectory) rmSync(directory, { recursive: true, force: true });
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const currentFile = fileURLToPath(import.meta.url);
|
||||||
|
if (process.argv[1] && resolve(process.argv[1]) === currentFile) {
|
||||||
|
const provider = await startFakeOidcProvider();
|
||||||
|
process.stdout.write('{"status":"ready"}\n');
|
||||||
|
const close = async () => {
|
||||||
|
await provider.close();
|
||||||
|
process.exit(0);
|
||||||
|
};
|
||||||
|
process.once("SIGINT", () => { void close(); });
|
||||||
|
process.once("SIGTERM", () => { void close(); });
|
||||||
|
}
|
||||||
@@ -4,14 +4,18 @@ This is a release-gate checklist, not evidence. Use one ordinary PSD test identi
|
|||||||
PSD test identity supplied through the approved test-identity process. Record only sanitized
|
PSD test identity supplied through the approved test-identity process. Record only sanitized
|
||||||
pass/fail results, timestamps, build identity, and diagnostic codes. Do not record names, internal
|
pass/fail results, timestamps, build identity, and diagnostic codes. Do not record names, internal
|
||||||
URLs, directory/LDAP details, tokens, passwords, hashes, cookies, or realistic secret examples.
|
URLs, directory/LDAP details, tokens, passwords, hashes, cookies, or realistic secret examples.
|
||||||
|
Keep the retained result under `.artifacts/manual-acceptance/authentication/<run-id>/` with a
|
||||||
|
sanitized digest. Do not retain raw browser traces, Compose environments, provider exports, or
|
||||||
|
unbounded logs. If the approved identities or access are unavailable, record **PENDING** rather
|
||||||
|
than inferring a PASS.
|
||||||
|
|
||||||
## Preconditions and ordering
|
## Preconditions and ordering
|
||||||
|
|
||||||
1. Resolve the two parked Task 13 restore preconditions before certification: acquire the lifecycle
|
1. Confirm retained Task 13 evidence for the restore prerequisites before certification: the
|
||||||
lock before any target-dependent preflight and stage/revalidate archive bytes and hashes inside
|
lifecycle lock is acquired before target-dependent preflight, archive bytes and hashes are
|
||||||
that lock immediately before extraction; replace convention-only
|
staged/revalidated inside that lock immediately before extraction, and checkpointing requires
|
||||||
`createWithDependenciesLockHeld` with an opaque installation-bound transaction capability or
|
an opaque installation-bound transaction capability. Manual acceptance never substitutes for
|
||||||
closure so lock-held primitives cannot be called without the capability.
|
those automated concurrency and mutation tests.
|
||||||
2. Run Workspace Validate first; it is the static authentication gate. Run `tht auth check` for
|
2. Run Workspace Validate first; it is the static authentication gate. Run `tht auth check` for
|
||||||
live non-interactive diagnosis, then `tht auth check --interactive` where Device Authorization
|
live non-interactive diagnosis, then `tht auth check --interactive` where Device Authorization
|
||||||
is available, then Workspace Test for aggregate live validation.
|
is available, then Workspace Test for aggregate live validation.
|
||||||
@@ -42,9 +46,13 @@ URLs, directory/LDAP details, tokens, passwords, hashes, cookies, or realistic s
|
|||||||
| Provider outage | Live check reports `oidc_discovery_unreachable`; browser login fails closed without exposing credentials. |
|
| Provider outage | Live check reports `oidc_discovery_unreachable`; browser login fails closed without exposing credentials. |
|
||||||
| Restore is completed | Sessions and OIDC state are absent; all users must reauthenticate. |
|
| Restore is completed | Sessions and OIDC state are absent; all users must reauthenticate. |
|
||||||
|
|
||||||
## Status at Task 14
|
## Status at Task 15
|
||||||
|
|
||||||
Documentation and deterministic contract checks are the Task 14 scope. Browser OIDC callback E2E,
|
The hermetic browser suite now covers the loopback provider discovery/JWKS/device/group-list
|
||||||
native Windows behavioral execution, the two parked restore-lock preconditions above, PSD/manual
|
surface and the complete OIDC Authorization Code + PKCE callback, including direct `groups`
|
||||||
identities, and any external L2 execution remain pending Task 15/release gates. Do not mark this
|
fail-closed cases. It also covers local ordinary, remembered/restart, logout, and administrator
|
||||||
matrix PASS until those gates have actual retained evidence.
|
flows. This deterministic evidence does not replace the manual PSD/AuthentiK acceptance.
|
||||||
|
|
||||||
|
Native Windows behavioral execution, approved PSD/AuthentiK identities and access, interactive
|
||||||
|
device acceptance, and external L2 remain **PENDING** until actual retained evidence exists. Do
|
||||||
|
not mark the feature or this matrix release-complete while any required gate remains pending.
|
||||||
|
|||||||
@@ -0,0 +1,155 @@
|
|||||||
|
import { expect, test, type Page } from "@playwright/test";
|
||||||
|
import { createAuthenticationStack } from "./fixtures/auth-stack.mjs";
|
||||||
|
|
||||||
|
test.describe.configure({ mode: "serial" });
|
||||||
|
// The only HTTPS navigation in this file is the test-scoped loopback provider.
|
||||||
|
test.use({ ignoreHTTPSErrors: true });
|
||||||
|
|
||||||
|
let stack: Awaited<ReturnType<typeof createAuthenticationStack>>;
|
||||||
|
|
||||||
|
test.beforeAll(async () => {
|
||||||
|
stack = await createAuthenticationStack();
|
||||||
|
});
|
||||||
|
|
||||||
|
test.afterAll(async () => {
|
||||||
|
await stack?.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("the loopback fixture exposes signed OIDC discovery, device authorization, and AuthentiK group lookup", async () => {
|
||||||
|
await expect(stack.providerSurface()).resolves.toEqual({
|
||||||
|
discovery: true,
|
||||||
|
jwks: true,
|
||||||
|
deviceAuthorization: true,
|
||||||
|
deviceToken: true,
|
||||||
|
groupList: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
async function expectShell(page: Page): Promise<void> {
|
||||||
|
await expect(page.getByTestId("app-shell")).toBeVisible({ timeout: 30_000 });
|
||||||
|
}
|
||||||
|
|
||||||
|
async function signInLocally(page: Page, account: "ordinary" | "admin", remember = false): Promise<void> {
|
||||||
|
await page.getByLabel("Username").fill(stack.localAccount(account).username);
|
||||||
|
await page.getByLabel("Password").fill(stack.localAccount(account).password);
|
||||||
|
const rememberControl = page.getByRole("checkbox", { name: /remember me/i });
|
||||||
|
if (remember) await rememberControl.check();
|
||||||
|
await page.getByRole("button", { name: "Sign in", exact: true }).click();
|
||||||
|
await expectShell(page);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function browserSession(page: Page): Promise<{ status: number; body: Record<string, unknown> }> {
|
||||||
|
return page.evaluate(async () => {
|
||||||
|
const response = await fetch("/api/me", { credentials: "same-origin" });
|
||||||
|
return { status: response.status, body: await response.json() as Record<string, unknown> };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function expectNoWebStorageTokens(page: Page): Promise<void> {
|
||||||
|
const entries = await page.evaluate(() => {
|
||||||
|
const values = (storage: Storage) => Array.from({ length: storage.length }, (_unused, index) => {
|
||||||
|
const key = storage.key(index) ?? "";
|
||||||
|
return [key, storage.getItem(key) ?? ""];
|
||||||
|
});
|
||||||
|
return [...values(localStorage), ...values(sessionStorage)];
|
||||||
|
});
|
||||||
|
expect(entries.filter(([key, value]) => /(?:access|refresh|id)?[_-]?token|bearer|jwt/i.test(`${key}\n${value}`))).toEqual([]);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function signInWithOidc(page: Page): Promise<void> {
|
||||||
|
await page.getByRole("button", { name: /continue with single sign-on/i }).click();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function expectOidcCallbackDenied(page: Page): Promise<void> {
|
||||||
|
await expect(page.locator("body")).toContainText("OIDC sign-in could not be completed", { timeout: 30_000 });
|
||||||
|
await expect(page.getByTestId("app-shell")).toHaveCount(0);
|
||||||
|
await expectNoWebStorageTokens(page);
|
||||||
|
}
|
||||||
|
|
||||||
|
test("local ordinary and remembered sessions survive restart, logout, and keep tokens out of Web Storage", async ({ page }) => {
|
||||||
|
await stack.useLocalMode();
|
||||||
|
await page.goto(stack.publicUrl);
|
||||||
|
await expect(page.getByRole("heading", { name: "Sign in to ThothII" })).toBeVisible();
|
||||||
|
|
||||||
|
await signInLocally(page, "ordinary");
|
||||||
|
expect((await browserSession(page)).body.roles).toEqual(["user"]);
|
||||||
|
await expectNoWebStorageTokens(page);
|
||||||
|
|
||||||
|
await stack.restartBackend();
|
||||||
|
await page.reload();
|
||||||
|
await expectShell(page);
|
||||||
|
|
||||||
|
await page.getByRole("button", { name: "Log out", exact: true }).click();
|
||||||
|
await expect(page.getByRole("heading", { name: "Sign in to ThothII" })).toBeVisible();
|
||||||
|
|
||||||
|
await signInLocally(page, "ordinary", true);
|
||||||
|
const remembered = (await page.context().cookies(stack.publicUrl)).find((cookie) => cookie.name === "thothii_session");
|
||||||
|
expect(remembered?.httpOnly).toBe(true);
|
||||||
|
expect(remembered?.expires ?? -1).toBeGreaterThan(Date.now() / 1_000);
|
||||||
|
|
||||||
|
await stack.restartBackend();
|
||||||
|
await page.reload();
|
||||||
|
await expectShell(page);
|
||||||
|
await expectNoWebStorageTokens(page);
|
||||||
|
|
||||||
|
await page.getByRole("button", { name: "Log out", exact: true }).click();
|
||||||
|
await expect(page.getByRole("heading", { name: "Sign in to ThothII" })).toBeVisible();
|
||||||
|
expect((await page.context().cookies(stack.publicUrl)).some((cookie) => cookie.name === "thothii_session")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("local administrator receives the administrator role", async ({ page }) => {
|
||||||
|
await stack.useLocalMode();
|
||||||
|
await page.goto(stack.publicUrl);
|
||||||
|
await signInLocally(page, "admin");
|
||||||
|
expect((await browserSession(page)).body.roles).toEqual(["admin"]);
|
||||||
|
await expectNoWebStorageTokens(page);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("OIDC Authorization Code plus PKCE redirects back and maps ordinary and administrator groups", async ({ page }) => {
|
||||||
|
await stack.useOidcMode("ordinary");
|
||||||
|
await page.goto(stack.publicUrl);
|
||||||
|
await signInWithOidc(page);
|
||||||
|
await expectShell(page);
|
||||||
|
expect((await browserSession(page)).body.roles).toEqual(["user"]);
|
||||||
|
expect(stack.lastAuthorization()).toMatchObject({ codeChallengeMethod: "S256", pkceVerified: true });
|
||||||
|
await expectNoWebStorageTokens(page);
|
||||||
|
|
||||||
|
await page.getByRole("button", { name: "Log out", exact: true }).click();
|
||||||
|
await expect(page.getByRole("heading", { name: "Sign in to ThothII" })).toBeVisible();
|
||||||
|
stack.setOidcIdentity("admin");
|
||||||
|
await signInWithOidc(page);
|
||||||
|
await expectShell(page);
|
||||||
|
expect((await browserSession(page)).body.roles).toEqual(["admin"]);
|
||||||
|
await expectNoWebStorageTokens(page);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("OIDC unmapped, missing, and malformed groups fail closed; an expired token can recover", async ({ page }) => {
|
||||||
|
await stack.useOidcMode("unmapped");
|
||||||
|
await page.goto(stack.publicUrl);
|
||||||
|
await signInWithOidc(page);
|
||||||
|
await expect(page.getByRole("heading", { name: "Access not permitted" })).toBeVisible({ timeout: 30_000 });
|
||||||
|
await expectNoWebStorageTokens(page);
|
||||||
|
|
||||||
|
await page.context().clearCookies();
|
||||||
|
stack.setOidcIdentity("missing-groups");
|
||||||
|
await page.goto(stack.publicUrl);
|
||||||
|
await signInWithOidc(page);
|
||||||
|
await expectOidcCallbackDenied(page);
|
||||||
|
|
||||||
|
stack.setOidcIdentity("malformed-groups");
|
||||||
|
await page.goto(stack.publicUrl);
|
||||||
|
await signInWithOidc(page);
|
||||||
|
await expectOidcCallbackDenied(page);
|
||||||
|
|
||||||
|
stack.setOidcIdentity("expired");
|
||||||
|
await page.goto(stack.publicUrl);
|
||||||
|
await signInWithOidc(page);
|
||||||
|
await expectOidcCallbackDenied(page);
|
||||||
|
|
||||||
|
stack.setOidcIdentity("ordinary");
|
||||||
|
await page.goto(stack.publicUrl);
|
||||||
|
await signInWithOidc(page);
|
||||||
|
await expectShell(page);
|
||||||
|
expect((await browserSession(page)).body.roles).toEqual(["user"]);
|
||||||
|
await expectNoWebStorageTokens(page);
|
||||||
|
});
|
||||||
+32
-9
@@ -1,4 +1,5 @@
|
|||||||
import { test, expect } from "@playwright/test";
|
import { expect, test, type Page } from "@playwright/test";
|
||||||
|
import { createAuthenticationStack } from "./fixtures/auth-stack.mjs";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* E2E F1 loop — hermetic (no VPN, no real Pi, no real Python).
|
* E2E F1 loop — hermetic (no VPN, no real Pi, no real Python).
|
||||||
@@ -9,20 +10,42 @@ import { test, expect } from "@playwright/test";
|
|||||||
* which emits the f1_disambiguation.json scenario (select widget with
|
* which emits the f1_disambiguation.json scenario (select widget with
|
||||||
* "interpretazione A" / "interpretazione B").
|
* "interpretazione A" / "interpretazione B").
|
||||||
*
|
*
|
||||||
* Flow: open app → open NewSessionDialog → fill question → submit (Crea)
|
* Flow: authenticate locally → focus the new-session composer → fill question → submit (Send)
|
||||||
* → wait for F1 select widget → click an option → assert no error shown.
|
* → wait for F1 select widget → click an option → assert no error shown.
|
||||||
*/
|
*/
|
||||||
test("F1 loop: new question → F1 widget → respond", async ({ page }) => {
|
test.describe.configure({ mode: "serial" });
|
||||||
await page.goto("/");
|
|
||||||
|
|
||||||
// Open the new-session dialog.
|
let stack: Awaited<ReturnType<typeof createAuthenticationStack>>;
|
||||||
await page.getByRole("button", { name: /nuova/i }).click();
|
|
||||||
|
test.beforeAll(async () => {
|
||||||
|
stack = await createAuthenticationStack({ withF1Workspace: true });
|
||||||
|
await stack.useLocalMode();
|
||||||
|
});
|
||||||
|
|
||||||
|
test.afterAll(async () => {
|
||||||
|
await stack?.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
async function signInLocally(page: Page): Promise<void> {
|
||||||
|
const account = stack.localAccount("ordinary");
|
||||||
|
await page.getByLabel("Username").fill(account.username);
|
||||||
|
await page.getByLabel("Password").fill(account.password);
|
||||||
|
await page.getByRole("button", { name: "Sign in", exact: true }).click();
|
||||||
|
await expect(page.getByTestId("app-shell")).toBeVisible({ timeout: 30_000 });
|
||||||
|
}
|
||||||
|
|
||||||
|
test("F1 loop: new question → F1 widget → respond", async ({ page }) => {
|
||||||
|
await page.goto(stack.publicUrl);
|
||||||
|
await signInLocally(page);
|
||||||
|
|
||||||
|
// Focus the composer for a new session.
|
||||||
|
await page.getByRole("button", { name: "New session", exact: true }).click();
|
||||||
|
|
||||||
// Fill in the question.
|
// Fill in the question.
|
||||||
await page.getByLabel(/domanda/i).fill("quante cardioversioni nel 2024");
|
await page.getByLabel("New question").fill("quante cardioversioni nel 2024");
|
||||||
|
|
||||||
// Submit — the backend creates the session (fake-tht) and spawns Pi (fake-pi).
|
// Submit — the backend creates the session (fake-tht) and spawns Pi (fake-pi).
|
||||||
await page.getByRole("button", { name: /^crea$/i }).click();
|
await page.getByRole("button", { name: "Send", exact: true }).click();
|
||||||
|
|
||||||
// Wait for the F1 disambiguation select widget to appear.
|
// Wait for the F1 disambiguation select widget to appear.
|
||||||
// The fake-pi emits extension_ui_request → backend bridges to SSE → frontend
|
// The fake-pi emits extension_ui_request → backend bridges to SSE → frontend
|
||||||
@@ -41,5 +64,5 @@ test("F1 loop: new question → F1 widget → respond", async ({ page }) => {
|
|||||||
await expect(
|
await expect(
|
||||||
page.getByRole("button", { name: /interpretazione A/i }),
|
page.getByRole("button", { name: /interpretazione A/i }),
|
||||||
).not.toBeVisible({ timeout: 10_000 });
|
).not.toBeVisible({ timeout: 10_000 });
|
||||||
await expect(page.locator("body")).not.toContainText(/errore/i);
|
await expect(page.locator("body")).not.toContainText(/error/i);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,521 @@
|
|||||||
|
import { spawn } from "node:child_process";
|
||||||
|
import { argon2 } from "node:crypto";
|
||||||
|
import { chmodSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
|
||||||
|
import { createServer as createHttpServer } from "node:http";
|
||||||
|
import { request as httpsRequest } from "node:https";
|
||||||
|
import { createServer } from "node:net";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { dirname, join, resolve } from "node:path";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
import { startFakeOidcProvider } from "../../../backend/test/fixtures/oidc-provider.mjs";
|
||||||
|
|
||||||
|
const __dir = dirname(fileURLToPath(import.meta.url));
|
||||||
|
const repositoryRoot = resolve(__dir, "../../..");
|
||||||
|
const frontendRoot = join(repositoryRoot, "frontend");
|
||||||
|
const backendRoot = join(repositoryRoot, "backend");
|
||||||
|
const harnessRoot = join(repositoryRoot, "harness");
|
||||||
|
const thtRoot = join(repositoryRoot, "tools", "tht");
|
||||||
|
const fakePi = join(frontendRoot, "e2e", "fixtures", "fake-pi.mjs");
|
||||||
|
const fakeTht = join(frontendRoot, "e2e", "fixtures", "fake-tht.mjs");
|
||||||
|
|
||||||
|
function safeError(code) {
|
||||||
|
return new Error(code);
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildAuthenticationStorageBridge(output) {
|
||||||
|
const result = spawn("go", ["build", "-o", output, "./cmd/tht"], {
|
||||||
|
cwd: thtRoot,
|
||||||
|
stdio: "ignore",
|
||||||
|
});
|
||||||
|
return new Promise((resolveBuild, rejectBuild) => {
|
||||||
|
result.once("error", () => rejectBuild(safeError("e2e_auth_storage_build_failed")));
|
||||||
|
result.once("exit", (code) => code === 0 ? resolveBuild() : rejectBuild(safeError("e2e_auth_storage_build_failed")));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function secureDirectory(path) {
|
||||||
|
mkdirSync(path, { recursive: true, mode: 0o700 });
|
||||||
|
chmodSync(path, 0o700);
|
||||||
|
}
|
||||||
|
|
||||||
|
function runFixtureCommand(command, args, cwd) {
|
||||||
|
const child = spawn(command, args, { cwd, stdio: "ignore" });
|
||||||
|
return new Promise((resolveCommand, rejectCommand) => {
|
||||||
|
child.once("error", () => rejectCommand(safeError("e2e_workspace_fixture_command_failed")));
|
||||||
|
child.once("exit", (code) => code === 0
|
||||||
|
? resolveCommand()
|
||||||
|
: rejectCommand(safeError("e2e_workspace_fixture_command_failed")));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const F1_WORKSPACE_ID = "fixture-workspace";
|
||||||
|
const F1_WORKSPACE_DESCRIPTOR = `workspace:
|
||||||
|
schema_version: 3
|
||||||
|
id: fixture-workspace
|
||||||
|
name: Fixture workspace
|
||||||
|
language: en
|
||||||
|
dwh:
|
||||||
|
engine: postgres
|
||||||
|
database: fixture
|
||||||
|
schema: fixture
|
||||||
|
supported_transports: [postgres_direct]
|
||||||
|
semantic_index:
|
||||||
|
vector_store:
|
||||||
|
engine: qdrant
|
||||||
|
collection: fixture-workspace
|
||||||
|
dimensions: 1024
|
||||||
|
distance: cosine
|
||||||
|
embedding:
|
||||||
|
provider: ollama_internal
|
||||||
|
model: qwen3-embedding:0.6b
|
||||||
|
dimensions: 1024
|
||||||
|
llm_policy:
|
||||||
|
allowed: [zai/glm-5.2]
|
||||||
|
default: zai/glm-5.2
|
||||||
|
`;
|
||||||
|
|
||||||
|
async function prepareF1Workspace(root) {
|
||||||
|
const source = join(root, "workspace-source");
|
||||||
|
const remote = join(root, "workspace-remote.git");
|
||||||
|
secureDirectory(source);
|
||||||
|
secureDirectory(join(source, F1_WORKSPACE_ID));
|
||||||
|
writeSecure(join(source, "thoth-workspaces.yaml"), [
|
||||||
|
"schema_version: 1",
|
||||||
|
"workspaces:",
|
||||||
|
` - id: ${F1_WORKSPACE_ID}`,
|
||||||
|
" name: Fixture workspace",
|
||||||
|
"",
|
||||||
|
].join("\n"));
|
||||||
|
writeSecure(join(source, F1_WORKSPACE_ID, "workspace.yaml"), F1_WORKSPACE_DESCRIPTOR);
|
||||||
|
|
||||||
|
await runFixtureCommand("git", ["init", "--bare", "--initial-branch=main", remote], root);
|
||||||
|
chmodSync(remote, 0o700);
|
||||||
|
await runFixtureCommand("git", ["init", "--initial-branch=main"], source);
|
||||||
|
await runFixtureCommand("git", ["config", "user.name", "ThothII E2E Fixture"], source);
|
||||||
|
await runFixtureCommand("git", ["config", "user.email", "thothii-e2e@example.invalid"], source);
|
||||||
|
await runFixtureCommand("git", ["add", "-A"], source);
|
||||||
|
await runFixtureCommand("git", ["commit", "-m", "Create deterministic fixture workspace"], source);
|
||||||
|
await runFixtureCommand("git", ["remote", "add", "origin", remote], source);
|
||||||
|
await runFixtureCommand("git", ["push", "origin", "main"], source);
|
||||||
|
return { id: F1_WORKSPACE_ID, remote };
|
||||||
|
}
|
||||||
|
|
||||||
|
function writeSecure(path, value) {
|
||||||
|
writeFileSync(path, value, { encoding: "utf8", mode: 0o600 });
|
||||||
|
chmodSync(path, 0o600);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function testPasswordHash(password) {
|
||||||
|
const salt = Buffer.from("thothii-e2e-salt");
|
||||||
|
const message = Buffer.from(password, "utf8");
|
||||||
|
let digest;
|
||||||
|
try {
|
||||||
|
digest = await new Promise((resolveDigest, rejectDigest) => {
|
||||||
|
argon2("argon2id", {
|
||||||
|
message,
|
||||||
|
nonce: salt,
|
||||||
|
memory: 65_536,
|
||||||
|
passes: 3,
|
||||||
|
parallelism: 1,
|
||||||
|
tagLength: 32,
|
||||||
|
}, (error, derived) => error || !derived ? rejectDigest(error ?? safeError("e2e_password_hash_failed")) : resolveDigest(derived));
|
||||||
|
});
|
||||||
|
return `$argon2id$v=19$m=65536,t=3,p=1$${salt.toString("base64").replaceAll("=", "")}$${digest.toString("base64").replaceAll("=", "")}`;
|
||||||
|
} finally {
|
||||||
|
message.fill(0);
|
||||||
|
salt.fill(0);
|
||||||
|
digest?.fill(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function pause(milliseconds) {
|
||||||
|
return new Promise((resolvePause) => setTimeout(resolvePause, milliseconds));
|
||||||
|
}
|
||||||
|
|
||||||
|
function providerJson(url, caFile, options = {}) {
|
||||||
|
return new Promise((resolveResponse, rejectResponse) => {
|
||||||
|
const body = options.body ?? "";
|
||||||
|
const request = httpsRequest(url, {
|
||||||
|
method: options.method ?? "GET",
|
||||||
|
ca: readFileSync(caFile),
|
||||||
|
headers: {
|
||||||
|
accept: "application/json",
|
||||||
|
...(body.length === 0 ? {} : {
|
||||||
|
"content-length": String(Buffer.byteLength(body)),
|
||||||
|
"content-type": "application/x-www-form-urlencoded",
|
||||||
|
}),
|
||||||
|
...options.headers,
|
||||||
|
},
|
||||||
|
}, (response) => {
|
||||||
|
const chunks = [];
|
||||||
|
let size = 0;
|
||||||
|
response.on("data", (chunk) => {
|
||||||
|
size += chunk.length;
|
||||||
|
if (size > 64 * 1024) request.destroy(safeError("e2e_provider_response_too_large"));
|
||||||
|
else chunks.push(chunk);
|
||||||
|
});
|
||||||
|
response.once("error", () => rejectResponse(safeError("e2e_provider_response_failed")));
|
||||||
|
response.once("end", () => {
|
||||||
|
try {
|
||||||
|
resolveResponse({ status: response.statusCode ?? 0, body: JSON.parse(Buffer.concat(chunks).toString("utf8")) });
|
||||||
|
} catch {
|
||||||
|
rejectResponse(safeError("e2e_provider_response_invalid"));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
request.once("error", () => rejectResponse(safeError("e2e_provider_request_failed")));
|
||||||
|
request.end(body);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function freeLoopbackPort() {
|
||||||
|
const server = createServer();
|
||||||
|
await new Promise((resolveListen, rejectListen) => {
|
||||||
|
server.once("error", rejectListen);
|
||||||
|
server.listen({ host: "127.0.0.1", port: 0 }, resolveListen);
|
||||||
|
});
|
||||||
|
const address = server.address();
|
||||||
|
await new Promise((resolveClose) => server.close(resolveClose));
|
||||||
|
if (!address || typeof address === "string") throw safeError("e2e_loopback_port_unavailable");
|
||||||
|
return address.port;
|
||||||
|
}
|
||||||
|
|
||||||
|
const F1_QDRANT_INDEXES = Object.freeze([
|
||||||
|
"content_hash", "document_id", "kind", "record_key",
|
||||||
|
"record_kind", "vector_generation", "workspace_id", "workspace_revision",
|
||||||
|
]);
|
||||||
|
|
||||||
|
async function startFakeQdrant() {
|
||||||
|
const payloadSchema = Object.fromEntries(F1_QDRANT_INDEXES.map((field) => [field, { data_type: "keyword" }]));
|
||||||
|
const server = createHttpServer((request, response) => {
|
||||||
|
const path = new URL(request.url ?? "/", "http://loopback.invalid").pathname;
|
||||||
|
if (request.method !== "GET" || path !== `/collections/${F1_WORKSPACE_ID}`) {
|
||||||
|
response.writeHead(404).end();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
response.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({
|
||||||
|
result: {
|
||||||
|
config: { params: { vectors: { size: 1024, distance: "Cosine" } } },
|
||||||
|
payload_schema: payloadSchema,
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
});
|
||||||
|
await new Promise((resolveListen, rejectListen) => {
|
||||||
|
server.once("error", rejectListen);
|
||||||
|
server.listen({ host: "127.0.0.1", port: 0 }, resolveListen);
|
||||||
|
});
|
||||||
|
const address = server.address();
|
||||||
|
if (!address || typeof address === "string") {
|
||||||
|
await new Promise((resolveClose) => server.close(resolveClose));
|
||||||
|
throw safeError("e2e_qdrant_loopback_port_unavailable");
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
baseUrl: `http://127.0.0.1:${address.port}/`,
|
||||||
|
close: () => new Promise((resolveClose) => server.close(resolveClose)),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function managedProcess(command, args, options) {
|
||||||
|
const child = spawn(command, args, {
|
||||||
|
cwd: options.cwd,
|
||||||
|
env: options.env,
|
||||||
|
stdio: ["ignore", "pipe", "pipe"],
|
||||||
|
});
|
||||||
|
let exited = false;
|
||||||
|
let diagnostic = "";
|
||||||
|
const captureDiagnostic = (chunk) => {
|
||||||
|
const sanitized = String(chunk)
|
||||||
|
.replace(/https?:\/\/[^\s)]+/g, "[url]")
|
||||||
|
.replace(/(?:THT_[A-Z_]+|PI_[A-Z_]+|AUTH_MODE)=\S+/g, "$1=[redacted]")
|
||||||
|
.replace(/[A-Za-z0-9_-]{43,}/g, "[redacted]");
|
||||||
|
diagnostic = `${diagnostic}${sanitized}`.slice(-800);
|
||||||
|
};
|
||||||
|
child.once("exit", () => { exited = true; });
|
||||||
|
child.stdout?.on("data", captureDiagnostic);
|
||||||
|
child.stderr?.on("data", captureDiagnostic);
|
||||||
|
return {
|
||||||
|
child,
|
||||||
|
exited: () => exited,
|
||||||
|
diagnostic: () => diagnostic.replace(/\s+/g, " ").trim(),
|
||||||
|
async close() {
|
||||||
|
if (exited) return;
|
||||||
|
child.kill("SIGTERM");
|
||||||
|
for (let attempt = 0; attempt < 50 && !exited; attempt += 1) await pause(20);
|
||||||
|
if (!exited) child.kill("SIGKILL");
|
||||||
|
for (let attempt = 0; attempt < 50 && !exited; attempt += 1) await pause(20);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForOk(url, processHandle) {
|
||||||
|
for (let attempt = 0; attempt < 300; attempt += 1) {
|
||||||
|
if (processHandle.exited()) {
|
||||||
|
const detail = processHandle.diagnostic();
|
||||||
|
throw safeError(detail ? `e2e_service_stopped_during_startup:${detail}` : "e2e_service_stopped_during_startup");
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const response = await fetch(url, { redirect: "error" });
|
||||||
|
if (response.ok) return;
|
||||||
|
} catch {
|
||||||
|
// The process is expected to race its listener setup.
|
||||||
|
}
|
||||||
|
await pause(100);
|
||||||
|
}
|
||||||
|
throw safeError("e2e_service_startup_timeout");
|
||||||
|
}
|
||||||
|
|
||||||
|
function cleanBackendEnvironment(overrides) {
|
||||||
|
const env = { ...process.env };
|
||||||
|
for (const name of [
|
||||||
|
"AUTH_MODE", "THT_AUTH_CONFIG_FILE", "THT_AUTH_STATE_ROOT", "THT_SECRETS_FILE", "NODE_EXTRA_CA_CERTS",
|
||||||
|
"SETTINGS_FILE", "THT_MAINTENANCE_FILE", "THT_WORKSPACE_REGISTRY_ROOT", "THT_WORKSPACE_SECRET_STORE_ROOT",
|
||||||
|
"THT_WORKSPACE_SECRET_RUNTIME_ROOT", "THT_WORKSPACE_GIT_REMOTE", "THT_WORKSPACE_GIT_BRANCH",
|
||||||
|
"THT_WORKSPACE_SECRET_ROOTS", "THT_WORKSPACE_INSTALLATION_ID", "THT_LEGACY_WORKSPACE_MODE",
|
||||||
|
"THT_DWH_PRECHECK", "THT_INTERNAL_QDRANT_URL", "THT_CONFIG",
|
||||||
|
]) delete env[name];
|
||||||
|
for (const name of Object.keys(env)) {
|
||||||
|
if (name.startsWith("THT_WS_")) delete env[name];
|
||||||
|
}
|
||||||
|
return { ...env, ...overrides };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function createAuthenticationStack({ withF1Workspace = false } = {}) {
|
||||||
|
const root = mkdtempSync(join(realpathSync(tmpdir()), "thothii-auth-e2e-"));
|
||||||
|
secureDirectory(root);
|
||||||
|
const stateRoot = join(root, "auth-state");
|
||||||
|
const registryRoot = join(root, "workspace-registry");
|
||||||
|
const workspaceSecretRoot = join(root, "workspace-secrets");
|
||||||
|
const workspaceRuntimeRoot = join(root, "workspace-runtime");
|
||||||
|
const fixtureSecretRoot = join(root, "fixture-runtime-secrets");
|
||||||
|
const providerRoot = join(root, "provider");
|
||||||
|
const authConfigFile = join(root, "auth.yaml");
|
||||||
|
const usersFile = join(root, "users.yaml");
|
||||||
|
const secretsFile = join(root, "test.secrets");
|
||||||
|
const settingsFile = join(root, "settings.json");
|
||||||
|
const maintenanceFile = join(root, "maintenance.json");
|
||||||
|
const authStorageBinary = join(root, "tht-auth-storage");
|
||||||
|
const fixtureDwhPasswordFile = join(fixtureSecretRoot, "fixture-dwh-password");
|
||||||
|
const fixtureDwhCaFile = join(fixtureSecretRoot, "fixture-dwh-ca.pem");
|
||||||
|
for (const path of [stateRoot, registryRoot, workspaceSecretRoot, workspaceRuntimeRoot, fixtureSecretRoot, providerRoot]) secureDirectory(path);
|
||||||
|
for (const child of ["sessions", "oidc"]) secureDirectory(join(stateRoot, child));
|
||||||
|
|
||||||
|
const [frontendPort, backendPort] = await Promise.all([freeLoopbackPort(), freeLoopbackPort()]);
|
||||||
|
const publicUrl = `http://127.0.0.1:${frontendPort}`;
|
||||||
|
const backendUrl = `http://127.0.0.1:${backendPort}`;
|
||||||
|
const workspace = withF1Workspace ? await prepareF1Workspace(root) : undefined;
|
||||||
|
const provider = await startFakeOidcProvider({ directory: providerRoot });
|
||||||
|
await buildAuthenticationStorageBridge(authStorageBinary);
|
||||||
|
const localPassword = "e2e-local-password";
|
||||||
|
const passwordHash = await testPasswordHash(localPassword);
|
||||||
|
const accounts = Object.freeze({
|
||||||
|
ordinary: Object.freeze({ username: "ordinary", password: localPassword }),
|
||||||
|
admin: Object.freeze({ username: "administrator", password: localPassword }),
|
||||||
|
});
|
||||||
|
writeSecure(usersFile, JSON.stringify({
|
||||||
|
version: 1,
|
||||||
|
users: [
|
||||||
|
{
|
||||||
|
id: "11111111-1111-4111-8111-111111111111",
|
||||||
|
username: accounts.ordinary.username,
|
||||||
|
displayName: "Fixture ordinary",
|
||||||
|
passwordHash,
|
||||||
|
roles: ["user"],
|
||||||
|
enabled: true,
|
||||||
|
authRevision: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "22222222-2222-4222-8222-222222222222",
|
||||||
|
username: accounts.admin.username,
|
||||||
|
displayName: "Fixture administrator",
|
||||||
|
passwordHash,
|
||||||
|
roles: ["admin"],
|
||||||
|
enabled: true,
|
||||||
|
authRevision: 1,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}));
|
||||||
|
writeSecure(secretsFile, [
|
||||||
|
"THT_OIDC_CLIENT_SECRET=e2e-client-secret-not-a-production-secret",
|
||||||
|
"THT_AUTHENTIK_API_TOKEN=e2e-group-catalog-token-not-a-production-secret",
|
||||||
|
"",
|
||||||
|
].join("\n"));
|
||||||
|
if (workspace) {
|
||||||
|
writeSecure(fixtureDwhPasswordFile, "fixture-password-not-a-secret\n");
|
||||||
|
writeSecure(fixtureDwhCaFile, "fixture-ca-not-a-production-certificate\n");
|
||||||
|
writeSecure(settingsFile, JSON.stringify({
|
||||||
|
workspace: workspace.id,
|
||||||
|
provider: "zai",
|
||||||
|
model: "glm-5.2",
|
||||||
|
thinking: "medium",
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mode = undefined;
|
||||||
|
let backend = undefined;
|
||||||
|
let qdrant = undefined;
|
||||||
|
const frontend = managedProcess(join(frontendRoot, "node_modules", ".bin", "vite"), [
|
||||||
|
"--host", "127.0.0.1", "--port", String(frontendPort), "--strictPort",
|
||||||
|
], {
|
||||||
|
cwd: frontendRoot,
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
THT_FRONTEND_API_UPSTREAM: backendUrl,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
await waitForOk(publicUrl, frontend);
|
||||||
|
qdrant = workspace ? await startFakeQdrant() : undefined;
|
||||||
|
|
||||||
|
const localConfig = () => ({
|
||||||
|
version: 1,
|
||||||
|
mode: "local",
|
||||||
|
publicUrl,
|
||||||
|
session: {
|
||||||
|
regularTtlSeconds: 600,
|
||||||
|
regularIdleSeconds: 600,
|
||||||
|
rememberTtlSeconds: 2_592_000,
|
||||||
|
rememberIdleSeconds: 604_800,
|
||||||
|
},
|
||||||
|
local: { usersFile: "users.yaml" },
|
||||||
|
});
|
||||||
|
const oidcConfig = () => ({
|
||||||
|
version: 1,
|
||||||
|
mode: "oidc",
|
||||||
|
publicUrl,
|
||||||
|
session: {
|
||||||
|
regularTtlSeconds: 600,
|
||||||
|
regularIdleSeconds: 600,
|
||||||
|
oidcTtlSeconds: 60,
|
||||||
|
},
|
||||||
|
oidc: {
|
||||||
|
issuer: provider.issuer,
|
||||||
|
clientId: "thothii-e2e-client",
|
||||||
|
clientSecretRef: "THT_OIDC_CLIENT_SECRET",
|
||||||
|
scopes: ["openid", "profile", "groups"],
|
||||||
|
groupsClaim: "groups",
|
||||||
|
},
|
||||||
|
groupCatalog: {
|
||||||
|
driver: "authentik",
|
||||||
|
baseUrl: provider.baseUrl,
|
||||||
|
apiTokenRef: "THT_AUTHENTIK_API_TOKEN",
|
||||||
|
},
|
||||||
|
authorization: {
|
||||||
|
groupRoles: {
|
||||||
|
"fixture-users": ["user"],
|
||||||
|
"fixture-admin": ["admin"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
async function startBackend() {
|
||||||
|
if (mode === undefined) throw safeError("e2e_auth_mode_not_configured");
|
||||||
|
backend = managedProcess(join(backendRoot, "node_modules", ".bin", "tsx"), ["src/server.ts"], {
|
||||||
|
cwd: backendRoot,
|
||||||
|
env: cleanBackendEnvironment({
|
||||||
|
NODE_ENV: "test",
|
||||||
|
HOST: "127.0.0.1",
|
||||||
|
PORT: String(backendPort),
|
||||||
|
PI_BIN: fakePi,
|
||||||
|
THT_BIN: fakeTht,
|
||||||
|
THT_AUTH_STORAGE_BIN: authStorageBinary,
|
||||||
|
THT_HARNESS_DIR: harnessRoot,
|
||||||
|
THT_AUTH_CONFIG_FILE: authConfigFile,
|
||||||
|
THT_AUTH_STATE_ROOT: stateRoot,
|
||||||
|
THT_SECRETS_FILE: secretsFile,
|
||||||
|
NODE_EXTRA_CA_CERTS: provider.caFile,
|
||||||
|
SETTINGS_FILE: settingsFile,
|
||||||
|
THT_MAINTENANCE_FILE: maintenanceFile,
|
||||||
|
THT_WORKSPACE_REGISTRY_ROOT: registryRoot,
|
||||||
|
THT_WORKSPACE_SECRET_STORE_ROOT: workspaceSecretRoot,
|
||||||
|
THT_WORKSPACE_SECRET_RUNTIME_ROOT: workspaceRuntimeRoot,
|
||||||
|
THT_WORKSPACE_INSTALLATION_ID: "e2e",
|
||||||
|
THT_DATA_ROOT: join(root, "data"),
|
||||||
|
...(qdrant ? { THT_INTERNAL_QDRANT_URL: qdrant.baseUrl } : {}),
|
||||||
|
...(workspace ? {
|
||||||
|
THT_WORKSPACE_GIT_REMOTE: workspace.remote,
|
||||||
|
THT_WORKSPACE_GIT_BRANCH: "main",
|
||||||
|
THT_WORKSPACE_SECRET_ROOTS: fixtureSecretRoot,
|
||||||
|
THT_WS_FIXTURE_WORKSPACE_DWH_TRANSPORT: "postgres_direct",
|
||||||
|
THT_WS_FIXTURE_WORKSPACE_DWH_HOST: "127.0.0.1",
|
||||||
|
THT_WS_FIXTURE_WORKSPACE_DWH_PORT: "5432",
|
||||||
|
THT_WS_FIXTURE_WORKSPACE_DWH_USER: "fixture",
|
||||||
|
THT_WS_FIXTURE_WORKSPACE_DWH_PASSWORD_FILE: fixtureDwhPasswordFile,
|
||||||
|
THT_WS_FIXTURE_WORKSPACE_DWH_TLS_CA_FILE: fixtureDwhCaFile,
|
||||||
|
} : {}),
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
await waitForOk(`${backendUrl}/health`, backend);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function restartBackend() {
|
||||||
|
await backend?.close();
|
||||||
|
backend = undefined;
|
||||||
|
await startBackend();
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
publicUrl,
|
||||||
|
localAccount(account) {
|
||||||
|
const found = accounts[account];
|
||||||
|
if (!found) throw safeError("e2e_local_account_unknown");
|
||||||
|
return found;
|
||||||
|
},
|
||||||
|
lastAuthorization() {
|
||||||
|
return provider.lastAuthorization();
|
||||||
|
},
|
||||||
|
async providerSurface() {
|
||||||
|
const discovery = await providerJson(`${provider.issuer}.well-known/openid-configuration`, provider.caFile);
|
||||||
|
const jwks = await providerJson(`${provider.issuer}jwks`, provider.caFile);
|
||||||
|
const device = await providerJson(`${provider.issuer}device_authorization`, provider.caFile, {
|
||||||
|
method: "POST",
|
||||||
|
body: "client_id=thothii-e2e-client",
|
||||||
|
});
|
||||||
|
const deviceCode = typeof device.body?.device_code === "string" ? device.body.device_code : "";
|
||||||
|
const deviceToken = deviceCode.length === 0 ? { status: 0, body: {} } : await providerJson(`${provider.issuer}token`, provider.caFile, {
|
||||||
|
method: "POST",
|
||||||
|
body: `grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Adevice_code&device_code=${encodeURIComponent(deviceCode)}`,
|
||||||
|
});
|
||||||
|
const groups = await providerJson(`${provider.baseUrl}/api/v3/core/groups/?name=fixture-users`, provider.caFile, {
|
||||||
|
headers: { authorization: "Bearer e2e-fixture" },
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
discovery: discovery.status === 200 && discovery.body?.issuer === provider.issuer,
|
||||||
|
jwks: jwks.status === 200 && jwks.body?.keys?.[0]?.alg === "RS256" && jwks.body?.keys?.[0]?.use === "sig",
|
||||||
|
deviceAuthorization: device.status === 200 && typeof device.body?.device_code === "string"
|
||||||
|
&& typeof device.body?.verification_uri === "string",
|
||||||
|
deviceToken: deviceToken.status === 200 && typeof deviceToken.body?.id_token === "string",
|
||||||
|
groupList: groups.status === 200 && groups.body?.results?.[0]?.name === "fixture-users",
|
||||||
|
};
|
||||||
|
},
|
||||||
|
setOidcIdentity(identity) {
|
||||||
|
provider.setIdentity(identity);
|
||||||
|
},
|
||||||
|
async useLocalMode() {
|
||||||
|
writeSecure(authConfigFile, JSON.stringify(localConfig()));
|
||||||
|
mode = "local";
|
||||||
|
await restartBackend();
|
||||||
|
},
|
||||||
|
async useOidcMode(identity) {
|
||||||
|
provider.setIdentity(identity);
|
||||||
|
writeSecure(authConfigFile, JSON.stringify(oidcConfig()));
|
||||||
|
mode = "oidc";
|
||||||
|
await restartBackend();
|
||||||
|
},
|
||||||
|
restartBackend,
|
||||||
|
async close() {
|
||||||
|
await backend?.close();
|
||||||
|
await frontend.close();
|
||||||
|
await provider.close();
|
||||||
|
await qdrant?.close();
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
},
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
await backend?.close();
|
||||||
|
await frontend.close();
|
||||||
|
await provider.close();
|
||||||
|
await qdrant?.close();
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,50 +1,14 @@
|
|||||||
import { defineConfig, devices } from "@playwright/test";
|
import { defineConfig, devices } from "@playwright/test";
|
||||||
import { resolve } from "node:path";
|
|
||||||
|
|
||||||
const __dir = new URL(".", import.meta.url).pathname;
|
|
||||||
const fakePi = resolve(__dir, "e2e/fixtures/fake-pi.mjs");
|
|
||||||
const fakeTht = resolve(__dir, "e2e/fixtures/fake-tht.mjs");
|
|
||||||
const harnessDir = resolve(__dir, "..", "harness");
|
|
||||||
|
|
||||||
export default defineConfig({
|
export default defineConfig({
|
||||||
testDir: "./e2e",
|
testDir: "./e2e",
|
||||||
timeout: 60_000,
|
timeout: 60_000,
|
||||||
retries: process.env.CI ? 2 : 0,
|
retries: process.env.CI ? 2 : 0,
|
||||||
use: {
|
workers: 1,
|
||||||
baseURL: "http://localhost:5199",
|
|
||||||
},
|
|
||||||
projects: [
|
projects: [
|
||||||
{
|
{
|
||||||
name: "chromium",
|
name: "chromium",
|
||||||
use: { ...devices["Desktop Chrome"] },
|
use: { ...devices["Desktop Chrome"] },
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
webServer: [
|
|
||||||
{
|
|
||||||
// Backend — wired to both fakes; no VPN/Pi/Python required.
|
|
||||||
command: "npm run dev",
|
|
||||||
cwd: resolve(__dir, "..", "backend"),
|
|
||||||
url: "http://localhost:8799/health",
|
|
||||||
timeout: 30_000,
|
|
||||||
reuseExistingServer: !process.env.CI,
|
|
||||||
env: {
|
|
||||||
PI_BIN: fakePi,
|
|
||||||
THT_BIN: fakeTht,
|
|
||||||
THT_HARNESS_DIR: harnessDir,
|
|
||||||
PORT: "8799",
|
|
||||||
AUTH_MODE: "none",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// Frontend dev server pointing at the hermetic backend.
|
|
||||||
command: "npm run dev -- --port 5199",
|
|
||||||
cwd: __dir,
|
|
||||||
url: "http://localhost:5199",
|
|
||||||
timeout: 30_000,
|
|
||||||
reuseExistingServer: !process.env.CI,
|
|
||||||
env: {
|
|
||||||
VITE_BACKEND_URL: "http://localhost:8799",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -13,11 +13,14 @@ def test_local_compose_uses_the_generic_external_endpoint_contract():
|
|||||||
}
|
}
|
||||||
# workspace-maintenance is profile-gated: it must not be part of the default local startup.
|
# workspace-maintenance is profile-gated: it must not be part of the default local startup.
|
||||||
assert compose["services"]["workspace-maintenance"].get("profiles") == ["workspace-maintenance"]
|
assert compose["services"]["workspace-maintenance"].get("profiles") == ["workspace-maintenance"]
|
||||||
assert local["services"]["core"]["environment"]["AUTH_MODE"] == "none"
|
# Authentication is controlled by the mounted configuration; the local profile
|
||||||
|
# must not silently re-enable the legacy unauthenticated development mode.
|
||||||
|
assert "AUTH_MODE" not in local["services"]["core"]["environment"]
|
||||||
assert local["services"]["core"]["ports"] == ["127.0.0.1:${THOTH_CORE_HTTP_PORT:-8787}:8787"]
|
assert local["services"]["core"]["ports"] == ["127.0.0.1:${THOTH_CORE_HTTP_PORT:-8787}:8787"]
|
||||||
assert local["services"]["frontend"]["ports"] == ["127.0.0.1:${THOTH_HTTP_PORT:-8080}:8080"]
|
assert local["services"]["frontend"]["ports"] == ["127.0.0.1:${THOTH_HTTP_PORT:-8080}:8080"]
|
||||||
|
|
||||||
environment = compose["services"]["core"]["environment"]
|
environment = compose["services"]["core"]["environment"]
|
||||||
|
assert {"THT_AUTH_CONFIG_FILE", "THT_AUTH_STATE_ROOT"} <= set(environment)
|
||||||
for name in ("THT_DWH_REST_URL", "THT_LLM_URL"):
|
for name in ("THT_DWH_REST_URL", "THT_LLM_URL"):
|
||||||
assert name in environment
|
assert name in environment
|
||||||
assert environment["THT_INTERNAL_QDRANT_URL"] == "http://qdrant:6333"
|
assert environment["THT_INTERNAL_QDRANT_URL"] == "http://qdrant:6333"
|
||||||
|
|||||||
Executable
+56
@@ -0,0 +1,56 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)
|
||||||
|
expected_node=${THT_EXPECTED_NODE_VERSION:-v24.16.0}
|
||||||
|
actual_node=$(node --version)
|
||||||
|
|
||||||
|
if [[ "$actual_node" != "$expected_node" ]]; then
|
||||||
|
printf 'authentication smoke requires Node %s; found %s\n' "$expected_node" "$actual_node" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
for command in go openssl; do
|
||||||
|
command -v "$command" >/dev/null 2>&1 || {
|
||||||
|
printf 'authentication smoke requires %s\n' "$command" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
done
|
||||||
|
|
||||||
|
playwright="$root/frontend/node_modules/.bin/playwright"
|
||||||
|
[[ -x "$playwright" ]] || {
|
||||||
|
echo "authentication smoke requires frontend dependencies (run npm ci in frontend)" >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
temporary_root=$(mktemp -d "${TMPDIR:-/tmp}/thothii-authentication-smoke.XXXXXX")
|
||||||
|
trap 'rm -rf "$temporary_root"' EXIT HUP INT TERM
|
||||||
|
log="$temporary_root/playwright.log"
|
||||||
|
output="$temporary_root/playwright"
|
||||||
|
mkdir -p "$output"
|
||||||
|
sentinel="task15-sentinel-$(openssl rand -hex 24)"
|
||||||
|
export THT_TASK15_SENTINEL="$sentinel"
|
||||||
|
|
||||||
|
sanitize_failure_log() {
|
||||||
|
sed -E \
|
||||||
|
-e "s/${sentinel}/[redacted]/g" \
|
||||||
|
-e 's#https?://[^[:space:])]+#[url]#g' \
|
||||||
|
-e 's/(THT_[A-Z_]+|PI_[A-Z_]+|AUTH_MODE)=[^[:space:]]+/\1=[redacted]/g' \
|
||||||
|
"$log" | tail -n 100 >&2
|
||||||
|
}
|
||||||
|
|
||||||
|
if ! (
|
||||||
|
cd "$root/frontend"
|
||||||
|
THT_E2E_AUTH_STACK=1 "$playwright" test e2e/auth.spec.ts --workers=1 --output="$output"
|
||||||
|
) >"$log" 2>&1; then
|
||||||
|
echo "authentication smoke: hermetic browser suite failed" >&2
|
||||||
|
sanitize_failure_log
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if rg -a -Fq -- "$sentinel" "$log" "$output"; then
|
||||||
|
echo "authentication smoke: sentinel appeared in retained test output" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf 'authentication smoke: hermetic OIDC/browser suite passed on Node %s\n' "$actual_node"
|
||||||
@@ -18,10 +18,56 @@ trap cleanup EXIT HUP INT TERM
|
|||||||
printf '%s\n' '{"zai":{"type":"api_key","key":"fixture-native-auth-key"}}' >"$tmp/pi-auth.json"
|
printf '%s\n' '{"zai":{"type":"api_key","key":"fixture-native-auth-key"}}' >"$tmp/pi-auth.json"
|
||||||
printf '%s\n' 'THT_MODEL_API_KEY=fixture-model-api-key' >"$tmp/thothii.secrets"
|
printf '%s\n' 'THT_MODEL_API_KEY=fixture-model-api-key' >"$tmp/thothii.secrets"
|
||||||
chmod 0600 "$tmp/pi-auth.json" "$tmp/thothii.secrets"
|
chmod 0600 "$tmp/pi-auth.json" "$tmp/thothii.secrets"
|
||||||
|
auth_config="$tmp/auth"
|
||||||
|
mkdir "$auth_config"
|
||||||
|
chmod 0700 "$auth_config"
|
||||||
|
printf '%s\n' \
|
||||||
|
'version: 1' \
|
||||||
|
'mode: local' \
|
||||||
|
'publicUrl: http://127.0.0.1:8080' \
|
||||||
|
'local:' \
|
||||||
|
' usersFile: users.yaml' \
|
||||||
|
>"$auth_config/auth.yaml"
|
||||||
|
node - "$auth_config/users.yaml" <<'NODE'
|
||||||
|
const { argon2 } = require("node:crypto");
|
||||||
|
const { writeFileSync } = require("node:fs");
|
||||||
|
|
||||||
|
const message = Buffer.from("fixture-local-password", "utf8");
|
||||||
|
const nonce = Buffer.from([...Array(16).keys()]);
|
||||||
|
argon2("argon2id", {
|
||||||
|
message,
|
||||||
|
nonce,
|
||||||
|
memory: 65_536,
|
||||||
|
parallelism: 1,
|
||||||
|
tagLength: 32,
|
||||||
|
passes: 3,
|
||||||
|
}, (error, digest) => {
|
||||||
|
message.fill(0);
|
||||||
|
nonce.fill(0);
|
||||||
|
if (error || !digest) throw error ?? new Error("fixture password hash failed");
|
||||||
|
const salt = Buffer.from([...Array(16).keys()]).toString("base64").replaceAll("=", "");
|
||||||
|
const hash = digest.toString("base64").replaceAll("=", "");
|
||||||
|
writeFileSync(process.argv[2], [
|
||||||
|
"version: 1",
|
||||||
|
"users:",
|
||||||
|
" - id: 00000000-0000-4000-8000-000000000001",
|
||||||
|
" username: fixture-user",
|
||||||
|
" displayName: Fixture user",
|
||||||
|
` passwordHash: $argon2id$v=19$m=65536,t=3,p=1$${salt}$${hash}`,
|
||||||
|
" roles:",
|
||||||
|
" - user",
|
||||||
|
" enabled: true",
|
||||||
|
" authRevision: 1",
|
||||||
|
"",
|
||||||
|
].join("\\n"), { mode: 0o600 });
|
||||||
|
});
|
||||||
|
NODE
|
||||||
|
chmod 0600 "$auth_config/auth.yaml" "$auth_config/users.yaml"
|
||||||
printf '%s\n' \
|
printf '%s\n' \
|
||||||
'THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git' \
|
'THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git' \
|
||||||
"PI_AUTH_FILE=$tmp/pi-auth.json" \
|
"PI_AUTH_FILE=$tmp/pi-auth.json" \
|
||||||
"THT_SECRETS_FILE=$tmp/thothii.secrets" \
|
"THT_SECRETS_FILE=$tmp/thothii.secrets" \
|
||||||
|
"THT_AUTH_CONFIG_ROOT=$auth_config" \
|
||||||
'THOTH_CORE_HTTP_PORT=0' \
|
'THOTH_CORE_HTTP_PORT=0' \
|
||||||
'THOTH_HTTP_PORT=0' \
|
'THOTH_HTTP_PORT=0' \
|
||||||
>"$tmp/local.env"
|
>"$tmp/local.env"
|
||||||
|
|||||||
@@ -8,12 +8,18 @@ trap 'rm -rf "$tmp"' EXIT HUP INT TERM
|
|||||||
auth_file="$tmp/auth.json"
|
auth_file="$tmp/auth.json"
|
||||||
printf '%s\n' '{}' >"$auth_file"
|
printf '%s\n' '{}' >"$auth_file"
|
||||||
chmod 0600 "$auth_file"
|
chmod 0600 "$auth_file"
|
||||||
|
auth_config="$tmp/auth"
|
||||||
|
mkdir "$auth_config"
|
||||||
|
chmod 0700 "$auth_config"
|
||||||
|
printf '%s\n' 'mode: local' >"$auth_config/auth.yaml"
|
||||||
|
chmod 0600 "$auth_config/auth.yaml"
|
||||||
secrets_file="$tmp/thothii.secrets"
|
secrets_file="$tmp/thothii.secrets"
|
||||||
printf '%s\n' 'THT_MODEL_API_KEY=fixture-model-api-key' >"$secrets_file"
|
printf '%s\n' 'THT_MODEL_API_KEY=fixture-model-api-key' >"$secrets_file"
|
||||||
chmod 0600 "$secrets_file"
|
chmod 0600 "$secrets_file"
|
||||||
|
|
||||||
rendered=$(THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git \
|
rendered=$(THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git \
|
||||||
PI_AUTH_FILE="$auth_file" THT_SECRETS_FILE="$secrets_file" docker compose config)
|
PI_AUTH_FILE="$auth_file" THT_SECRETS_FILE="$secrets_file" \
|
||||||
|
THT_AUTH_CONFIG_ROOT="$auth_config" docker compose config)
|
||||||
printf '%s\n' "$rendered" | grep -q "source: $auth_file"
|
printf '%s\n' "$rendered" | grep -q "source: $auth_file"
|
||||||
printf '%s\n' "$rendered" | grep -q 'target: /home/thoth/.pi/agent/auth.json'
|
printf '%s\n' "$rendered" | grep -q 'target: /home/thoth/.pi/agent/auth.json'
|
||||||
printf '%s\n' "$rendered" | grep -A4 'target: /home/thoth/.pi/agent/auth.json' \
|
printf '%s\n' "$rendered" | grep -A4 'target: /home/thoth/.pi/agent/auth.json' \
|
||||||
@@ -33,6 +39,7 @@ fi
|
|||||||
|
|
||||||
dev_rendered=$(THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git \
|
dev_rendered=$(THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git \
|
||||||
PI_AUTH_FILE="$auth_file" THT_SECRETS_FILE="$secrets_file" \
|
PI_AUTH_FILE="$auth_file" THT_SECRETS_FILE="$secrets_file" \
|
||||||
|
THT_AUTH_CONFIG_ROOT="$auth_config" \
|
||||||
docker compose --env-file deploy/env/local.env.example -f docker-compose.dev.yml config)
|
docker compose --env-file deploy/env/local.env.example -f docker-compose.dev.yml config)
|
||||||
printf '%s\n' "$dev_rendered" | grep -q "source: $auth_file"
|
printf '%s\n' "$dev_rendered" | grep -q "source: $auth_file"
|
||||||
printf '%s\n' "$dev_rendered" | grep -q 'target: /home/thoth/.pi/agent/auth.json'
|
printf '%s\n' "$dev_rendered" | grep -q 'target: /home/thoth/.pi/agent/auth.json'
|
||||||
|
|||||||
@@ -5,13 +5,18 @@ cd "$(dirname "$0")/.."
|
|||||||
|
|
||||||
tmp_bundle=$(mktemp)
|
tmp_bundle=$(mktemp)
|
||||||
tmp_auth=$(mktemp)
|
tmp_auth=$(mktemp)
|
||||||
trap 'rm -f "$tmp_bundle" "$tmp_auth"' EXIT HUP INT TERM
|
tmp_auth_config=$(mktemp -d)
|
||||||
|
trap 'rm -f "$tmp_bundle" "$tmp_auth"; rm -rf "$tmp_auth_config"' EXIT HUP INT TERM
|
||||||
printf '%s\n' 'THT_MODEL_API_KEY=test-model' >"$tmp_bundle"
|
printf '%s\n' 'THT_MODEL_API_KEY=test-model' >"$tmp_bundle"
|
||||||
chmod 0600 "$tmp_bundle"
|
chmod 0600 "$tmp_bundle"
|
||||||
printf '%s\n' '{}' >"$tmp_auth"
|
printf '%s\n' '{}' >"$tmp_auth"
|
||||||
chmod 0600 "$tmp_auth"
|
chmod 0600 "$tmp_auth"
|
||||||
|
chmod 0700 "$tmp_auth_config"
|
||||||
|
printf '%s\n' 'mode: local' >"$tmp_auth_config/auth.yaml"
|
||||||
|
chmod 0600 "$tmp_auth_config/auth.yaml"
|
||||||
export THT_SECRETS_FILE="$tmp_bundle"
|
export THT_SECRETS_FILE="$tmp_bundle"
|
||||||
export PI_AUTH_FILE="$tmp_auth"
|
export PI_AUTH_FILE="$tmp_auth"
|
||||||
|
export THT_AUTH_CONFIG_ROOT="$tmp_auth_config"
|
||||||
export THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git
|
export THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git
|
||||||
|
|
||||||
config_json=$(docker compose -f compose.yaml -f deploy/compose.preprocess.yaml --profile preprocess config --format json)
|
config_json=$(docker compose -f compose.yaml -f deploy/compose.preprocess.yaml --profile preprocess config --format json)
|
||||||
|
|||||||
@@ -9,7 +9,12 @@ fixture="$(mktemp -d "$tmp_parent/thoth-server-pi-state.XXXXXX")"
|
|||||||
trap 'rm -rf "$fixture"' EXIT HUP INT TERM
|
trap 'rm -rf "$fixture"' EXIT HUP INT TERM
|
||||||
|
|
||||||
pi_state="$fixture/empty pi state"
|
pi_state="$fixture/empty pi state"
|
||||||
|
auth_config="$fixture/auth"
|
||||||
mkdir -p "$pi_state"
|
mkdir -p "$pi_state"
|
||||||
|
mkdir -p "$auth_config"
|
||||||
|
chmod 0700 "$auth_config"
|
||||||
|
printf 'mode: local\n' >"$auth_config/auth.yaml"
|
||||||
|
chmod 0600 "$auth_config/auth.yaml"
|
||||||
"$root/scripts/prepare-server-pi-state.sh" "$pi_state" "$(id -u)" "$(id -g)"
|
"$root/scripts/prepare-server-pi-state.sh" "$pi_state" "$(id -u)" "$(id -g)"
|
||||||
|
|
||||||
for target in auth.json models.json settings.json; do
|
for target in auth.json models.json settings.json; do
|
||||||
@@ -40,6 +45,7 @@ THOTH_SERVER_BIND=127.0.0.1
|
|||||||
THOTH_HTTP_PORT=0
|
THOTH_HTTP_PORT=0
|
||||||
PI_AUTH_FILE=$fixture/pi-auth.json
|
PI_AUTH_FILE=$fixture/pi-auth.json
|
||||||
THT_SECRETS_FILE=$fixture/thothii.secrets
|
THT_SECRETS_FILE=$fixture/thothii.secrets
|
||||||
|
THT_AUTH_CONFIG_ROOT=$auth_config
|
||||||
THT_DATA_ROOT=$fixture/data
|
THT_DATA_ROOT=$fixture/data
|
||||||
THT_PI_STATE_ROOT=$pi_state
|
THT_PI_STATE_ROOT=$pi_state
|
||||||
THT_WORKSPACE_REGISTRY_ROOT=$fixture/workspace-registry
|
THT_WORKSPACE_REGISTRY_ROOT=$fixture/workspace-registry
|
||||||
|
|||||||
@@ -127,23 +127,32 @@ func createWithDependencies(ctx context.Context, installation config.Installatio
|
|||||||
if request.IncludeSecrets && !request.Confirm {
|
if request.IncludeSecrets && !request.Confirm {
|
||||||
return Result{}, ErrConfirmationRequired
|
return Result{}, ErrConfirmationRequired
|
||||||
}
|
}
|
||||||
lock, err := lifecycle.Acquire(installation)
|
transaction, err := lifecycle.AcquireTransaction(installation)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Result{}, err
|
return Result{}, err
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
if releaseErr := lock.Release(); releaseErr != nil {
|
if releaseErr := transaction.Release(); releaseErr != nil {
|
||||||
result = Result{}
|
result = Result{}
|
||||||
resultErr = errors.Join(resultErr, fmt.Errorf("release backup lifecycle lock: %w", releaseErr))
|
resultErr = errors.Join(resultErr, fmt.Errorf("release backup lifecycle lock: %w", releaseErr))
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
return createWithDependenciesLockHeld(ctx, installation, request, dependencies)
|
return createWithDependenciesTransaction(ctx, transaction, installation, request, dependencies)
|
||||||
}
|
}
|
||||||
|
|
||||||
// createWithDependenciesLockHeld performs backup creation while the caller owns the installation
|
// createWithDependenciesTransaction performs backup creation only with an active, opaque
|
||||||
// lifecycle lock. It must never acquire a lifecycle lock itself: Restore uses this primitive to
|
// installation-bound lifecycle capability. Restore passes the capability it acquired for the
|
||||||
// create its recovery checkpoint inside its already-locked transaction.
|
// enclosing transaction, so a recovery checkpoint cannot run without the same lifecycle lock.
|
||||||
func createWithDependenciesLockHeld(ctx context.Context, installation config.Installation, request CreateRequest, dependencies dependencies) (result Result, resultErr error) {
|
func createWithDependenciesTransaction(ctx context.Context, transaction *lifecycle.Transaction, installation config.Installation, request CreateRequest, dependencies dependencies) (result Result, resultErr error) {
|
||||||
|
if err := transaction.Verify(installation); err != nil {
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
|
if dependencies.runner == nil || dependencies.now == nil || dependencies.homeDir == nil || dependencies.revision == nil || dependencies.sleep == nil || dependencies.reserveOutput == nil || dependencies.publishReserved == nil {
|
||||||
|
return Result{}, errors.New("backup dependencies are incomplete")
|
||||||
|
}
|
||||||
|
if request.IncludeSecrets && !request.Confirm {
|
||||||
|
return Result{}, ErrConfirmationRequired
|
||||||
|
}
|
||||||
if err := rejectInlineSecretValues(installation.EnvFile); err != nil {
|
if err := rejectInlineSecretValues(installation.EnvFile); err != nil {
|
||||||
return Result{}, err
|
return Result{}, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -537,6 +537,33 @@ func TestCreateHonorsTheSharedInstallationLifecycleLock(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCreateTransactionCapabilityRefusesUnlockedOrForeignInstallation(t *testing.T) {
|
||||||
|
fixture := newBackupFixture(t, "local")
|
||||||
|
runner := newBackupRunner(fixture.installation, false)
|
||||||
|
request := CreateRequest{Output: filepath.Join(t.TempDir(), "transaction.zip")}
|
||||||
|
|
||||||
|
if _, err := createWithDependenciesTransaction(context.Background(), nil, fixture.installation, request, testDependencies(t, runner)); !errors.Is(err, lifecycle.ErrTransactionInactive) {
|
||||||
|
t.Fatalf("nil transaction error = %v, want ErrTransactionInactive", err)
|
||||||
|
}
|
||||||
|
if len(runner.calls) != 0 {
|
||||||
|
t.Fatalf("Docker runner was called without a transaction capability: %v", runner.calls)
|
||||||
|
}
|
||||||
|
|
||||||
|
otherRoot := t.TempDir()
|
||||||
|
other := config.Installation{ProjectDirectory: otherRoot, Path: filepath.Join(otherRoot, "thothii-installation.yaml")}
|
||||||
|
transaction, err := lifecycle.AcquireTransaction(other)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = transaction.Release() })
|
||||||
|
if _, err := createWithDependenciesTransaction(context.Background(), transaction, fixture.installation, request, testDependencies(t, runner)); !errors.Is(err, lifecycle.ErrTransactionInstallation) {
|
||||||
|
t.Fatalf("foreign transaction error = %v, want ErrTransactionInstallation", err)
|
||||||
|
}
|
||||||
|
if len(runner.calls) != 0 {
|
||||||
|
t.Fatalf("Docker runner was called with a foreign transaction capability: %v", runner.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCreateRefusesMutableNonRunningServiceStates(t *testing.T) {
|
func TestCreateRefusesMutableNonRunningServiceStates(t *testing.T) {
|
||||||
for _, state := range []string{"paused", "restarting", "created", "removing"} {
|
for _, state := range []string{"paused", "restarting", "created", "removing"} {
|
||||||
t.Run(state, func(t *testing.T) {
|
t.Run(state, func(t *testing.T) {
|
||||||
|
|||||||
@@ -91,6 +91,14 @@ type verifiedArchive struct {
|
|||||||
limits PreflightLimits
|
limits PreflightLimits
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// stagedArchive holds an installation-private, immutable copy of the exact bytes accepted by
|
||||||
|
// Preflight. The original archive remains retained only for provenance revalidation.
|
||||||
|
type stagedArchive struct {
|
||||||
|
file *os.File
|
||||||
|
path string
|
||||||
|
directory string
|
||||||
|
}
|
||||||
|
|
||||||
type inspectedArchiveEntry struct {
|
type inspectedArchiveEntry struct {
|
||||||
metadata ArchiveEntryMetadata
|
metadata ArchiveEntryMetadata
|
||||||
member *zip.File
|
member *zip.File
|
||||||
@@ -176,12 +184,17 @@ func Preflight(ctx context.Context, installation config.Installation, request Pr
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return PreflightResult{}, err
|
return PreflightResult{}, err
|
||||||
}
|
}
|
||||||
|
stagingBytes := uint64(openedInfo.Size())
|
||||||
|
if stagingBytes > ^uint64(0)-requiredBytes {
|
||||||
|
return PreflightResult{}, errors.New("restore staging requirement exceeds supported size")
|
||||||
|
}
|
||||||
|
requiredWithStaging := requiredBytes + stagingBytes
|
||||||
freeBytes, err := dependencies.FreeBytes(installation.ProjectDirectory)
|
freeBytes, err := dependencies.FreeBytes(installation.ProjectDirectory)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return PreflightResult{}, fmt.Errorf("check free disk space: %w", err)
|
return PreflightResult{}, fmt.Errorf("check free disk space: %w", err)
|
||||||
}
|
}
|
||||||
if freeBytes < requiredBytes {
|
if freeBytes < requiredWithStaging {
|
||||||
return PreflightResult{}, fmt.Errorf("insufficient free disk space for restore: need %d bytes, have %d", requiredBytes, freeBytes)
|
return PreflightResult{}, fmt.Errorf("insufficient free disk space for restore: need %d bytes, have %d", requiredWithStaging, freeBytes)
|
||||||
}
|
}
|
||||||
if err := contextError(ctx); err != nil {
|
if err := contextError(ctx); err != nil {
|
||||||
return PreflightResult{}, err
|
return PreflightResult{}, err
|
||||||
@@ -214,6 +227,10 @@ func Preflight(ctx context.Context, installation config.Installation, request Pr
|
|||||||
// it immediately before a restore transaction and use the returned retained handle, never reopen
|
// it immediately before a restore transaction and use the returned retained handle, never reopen
|
||||||
// ArchivePath. It refuses a path replacement or in-place content change.
|
// ArchivePath. It refuses a path replacement or in-place content change.
|
||||||
func (result PreflightResult) RevalidateArchive() (*os.File, error) {
|
func (result PreflightResult) RevalidateArchive() (*os.File, error) {
|
||||||
|
return result.revalidateArchive(context.Background())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (result PreflightResult) revalidateArchive(ctx context.Context) (*os.File, error) {
|
||||||
if result.archive == nil || result.archive.file == nil {
|
if result.archive == nil || result.archive.file == nil {
|
||||||
return nil, errors.New("backup archive has not been retained by preflight")
|
return nil, errors.New("backup archive has not been retained by preflight")
|
||||||
}
|
}
|
||||||
@@ -228,13 +245,115 @@ func (result PreflightResult) RevalidateArchive() (*os.File, error) {
|
|||||||
if err != nil || !os.SameFile(result.archive.info, heldInfo) || heldInfo.Size() != result.ArchiveSize {
|
if err != nil || !os.SameFile(result.archive.info, heldInfo) || heldInfo.Size() != result.ArchiveSize {
|
||||||
return nil, errors.New("backup archive changed after preflight")
|
return nil, errors.New("backup archive changed after preflight")
|
||||||
}
|
}
|
||||||
digest, err := digestArchive(context.Background(), result.archive.file, result.archive.limits.MaxArchiveBytes)
|
digest, err := digestArchive(ctx, result.archive.file, result.archive.limits.MaxArchiveBytes)
|
||||||
if err != nil || digest != result.archive.digest {
|
if err != nil || digest != result.archive.digest {
|
||||||
return nil, errors.New("backup archive changed after preflight")
|
return nil, errors.New("backup archive changed after preflight")
|
||||||
}
|
}
|
||||||
return result.archive.file, nil
|
return result.archive.file, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StageArchive revalidates the retained archive and copies its exact bytes into a private file
|
||||||
|
// immediately before extraction. Later writes to the source archive cannot affect extraction.
|
||||||
|
func (result PreflightResult) StageArchive(ctx context.Context) (_ *stagedArchive, resultErr error) {
|
||||||
|
source, err := result.revalidateArchive(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
directory, err := os.MkdirTemp("", "tht-restore-stage-")
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.New("create private restore staging directory")
|
||||||
|
}
|
||||||
|
if err := os.Chmod(directory, 0o700); err != nil {
|
||||||
|
_ = os.Remove(directory)
|
||||||
|
return nil, errors.New("protect private restore staging directory")
|
||||||
|
}
|
||||||
|
path := filepath.Join(directory, "archive.zip")
|
||||||
|
file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600)
|
||||||
|
if err != nil {
|
||||||
|
_ = os.Remove(directory)
|
||||||
|
return nil, errors.New("create private restore staging archive")
|
||||||
|
}
|
||||||
|
staged := &stagedArchive{file: file, path: path, directory: directory}
|
||||||
|
completed := false
|
||||||
|
defer func() {
|
||||||
|
if !completed {
|
||||||
|
_ = staged.Close()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
if _, err := source.Seek(0, io.SeekStart); err != nil {
|
||||||
|
return nil, errors.New("seek verified backup archive for staging")
|
||||||
|
}
|
||||||
|
digest := sha256.New()
|
||||||
|
buffer := make([]byte, 128*1024)
|
||||||
|
var total int64
|
||||||
|
for {
|
||||||
|
if err := contextError(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
count, readErr := source.Read(buffer)
|
||||||
|
if count > 0 {
|
||||||
|
if int64(count) > result.ArchiveSize-total {
|
||||||
|
return nil, errors.New("backup archive changed after preflight")
|
||||||
|
}
|
||||||
|
written, writeErr := file.Write(buffer[:count])
|
||||||
|
if writeErr != nil || written != count {
|
||||||
|
return nil, errors.New("write private restore staging archive")
|
||||||
|
}
|
||||||
|
if _, writeErr := digest.Write(buffer[:count]); writeErr != nil {
|
||||||
|
return nil, errors.New("hash private restore staging archive")
|
||||||
|
}
|
||||||
|
total += int64(count)
|
||||||
|
}
|
||||||
|
if errors.Is(readErr, io.EOF) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if readErr != nil {
|
||||||
|
return nil, errors.New("read verified backup archive for staging")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if total != result.ArchiveSize || digestForHash(digest) != result.archive.digest {
|
||||||
|
return nil, errors.New("backup archive changed after preflight")
|
||||||
|
}
|
||||||
|
if err := file.Sync(); err != nil {
|
||||||
|
return nil, errors.New("sync private restore staging archive")
|
||||||
|
}
|
||||||
|
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
||||||
|
return nil, errors.New("rewind private restore staging archive")
|
||||||
|
}
|
||||||
|
completed = true
|
||||||
|
return staged, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close removes only the staging file and directory created by StageArchive.
|
||||||
|
func (staged *stagedArchive) Close() error {
|
||||||
|
if staged == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var failed bool
|
||||||
|
if staged.file != nil {
|
||||||
|
if err := staged.file.Close(); err != nil {
|
||||||
|
failed = true
|
||||||
|
}
|
||||||
|
staged.file = nil
|
||||||
|
}
|
||||||
|
if staged.path != "" {
|
||||||
|
if err := os.Remove(staged.path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
failed = true
|
||||||
|
}
|
||||||
|
staged.path = ""
|
||||||
|
}
|
||||||
|
if staged.directory != "" {
|
||||||
|
if err := os.Remove(staged.directory); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
failed = true
|
||||||
|
}
|
||||||
|
staged.directory = ""
|
||||||
|
}
|
||||||
|
if failed {
|
||||||
|
return errors.New("destroy private restore staging archive")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// CloseArchive releases the retained read-only archive handle after the caller finishes the
|
// CloseArchive releases the retained read-only archive handle after the caller finishes the
|
||||||
// restore transaction or decides not to proceed.
|
// restore transaction or decides not to proceed.
|
||||||
func (result PreflightResult) CloseArchive() error {
|
func (result PreflightResult) CloseArchive() error {
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"errors"
|
"errors"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -356,6 +357,79 @@ func TestPreflightRevalidationRefusesAnArchivePathThatWasReplaced(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPreflightStagesArchiveIntoImmutablePrivateBytes(t *testing.T) {
|
||||||
|
installation := preflightTestInstallation(t)
|
||||||
|
archive := filepath.Join(t.TempDir(), "checked.zip")
|
||||||
|
writePreflightArchive(t, archive, preflightArchiveSpec{
|
||||||
|
entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("before")}},
|
||||||
|
})
|
||||||
|
|
||||||
|
result, err := Preflight(context.Background(), installation, PreflightRequest{Archive: archive, Confirm: true}, permissivePreflightDependencies())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer result.CloseArchive()
|
||||||
|
staged, err := result.StageArchive(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer staged.Close()
|
||||||
|
|
||||||
|
writePreflightArchive(t, archive, preflightArchiveSpec{
|
||||||
|
entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("after!")}},
|
||||||
|
})
|
||||||
|
reader, err := zip.NewReader(staged.file, result.ArchiveSize)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(reader.File) < 2 {
|
||||||
|
t.Fatalf("staged archive members = %d, want manifest and payload", len(reader.File))
|
||||||
|
}
|
||||||
|
var payload *zip.File
|
||||||
|
for _, member := range reader.File {
|
||||||
|
if member.Name == "configuration/operator.env" {
|
||||||
|
payload = member
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if payload == nil {
|
||||||
|
t.Fatal("staged archive is missing the configured payload")
|
||||||
|
}
|
||||||
|
stream, err := payload.Open()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer stream.Close()
|
||||||
|
body, err := io.ReadAll(stream)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if string(body) != "before" {
|
||||||
|
t.Fatalf("staged payload = %q, want preflighted bytes", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreflightStagingRejectsInPlaceArchiveHashMutation(t *testing.T) {
|
||||||
|
installation := preflightTestInstallation(t)
|
||||||
|
archive := filepath.Join(t.TempDir(), "checked.zip")
|
||||||
|
writePreflightArchive(t, archive, preflightArchiveSpec{
|
||||||
|
entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("before")}},
|
||||||
|
})
|
||||||
|
|
||||||
|
result, err := Preflight(context.Background(), installation, PreflightRequest{Archive: archive, Confirm: true}, permissivePreflightDependencies())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer result.CloseArchive()
|
||||||
|
writePreflightArchive(t, archive, preflightArchiveSpec{
|
||||||
|
entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("after!")}},
|
||||||
|
})
|
||||||
|
|
||||||
|
if _, err := result.StageArchive(context.Background()); err == nil || !strings.Contains(err.Error(), "changed") {
|
||||||
|
t.Fatalf("StageArchive() error = %v, want changed archive refusal", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPreflightRejectsArchiveEntryWithModeDifferentFromManifest(t *testing.T) {
|
func TestPreflightRejectsArchiveEntryWithModeDifferentFromManifest(t *testing.T) {
|
||||||
installation := preflightTestInstallation(t)
|
installation := preflightTestInstallation(t)
|
||||||
archive := filepath.Join(t.TempDir(), "mode-mismatch.zip")
|
archive := filepath.Join(t.TempDir(), "mode-mismatch.zip")
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
||||||
|
"github.com/aritmolab/thothii/tools/tht/internal/lifecycle"
|
||||||
)
|
)
|
||||||
|
|
||||||
var ErrRestoreConfirmationRequired = errors.New("restore requires --yes")
|
var ErrRestoreConfirmationRequired = errors.New("restore requires --yes")
|
||||||
@@ -29,19 +30,17 @@ type RestoreResult struct {
|
|||||||
Verified bool
|
Verified bool
|
||||||
}
|
}
|
||||||
|
|
||||||
type restoreLock interface{ Release() error }
|
|
||||||
type restoreVerify func(context.Context, config.Installation, archiveRunner) error
|
type restoreVerify func(context.Context, config.Installation, archiveRunner) error
|
||||||
|
|
||||||
type restoreDependencies struct {
|
type restoreDependencies struct {
|
||||||
preflight func(context.Context, config.Installation, PreflightRequest) (PreflightResult, error)
|
preflight func(context.Context, config.Installation, PreflightRequest) (PreflightResult, error)
|
||||||
// checkpointLocked creates the secret-aware recovery archive while the caller already owns
|
// checkpoint requires the opaque capability created by lifecycle acquisition. It must not call
|
||||||
// the installation lifecycle lock. It must not call public Create, which would re-acquire the
|
// public Create, which would re-acquire the non-reentrant lock and deadlock the transaction.
|
||||||
// non-reentrant lock and deadlock the restore transaction.
|
checkpoint func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error)
|
||||||
checkpointLocked func(context.Context, config.Installation, CreateRequest) (Result, error)
|
|
||||||
prepareRecovery func(context.Context, config.Installation, string) (PreflightResult, error)
|
prepareRecovery func(context.Context, config.Installation, string) (PreflightResult, error)
|
||||||
recover func(context.Context, config.Installation, PreflightResult, bool) error
|
recover func(context.Context, config.Installation, PreflightResult, bool) error
|
||||||
cleanupCheckpoint func(string) error
|
cleanupCheckpoint func(string) error
|
||||||
acquireLock func(config.Installation) (restoreLock, error)
|
acquireTransaction func(config.Installation) (*lifecycle.Transaction, error)
|
||||||
runner archiveRunner
|
runner archiveRunner
|
||||||
sleep func(duration time.Duration)
|
sleep func(duration time.Duration)
|
||||||
restoreFile func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error
|
restoreFile func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error
|
||||||
@@ -84,36 +83,32 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
|
|||||||
if request.Archive == "" {
|
if request.Archive == "" {
|
||||||
return RestoreResult{}, errors.New("restore archive is required")
|
return RestoreResult{}, errors.New("restore archive is required")
|
||||||
}
|
}
|
||||||
if deps.preflight == nil || deps.checkpointLocked == nil || deps.prepareRecovery == nil || deps.recover == nil || deps.cleanupCheckpoint == nil || deps.acquireLock == nil || deps.runner == nil || deps.restoreFile == nil || deps.restoreVolume == nil || deps.resetAuthenticationState == nil || deps.verify == nil {
|
if deps.preflight == nil || deps.checkpoint == nil || deps.prepareRecovery == nil || deps.recover == nil || deps.cleanupCheckpoint == nil || deps.acquireTransaction == nil || deps.runner == nil || deps.restoreFile == nil || deps.restoreVolume == nil || deps.resetAuthenticationState == nil || deps.verify == nil {
|
||||||
return RestoreResult{}, errors.New("restore dependencies are incomplete")
|
return RestoreResult{}, errors.New("restore dependencies are incomplete")
|
||||||
}
|
}
|
||||||
preflight, err := deps.preflight(ctx, installation, PreflightRequest{Archive: request.Archive, Confirm: true, AllowExternalSecrets: true})
|
|
||||||
if err != nil {
|
|
||||||
return RestoreResult{}, err
|
|
||||||
}
|
|
||||||
archive, err := preflight.RevalidateArchive()
|
|
||||||
if err != nil {
|
|
||||||
_ = preflight.CloseArchive()
|
|
||||||
return RestoreResult{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Lock ordering is lifecycle lock -> Compose/operator maintenance barrier. The core operator
|
// Lock ordering is lifecycle lock -> Compose/operator maintenance barrier. The core operator
|
||||||
// command never acquires the host lifecycle lock, so this order cannot form a lock cycle with
|
// command never acquires the host lifecycle lock, so this order cannot form a lock cycle with
|
||||||
// Docker Compose or the durable maintenance marker.
|
// Docker Compose or the durable maintenance marker.
|
||||||
lock, err := deps.acquireLock(installation)
|
transaction, err := deps.acquireTransaction(installation)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = preflight.CloseArchive()
|
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
if releaseErr := lock.Release(); releaseErr != nil {
|
if releaseErr := transaction.Release(); releaseErr != nil {
|
||||||
result = RestoreResult{}
|
result = RestoreResult{}
|
||||||
resultErr = errors.Join(resultErr, fmt.Errorf("release restore lifecycle lock: %w", releaseErr))
|
resultErr = errors.Join(resultErr, fmt.Errorf("release restore lifecycle lock: %w", releaseErr))
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
// Target-dependent preflight is intentionally inside the lifecycle transaction. This binds
|
||||||
|
// target ownership, volume, image, and free-space checks to the later mutation.
|
||||||
|
preflight, err := deps.preflight(ctx, installation, PreflightRequest{Archive: request.Archive, Confirm: true, AllowExternalSecrets: true})
|
||||||
|
if err != nil {
|
||||||
|
return RestoreResult{}, err
|
||||||
|
}
|
||||||
defer preflight.CloseArchive()
|
defer preflight.CloseArchive()
|
||||||
|
|
||||||
checkpoint, err := deps.checkpointLocked(ctx, installation, CreateRequest{IncludeSecrets: true, Confirm: true})
|
checkpoint, err := deps.checkpoint(ctx, transaction, installation, CreateRequest{IncludeSecrets: true, Confirm: true})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, fmt.Errorf("create recovery checkpoint: %w", err)
|
return result, fmt.Errorf("create recovery checkpoint: %w", err)
|
||||||
}
|
}
|
||||||
@@ -217,8 +212,18 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
|
|||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
staged, err := preflight.StageArchive(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if closeErr := staged.Close(); closeErr != nil {
|
||||||
|
result = RestoreResult{}
|
||||||
|
resultErr = errors.Join(resultErr, closeErr)
|
||||||
|
}
|
||||||
|
}()
|
||||||
state.mutated = true
|
state.mutated = true
|
||||||
if err := restoreVerifiedEntries(ctx, installation, preflight, archive, deps.restoreFile, deps.restoreVolume); err != nil {
|
if err := restoreVerifiedEntries(ctx, installation, preflight, staged.file, deps.restoreFile, deps.restoreVolume); err != nil {
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
if err := deps.resetAuthenticationState(ctx, installation, deps.runner); err != nil {
|
if err := deps.resetAuthenticationState(ctx, installation, deps.runner); err != nil {
|
||||||
|
|||||||
@@ -41,21 +41,19 @@ func productionRestoreDependencies(installation config.Installation) restoreDepe
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
},
|
},
|
||||||
checkpointLocked: func(ctx context.Context, target config.Installation, request CreateRequest) (Result, error) {
|
checkpoint: func(ctx context.Context, transaction *lifecycle.Transaction, target config.Installation, request CreateRequest) (Result, error) {
|
||||||
path, err := restoreCheckpointPath(target, time.Now().UTC())
|
path, err := restoreCheckpointPath(target, time.Now().UTC())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Result{}, err
|
return Result{}, err
|
||||||
}
|
}
|
||||||
request.Output = path
|
request.Output = path
|
||||||
return createWithDependenciesLockHeld(ctx, target, request, productionCreateDependencies(target))
|
return createWithDependenciesTransaction(ctx, transaction, target, request, productionCreateDependencies(target))
|
||||||
},
|
},
|
||||||
cleanupCheckpoint: cleanupRecoveryCheckpoint,
|
cleanupCheckpoint: cleanupRecoveryCheckpoint,
|
||||||
acquireLock: func(target config.Installation) (restoreLock, error) {
|
acquireTransaction: lifecycle.AcquireTransaction,
|
||||||
return lifecycle.Acquire(target)
|
runner: runner,
|
||||||
},
|
sleep: time.Sleep,
|
||||||
runner: runner,
|
restoreFile: restoreFilePayload,
|
||||||
sleep: time.Sleep,
|
|
||||||
restoreFile: restoreFilePayload,
|
|
||||||
restoreVolume: func(ctx context.Context, _ config.Installation, volume VolumeMetadata, input io.Reader) error {
|
restoreVolume: func(ctx context.Context, _ config.Installation, volume VolumeMetadata, input io.Reader) error {
|
||||||
result, err := runner.Stream(ctx, volumeRestoreCommand(volume.Name), input, io.Discard)
|
result, err := runner.Stream(ctx, volumeRestoreCommand(volume.Name), input, io.Discard)
|
||||||
if err != nil || result.ExitCode != 0 {
|
if err != nil || result.ExitCode != 0 {
|
||||||
@@ -90,8 +88,7 @@ func cleanupRecoveryCheckpoint(path string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func recoverRestoreTransaction(ctx context.Context, installation config.Installation, recovery PreflightResult, wasRunning bool, deps restoreDependencies) error {
|
func recoverRestoreTransaction(ctx context.Context, installation config.Installation, recovery PreflightResult, wasRunning bool, deps restoreDependencies) (resultErr error) {
|
||||||
var resultErr error
|
|
||||||
if err := runCompose(ctx, installation, deps.runner, "stop"); err != nil {
|
if err := runCompose(ctx, installation, deps.runner, "stop"); err != nil {
|
||||||
resultErr = errors.Join(resultErr, err)
|
resultErr = errors.Join(resultErr, err)
|
||||||
// The first stop may already have taken effect before Docker lost its response. Retry the
|
// The first stop may already have taken effect before Docker lost its response. Retry the
|
||||||
@@ -100,11 +97,16 @@ func recoverRestoreTransaction(ctx context.Context, installation config.Installa
|
|||||||
return errors.Join(resultErr, retryErr)
|
return errors.Join(resultErr, retryErr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
archive, err := recovery.RevalidateArchive()
|
staged, err := recovery.StageArchive(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Join(resultErr, err)
|
return errors.Join(resultErr, err)
|
||||||
}
|
}
|
||||||
if err := restoreVerifiedEntries(ctx, installation, recovery, archive, deps.restoreFile, deps.restoreVolume); err != nil {
|
defer func() {
|
||||||
|
if closeErr := staged.Close(); closeErr != nil {
|
||||||
|
resultErr = errors.Join(resultErr, closeErr)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
if err := restoreVerifiedEntries(ctx, installation, recovery, staged.file, deps.restoreFile, deps.restoreVolume); err != nil {
|
||||||
return errors.Join(resultErr, err)
|
return errors.Join(resultErr, err)
|
||||||
}
|
}
|
||||||
// Recovery restores only configuration/secret files and durable application volumes. Runtime
|
// Recovery restores only configuration/secret files and durable application volumes. Runtime
|
||||||
|
|||||||
@@ -120,7 +120,7 @@ func TestRestoreStoppedInstallationRunsCheckpointRestoreAndVerification(t *testi
|
|||||||
var events []string
|
var events []string
|
||||||
var checkpointRequest CreateRequest
|
var checkpointRequest CreateRequest
|
||||||
deps := restoreTestDependencies(t, runner)
|
deps := restoreTestDependencies(t, runner)
|
||||||
deps.checkpointLocked = func(_ context.Context, _ config.Installation, request CreateRequest) (Result, error) {
|
deps.checkpoint = func(_ context.Context, _ *lifecycle.Transaction, _ config.Installation, request CreateRequest) (Result, error) {
|
||||||
events = append(events, "checkpoint")
|
events = append(events, "checkpoint")
|
||||||
checkpointRequest = request
|
checkpointRequest = request
|
||||||
return Result{Path: "/tmp/checkpoint.zip"}, nil
|
return Result{Path: "/tmp/checkpoint.zip"}, nil
|
||||||
@@ -133,9 +133,9 @@ func TestRestoreStoppedInstallationRunsCheckpointRestoreAndVerification(t *testi
|
|||||||
events = append(events, "cleanup-checkpoint")
|
events = append(events, "cleanup-checkpoint")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
deps.acquireLock = func(config.Installation) (restoreLock, error) {
|
deps.acquireTransaction = func(target config.Installation) (*lifecycle.Transaction, error) {
|
||||||
events = append(events, "lock")
|
events = append(events, "lock")
|
||||||
return fakeRestoreLock{release: func() { events = append(events, "unlock") }}, nil
|
return lifecycle.AcquireTransaction(target)
|
||||||
}
|
}
|
||||||
deps.restoreFile = func(_ context.Context, _ config.Installation, entry ArchiveEntryMetadata, _ io.Reader) error {
|
deps.restoreFile = func(_ context.Context, _ config.Installation, entry ArchiveEntryMetadata, _ io.Reader) error {
|
||||||
events = append(events, "file:"+entry.Path)
|
events = append(events, "file:"+entry.Path)
|
||||||
@@ -159,9 +159,12 @@ func TestRestoreStoppedInstallationRunsCheckpointRestoreAndVerification(t *testi
|
|||||||
if !checkpointRequest.IncludeSecrets || !checkpointRequest.Confirm {
|
if !checkpointRequest.IncludeSecrets || !checkpointRequest.Confirm {
|
||||||
t.Fatalf("checkpoint request = %#v, want private confirmed secret-aware checkpoint", checkpointRequest)
|
t.Fatalf("checkpoint request = %#v, want private confirmed secret-aware checkpoint", checkpointRequest)
|
||||||
}
|
}
|
||||||
if got, want := events, []string{"lock", "checkpoint", "prepare-recovery", "file:configuration/operator.env", "health", "doctor", "pi", "workspace", "cleanup-checkpoint", "unlock"}; !equalStrings(got, want) {
|
if got, want := events, []string{"lock", "checkpoint", "prepare-recovery", "file:configuration/operator.env", "health", "doctor", "pi", "workspace", "cleanup-checkpoint"}; !equalStrings(got, want) {
|
||||||
t.Fatalf("restore events = %v, want %v", got, want)
|
t.Fatalf("restore events = %v, want %v", got, want)
|
||||||
}
|
}
|
||||||
|
if err := lifecycleLockFreeAfterTerminalRestore(installation); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRestoreClosesTargetArchiveBeforeReleasingLifecycleLock(t *testing.T) {
|
func TestRestoreClosesTargetArchiveBeforeReleasingLifecycleLock(t *testing.T) {
|
||||||
@@ -179,22 +182,85 @@ func TestRestoreClosesTargetArchiveBeforeReleasingLifecycleLock(t *testing.T) {
|
|||||||
target = result
|
target = result
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
unlockBeforeTargetClose := false
|
deps.acquireTransaction = lifecycle.AcquireTransaction
|
||||||
deps.acquireLock = func(config.Installation) (restoreLock, error) {
|
|
||||||
return fakeRestoreLock{release: func() {
|
|
||||||
if target.archive != nil && target.archive.file != nil {
|
|
||||||
if _, err := target.archive.file.Stat(); err == nil {
|
|
||||||
unlockBeforeTargetClose = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err != nil {
|
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if unlockBeforeTargetClose {
|
if target.archive != nil && target.archive.file != nil {
|
||||||
t.Fatal("restore released its lifecycle lock before closing the target archive")
|
if _, err := target.archive.file.Stat(); err == nil {
|
||||||
|
t.Fatal("restore did not close the target archive")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := lifecycleLockFreeAfterTerminalRestore(installation); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRestoreAcquiresLifecycleLockBeforeTargetDependentPreflight(t *testing.T) {
|
||||||
|
installation := preflightTestInstallation(t)
|
||||||
|
archive := filepath.Join(t.TempDir(), "restore.zip")
|
||||||
|
writePreflightArchive(t, archive, preflightArchiveSpec{
|
||||||
|
entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("safe")}},
|
||||||
|
})
|
||||||
|
|
||||||
|
runner := newBackupRunner(installation, false)
|
||||||
|
deps := restoreTestDependencies(t, runner)
|
||||||
|
deps.acquireTransaction = lifecycle.AcquireTransaction
|
||||||
|
deps.preflight = func(ctx context.Context, target config.Installation, request PreflightRequest) (PreflightResult, error) {
|
||||||
|
probe, err := lifecycle.Acquire(target)
|
||||||
|
if err == nil {
|
||||||
|
_ = probe.Release()
|
||||||
|
return PreflightResult{}, errors.New("target-dependent preflight ran before lifecycle lock acquisition")
|
||||||
|
}
|
||||||
|
if !errors.Is(err, lifecycle.ErrLocked) {
|
||||||
|
return PreflightResult{}, fmt.Errorf("probe lifecycle lock: %w", err)
|
||||||
|
}
|
||||||
|
return Preflight(ctx, target, request, permissivePreflightDependencies())
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err != nil {
|
||||||
|
t.Fatalf("Restore() error = %v, want preflight protected by lifecycle lock", err)
|
||||||
|
}
|
||||||
|
if err := lifecycleLockFreeAfterTerminalRestore(installation); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRestoreStagesArchiveAfterCheckpointAndRejectsMutation(t *testing.T) {
|
||||||
|
installation := preflightTestInstallation(t)
|
||||||
|
archive := filepath.Join(t.TempDir(), "restore.zip")
|
||||||
|
writePreflightArchive(t, archive, preflightArchiveSpec{
|
||||||
|
entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("before")}},
|
||||||
|
})
|
||||||
|
|
||||||
|
runner := newBackupRunner(installation, false)
|
||||||
|
deps := restoreTestDependencies(t, runner)
|
||||||
|
deps.acquireTransaction = lifecycle.AcquireTransaction
|
||||||
|
deps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
|
||||||
|
writePreflightArchive(t, archive, preflightArchiveSpec{
|
||||||
|
entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("after!")}},
|
||||||
|
})
|
||||||
|
return Result{Path: filepath.Join(t.TempDir(), "checkpoint.zip")}, nil
|
||||||
|
}
|
||||||
|
var restored [][]byte
|
||||||
|
deps.restoreFile = func(_ context.Context, _ config.Installation, _ ArchiveEntryMetadata, stream io.Reader) error {
|
||||||
|
body, err := io.ReadAll(stream)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
restored = append(restored, body)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err == nil || !strings.Contains(err.Error(), "changed") {
|
||||||
|
t.Fatalf("Restore() error = %v, want archive mutation refusal", err)
|
||||||
|
}
|
||||||
|
if len(restored) != 0 {
|
||||||
|
t.Fatalf("restore applied mutated archive payloads: %q", restored)
|
||||||
|
}
|
||||||
|
if err := lifecycleLockFreeAfterTerminalRestore(installation); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -297,8 +363,8 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t
|
|||||||
caller, cancel := context.WithCancel(context.Background())
|
caller, cancel := context.WithCancel(context.Background())
|
||||||
defer cancel()
|
defer cancel()
|
||||||
deps := restoreTestDependencies(t, runner)
|
deps := restoreTestDependencies(t, runner)
|
||||||
deps.acquireLock = func(target config.Installation) (restoreLock, error) { return lifecycle.Acquire(target) }
|
deps.acquireTransaction = lifecycle.AcquireTransaction
|
||||||
deps.checkpointLocked = func(context.Context, config.Installation, CreateRequest) (Result, error) {
|
deps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
|
||||||
gate("checkpoint")
|
gate("checkpoint")
|
||||||
return Result{Path: filepath.Join(t.TempDir(), "checkpoint.zip")}, nil
|
return Result{Path: filepath.Join(t.TempDir(), "checkpoint.zip")}, nil
|
||||||
}
|
}
|
||||||
@@ -369,8 +435,8 @@ func competingRestoreAndBackupEntry(installation config.Installation, archive st
|
|||||||
|
|
||||||
checkpointCalled := false
|
checkpointCalled := false
|
||||||
restoreDeps := restoreTestDependencies(t, newBackupRunner(installation, false))
|
restoreDeps := restoreTestDependencies(t, newBackupRunner(installation, false))
|
||||||
restoreDeps.acquireLock = func(target config.Installation) (restoreLock, error) { return lifecycle.Acquire(target) }
|
restoreDeps.acquireTransaction = lifecycle.AcquireTransaction
|
||||||
restoreDeps.checkpointLocked = func(context.Context, config.Installation, CreateRequest) (Result, error) {
|
restoreDeps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
|
||||||
checkpointCalled = true
|
checkpointCalled = true
|
||||||
return Result{Path: filepath.Join(t.TempDir(), "competing-checkpoint.zip")}, nil
|
return Result{Path: filepath.Join(t.TempDir(), "competing-checkpoint.zip")}, nil
|
||||||
}
|
}
|
||||||
@@ -408,8 +474,8 @@ func TestRestoreCannotApplyAStaleCheckpointOverAnInterleavedRestore(t *testing.T
|
|||||||
continueCheckpoint := make(chan struct{})
|
continueCheckpoint := make(chan struct{})
|
||||||
firstRunner := newBackupRunner(installation, true)
|
firstRunner := newBackupRunner(installation, true)
|
||||||
firstDeps := restoreTestDependencies(t, firstRunner)
|
firstDeps := restoreTestDependencies(t, firstRunner)
|
||||||
firstDeps.acquireLock = func(target config.Installation) (restoreLock, error) { return lifecycle.Acquire(target) }
|
firstDeps.acquireTransaction = lifecycle.AcquireTransaction
|
||||||
firstDeps.checkpointLocked = func(context.Context, config.Installation, CreateRequest) (Result, error) {
|
firstDeps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
|
||||||
checkpointState = targetState
|
checkpointState = targetState
|
||||||
close(checkpointEntered)
|
close(checkpointEntered)
|
||||||
<-continueCheckpoint
|
<-continueCheckpoint
|
||||||
@@ -439,8 +505,8 @@ func TestRestoreCannotApplyAStaleCheckpointOverAnInterleavedRestore(t *testing.T
|
|||||||
interleavedCheckpoint := false
|
interleavedCheckpoint := false
|
||||||
interleavedMutation := false
|
interleavedMutation := false
|
||||||
interleavedDeps := restoreTestDependencies(t, newBackupRunner(installation, false))
|
interleavedDeps := restoreTestDependencies(t, newBackupRunner(installation, false))
|
||||||
interleavedDeps.acquireLock = func(target config.Installation) (restoreLock, error) { return lifecycle.Acquire(target) }
|
interleavedDeps.acquireTransaction = lifecycle.AcquireTransaction
|
||||||
interleavedDeps.checkpointLocked = func(context.Context, config.Installation, CreateRequest) (Result, error) {
|
interleavedDeps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
|
||||||
interleavedCheckpoint = true
|
interleavedCheckpoint = true
|
||||||
return Result{Path: filepath.Join(t.TempDir(), "interleaved-checkpoint.zip")}, nil
|
return Result{Path: filepath.Join(t.TempDir(), "interleaved-checkpoint.zip")}, nil
|
||||||
}
|
}
|
||||||
@@ -542,7 +608,7 @@ func TestRestorePreflightFailureDoesNotMutateTarget(t *testing.T) {
|
|||||||
return PreflightResult{}, preflightErr
|
return PreflightResult{}, preflightErr
|
||||||
}
|
}
|
||||||
checkpointCalls := 0
|
checkpointCalls := 0
|
||||||
deps.checkpointLocked = func(context.Context, config.Installation, CreateRequest) (Result, error) {
|
deps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
|
||||||
checkpointCalls++
|
checkpointCalls++
|
||||||
return Result{}, nil
|
return Result{}, nil
|
||||||
}
|
}
|
||||||
@@ -567,7 +633,7 @@ func TestRestoreCheckpointFailureDoesNotMutateTarget(t *testing.T) {
|
|||||||
runner := newBackupRunner(installation, true)
|
runner := newBackupRunner(installation, true)
|
||||||
deps := restoreTestDependencies(t, runner)
|
deps := restoreTestDependencies(t, runner)
|
||||||
checkpointErr := errors.New("checkpoint unavailable")
|
checkpointErr := errors.New("checkpoint unavailable")
|
||||||
deps.checkpointLocked = func(context.Context, config.Installation, CreateRequest) (Result, error) {
|
deps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
|
||||||
return Result{}, checkpointErr
|
return Result{}, checkpointErr
|
||||||
}
|
}
|
||||||
restoredFiles := 0
|
restoredFiles := 0
|
||||||
@@ -591,7 +657,7 @@ func TestRestoreFileFailureRollsBackSecretAwareCheckpointBeforeCleanup(t *testin
|
|||||||
runner := newBackupRunner(installation, true)
|
runner := newBackupRunner(installation, true)
|
||||||
deps := restoreTestDependencies(t, runner)
|
deps := restoreTestDependencies(t, runner)
|
||||||
fileErr := errors.New("cannot restore operator configuration")
|
fileErr := errors.New("cannot restore operator configuration")
|
||||||
deps.checkpointLocked = func(context.Context, config.Installation, CreateRequest) (Result, error) {
|
deps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
|
||||||
return Result{Path: "/tmp/recovery.zip"}, nil
|
return Result{Path: "/tmp/recovery.zip"}, nil
|
||||||
}
|
}
|
||||||
var events []string
|
var events []string
|
||||||
@@ -1095,7 +1161,7 @@ func TestRestoreReleasesBarrierOnlyAfterVerifiedRecoveryFromLostResponse(t *test
|
|||||||
}
|
}
|
||||||
runner := &commandFailureRunner{fakeBackupRunner: backing, failures: []*commandFailure{&failure}}
|
runner := &commandFailureRunner{fakeBackupRunner: backing, failures: []*commandFailure{&failure}}
|
||||||
deps := restoreTestDependencies(t, runner)
|
deps := restoreTestDependencies(t, runner)
|
||||||
deps.checkpointLocked = func(context.Context, config.Installation, CreateRequest) (Result, error) {
|
deps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
|
||||||
return Result{Path: "/tmp/recovery.zip"}, nil
|
return Result{Path: "/tmp/recovery.zip"}, nil
|
||||||
}
|
}
|
||||||
deps.prepareRecovery = func(context.Context, config.Installation, string) (PreflightResult, error) {
|
deps.prepareRecovery = func(context.Context, config.Installation, string) (PreflightResult, error) {
|
||||||
@@ -1322,10 +1388,6 @@ func (runner *lifecycleGateRunner) SessionInventoryScope() string {
|
|||||||
return runner.fakeBackupRunner.SessionInventoryScope()
|
return runner.fakeBackupRunner.SessionInventoryScope()
|
||||||
}
|
}
|
||||||
|
|
||||||
type fakeRestoreLock struct {
|
|
||||||
release func()
|
|
||||||
}
|
|
||||||
|
|
||||||
type authenticationStateResetRunner struct {
|
type authenticationStateResetRunner struct {
|
||||||
args []string
|
args []string
|
||||||
result compose.Result
|
result compose.Result
|
||||||
@@ -1431,13 +1493,6 @@ func TestVerifyRestoreWorkspaceRejectsInvalidOperatorResults(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (lock fakeRestoreLock) Release() error {
|
|
||||||
if lock.release != nil {
|
|
||||||
lock.release()
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func equalStrings(got, want []string) bool {
|
func equalStrings(got, want []string) bool {
|
||||||
if len(got) != len(want) {
|
if len(got) != len(want) {
|
||||||
return false
|
return false
|
||||||
@@ -1456,18 +1511,18 @@ func restoreTestDependencies(t *testing.T, runner archiveRunner) restoreDependen
|
|||||||
preflight: func(ctx context.Context, installation config.Installation, request PreflightRequest) (PreflightResult, error) {
|
preflight: func(ctx context.Context, installation config.Installation, request PreflightRequest) (PreflightResult, error) {
|
||||||
return Preflight(ctx, installation, request, permissivePreflightDependencies())
|
return Preflight(ctx, installation, request, permissivePreflightDependencies())
|
||||||
},
|
},
|
||||||
checkpointLocked: func(context.Context, config.Installation, CreateRequest) (Result, error) {
|
checkpoint: func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
|
||||||
return Result{Path: "/tmp/default-checkpoint.zip"}, nil
|
return Result{Path: "/tmp/default-checkpoint.zip"}, nil
|
||||||
},
|
},
|
||||||
prepareRecovery: func(context.Context, config.Installation, string) (PreflightResult, error) {
|
prepareRecovery: func(context.Context, config.Installation, string) (PreflightResult, error) {
|
||||||
return PreflightResult{}, nil
|
return PreflightResult{}, nil
|
||||||
},
|
},
|
||||||
recover: func(context.Context, config.Installation, PreflightResult, bool) error { return nil },
|
recover: func(context.Context, config.Installation, PreflightResult, bool) error { return nil },
|
||||||
cleanupCheckpoint: func(string) error { return nil },
|
cleanupCheckpoint: func(string) error { return nil },
|
||||||
acquireLock: func(config.Installation) (restoreLock, error) { return fakeRestoreLock{}, nil },
|
acquireTransaction: lifecycle.AcquireTransaction,
|
||||||
runner: runner,
|
runner: runner,
|
||||||
sleep: func(time.Duration) {},
|
sleep: func(time.Duration) {},
|
||||||
restoreFile: func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return nil },
|
restoreFile: func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return nil },
|
||||||
restoreVolume: func(context.Context, config.Installation, VolumeMetadata, io.Reader) error {
|
restoreVolume: func(context.Context, config.Installation, VolumeMetadata, io.Reader) error {
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -16,8 +16,10 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
ErrLocked = errors.New("another lifecycle operation is already running for this installation")
|
ErrLocked = errors.New("another lifecycle operation is already running for this installation")
|
||||||
ErrOwnership = errors.New("lifecycle lock ownership changed; refusing to remove it")
|
ErrOwnership = errors.New("lifecycle lock ownership changed; refusing to remove it")
|
||||||
|
ErrTransactionInactive = errors.New("lifecycle transaction capability is not active")
|
||||||
|
ErrTransactionInstallation = errors.New("lifecycle transaction capability belongs to another installation")
|
||||||
)
|
)
|
||||||
|
|
||||||
const lockFileName = "lifecycle.lock.owner.json"
|
const lockFileName = "lifecycle.lock.owner.json"
|
||||||
@@ -36,6 +38,14 @@ type Lock struct {
|
|||||||
released bool
|
released bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Transaction is an opaque, installation-bound capability for work that must run while a
|
||||||
|
// lifecycle lock remains owned. Its fields are deliberately private so callers can obtain one
|
||||||
|
// only through AcquireTransaction.
|
||||||
|
type Transaction struct {
|
||||||
|
lock *Lock
|
||||||
|
controlDirectory string
|
||||||
|
}
|
||||||
|
|
||||||
// Acquire obtains the shared lock used by backup, restore, Pi lifecycle and product updates.
|
// Acquire obtains the shared lock used by backup, restore, Pi lifecycle and product updates.
|
||||||
func Acquire(installation config.Installation) (*Lock, error) {
|
func Acquire(installation config.Installation) (*Lock, error) {
|
||||||
directory := installation.ControlDirectory()
|
directory := installation.ControlDirectory()
|
||||||
@@ -76,6 +86,39 @@ func Acquire(installation config.Installation) (*Lock, error) {
|
|||||||
return &Lock{path: path, token: token}, nil
|
return &Lock{path: path, token: token}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AcquireTransaction obtains a lifecycle lock and returns the capability required by callers
|
||||||
|
// that perform nested work inside the same non-reentrant transaction.
|
||||||
|
func AcquireTransaction(installation config.Installation) (*Transaction, error) {
|
||||||
|
lock, err := Acquire(installation)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &Transaction{
|
||||||
|
lock: lock,
|
||||||
|
controlDirectory: filepath.Clean(installation.ControlDirectory()),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify refuses a nil, released, replaced, or foreign-installation capability before a nested
|
||||||
|
// lifecycle operation can begin.
|
||||||
|
func (transaction *Transaction) Verify(installation config.Installation) error {
|
||||||
|
if transaction == nil || transaction.lock == nil {
|
||||||
|
return ErrTransactionInactive
|
||||||
|
}
|
||||||
|
if transaction.controlDirectory != filepath.Clean(installation.ControlDirectory()) {
|
||||||
|
return ErrTransactionInstallation
|
||||||
|
}
|
||||||
|
return transaction.lock.verifyHeld()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Release relinquishes the lifecycle lock associated with this transaction capability.
|
||||||
|
func (transaction *Transaction) Release() error {
|
||||||
|
if transaction == nil || transaction.lock == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return transaction.lock.Release()
|
||||||
|
}
|
||||||
|
|
||||||
// Path returns the installation-private owner-file path for diagnostics and tests.
|
// Path returns the installation-private owner-file path for diagnostics and tests.
|
||||||
func (lock *Lock) Path() string {
|
func (lock *Lock) Path() string {
|
||||||
if lock == nil {
|
if lock == nil {
|
||||||
@@ -111,3 +154,26 @@ func (lock *Lock) Release() error {
|
|||||||
lock.released = true
|
lock.released = true
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (lock *Lock) verifyHeld() error {
|
||||||
|
if lock == nil {
|
||||||
|
return ErrTransactionInactive
|
||||||
|
}
|
||||||
|
lock.mu.Lock()
|
||||||
|
defer lock.mu.Unlock()
|
||||||
|
if lock.released {
|
||||||
|
return ErrTransactionInactive
|
||||||
|
}
|
||||||
|
contents, err := os.ReadFile(lock.path)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return ErrOwnership
|
||||||
|
}
|
||||||
|
return fmt.Errorf("read lifecycle lock owner: %w", err)
|
||||||
|
}
|
||||||
|
var current owner
|
||||||
|
if json.Unmarshal(contents, ¤t) != nil || current.Token == "" || current.Token != lock.token {
|
||||||
|
return ErrOwnership
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -48,3 +48,27 @@ func TestLifecycleLockReleaseDoesNotRemoveAnotherOwnersFile(t *testing.T) {
|
|||||||
t.Fatalf("foreign lock was removed: %v", err)
|
t.Fatalf("foreign lock was removed: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTransactionCapabilityIsInstallationBoundAndExpiresOnRelease(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
installation := config.Installation{ProjectDirectory: root, Path: filepath.Join(root, "thothii-installation.yaml")}
|
||||||
|
transaction, err := AcquireTransaction(installation)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := transaction.Verify(installation); err != nil {
|
||||||
|
t.Fatalf("Verify() active capability error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
otherRoot := t.TempDir()
|
||||||
|
other := config.Installation{ProjectDirectory: otherRoot, Path: filepath.Join(otherRoot, "thothii-installation.yaml")}
|
||||||
|
if err := transaction.Verify(other); !errors.Is(err, ErrTransactionInstallation) {
|
||||||
|
t.Fatalf("Verify() for another installation error = %v, want ErrTransactionInstallation", err)
|
||||||
|
}
|
||||||
|
if err := transaction.Release(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := transaction.Verify(installation); !errors.Is(err, ErrTransactionInactive) {
|
||||||
|
t.Fatalf("Verify() after Release() error = %v, want ErrTransactionInactive", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user