From 8a3fa5031dd7f6f5c2a54dfdd693ccf1082a8577 Mon Sep 17 00:00:00 2001 From: mptyl Date: Tue, 18 Aug 2026 06:02:25 +0200 Subject: [PATCH] test(auth): gate local and OIDC authentication release --- .github/workflows/deployment.yml | 32 ++ PROJECT_STATE.md | 42 +- backend/test/fixtures/oidc-provider.mjs | 328 +++++++++++ .../authentication-manual-acceptance.md | 28 +- frontend/e2e/auth.spec.ts | 155 ++++++ frontend/e2e/f1.spec.ts | 41 +- frontend/e2e/fixtures/auth-stack.mjs | 521 ++++++++++++++++++ frontend/playwright.config.ts | 38 +- harness/tests/test_local_compose_contract.py | 5 +- scripts/authentication-smoke.sh | 56 ++ scripts/test-compose-provider-readiness.sh | 46 ++ scripts/test-pi-user-auth-compose.sh | 9 +- scripts/test-preprocess-compose-config.sh | 7 +- scripts/test-server-pi-state-topology.sh | 6 + tools/tht/internal/backup/create.go | 23 +- tools/tht/internal/backup/create_test.go | 27 + tools/tht/internal/backup/preflight.go | 125 ++++- tools/tht/internal/backup/preflight_test.go | 74 +++ tools/tht/internal/backup/restore.go | 47 +- tools/tht/internal/backup/restore_host.go | 28 +- tools/tht/internal/backup/restore_test.go | 147 +++-- tools/tht/internal/lifecycle/lock.go | 70 ++- tools/tht/internal/lifecycle/lock_test.go | 24 + 23 files changed, 1711 insertions(+), 168 deletions(-) create mode 100644 backend/test/fixtures/oidc-provider.mjs create mode 100644 frontend/e2e/auth.spec.ts create mode 100644 frontend/e2e/fixtures/auth-stack.mjs create mode 100755 scripts/authentication-smoke.sh diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 80abaf9e..b94c7600 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -74,6 +74,38 @@ jobs: npx vitest run npx tsc -b + authentication-browser: + name: Hermetic authentication browser gate + needs: deterministic + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - name: Check out source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "24.16.0" + package-manager-cache: false + - name: Set up Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version: "1.26.5" + cache-dependency-path: tools/tht/go.sum + - name: Install backend dependencies + working-directory: backend + run: npm ci + - name: Install frontend dependencies + working-directory: frontend + run: npm ci + - name: Install Chromium for Playwright + working-directory: frontend + run: npx playwright install --with-deps chromium + - name: Run authentication browser smoke + run: bash scripts/authentication-smoke.sh + linux-docker: name: Linux Docker deployment and rollback runs-on: ubuntu-24.04 diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index 97ce63c0..89bc3b53 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -7,26 +7,34 @@ > ThothII per il repository (app + CLI `thothctl`), (3) come usare l'applicazione ThothII di base > (sessioni, domande, gate). Il documento userà parole semplici ed esempi; i dettagli tecnici > resteranno nei contratti esistenti. Esempio pratico completo: Policlinico San Donato. -> Last updated: 2026-08-18 (Task 14 documentation implemented; strict documentation and -> authentication release gates remain blocking or pending as listed below). +> Last updated: 2026-08-18 (Task 15 automated evidence recorded; the authentication feature is +> not complete or release-accepted while the required FAIL/PENDING gates listed below remain). > Point a fresh session here ("read PROJECT_STATE.md") before substantial work. -### Task 14 authentication documentation — implementation status (2026-08-18) +### Task 15 authentication release — automated evidence, release incomplete (2026-08-18) -- Documentation now describes local Argon2id users, ordinary and remembered session expiry, - revision invalidation, generic OIDC direct groups claims, exact group-role mapping, Authentik - group-view-only catalog checks, tht auth/tht doctor ordering, diagnostics, CSRF, and restore - reauthentication. -- The Task 14 authentication-documentation smoke passes. The MkDocs `--strict` baseline remains a - blocking release gate: it exits 1 with 69 warnings, the same warning count before and after Task - 14. It is not a PASS or release evidence. -- Browser OIDC callback E2E, native Windows behavioral execution, PSD/manual test identities, and - external L2 remain pending Task 15/release gates. Task 14 documentation is implemented, not - release-accepted. -- Task 13 has two parked restore-lock preconditions that remain mandatory before certification: - lock before target-dependent preflight with archive bytes/hashes staged and revalidated inside the - lock immediately before extraction; and an opaque installation-bound transaction capability or - closure replacing convention-only lock-held helpers. +- Task 13 carry-ins are closed with focused TDD: restore acquires the lifecycle lock before + target-dependent preflight, stages/revalidates archive bytes and hashes under that lock, and uses + an opaque installation-bound transaction capability. Focused mutation/concurrency/lock-leak + tests and the full Go race/build gate pass. +- PASS on the pinned Node `v24.16.0`: backend 75 files / 1081 tests plus typecheck/build; frontend + 61 files / 444 tests plus typecheck/build and 6 Playwright tests; hermetic loopback OIDC browser + smoke; harness 921 passed / 4 L2 deselected; authentication-documentation smoke; unified Docker + smoke with scoped cleanup; installer shell test; relevant Compose/security contracts. The default + host Node is `v25.6.1`; it is not the release contract and no tracked `v24.19.0` pin exists. +- Task15 fixture regressions in preprocess, server Pi-state, and Pi-auth Compose contracts were + repaired and their focused checks pass. Static Windows amd64 cross-build and test compilation + pass for 18 Go packages. +- FAIL baseline evidence: Ruff reports 192 errors; MkDocs `--strict` exits 1 with 69 warnings; + the canonical-install and workspace-install-doc checks report existing manual wording mismatches; + the Pi user-auth Compose check reaches an existing model allow-list mismatch; and the broad + deployment-coupling scanner sees preserved ignored private deployment material. +- PENDING evidence: native Windows execution (no KVM, `sshpass`, Windows container, or PowerShell); + real PSD/manual acceptance (no real test identity/access); L2 (no configured local secret layout); + and isolated provider-readiness Docker smoke (a host port is already occupied by an unrelated + resource). These are not PASS claims. +- **Release state: NOT COMPLETE.** Do not mark authentication release-complete until every required + gate is rerun in an eligible environment and is PASS. ### P3 effective configuration and `.tht-dwh` — implementation complete, automated PASS, manual PASS (2026-08-13) diff --git a/backend/test/fixtures/oidc-provider.mjs b/backend/test/fixtures/oidc-provider.mjs new file mode 100644 index 00000000..6729e000 --- /dev/null +++ b/backend/test/fixtures/oidc-provider.mjs @@ -0,0 +1,328 @@ +#!/usr/bin/env node +/** + * Hermetic loopback OIDC/AuthentiK-shaped provider for browser smoke tests. + * + * It deliberately has no network dependency and binds only to 127.0.0.1. Its + * ephemeral TLS and signing keys are test-scoped; callers receive the CA path + * needed to trust the provider from a spawned backend process. + */ +import { spawnSync } from "node:child_process"; +import { createHash, generateKeyPairSync, randomBytes, sign as signRsa } from "node:crypto"; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { createServer } from "node:https"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const LOOPBACK_HOST = "127.0.0.1"; +const ISSUER_PATH = "/application/o/thothii"; +const MAX_BODY_BYTES = 32 * 1024; +const VALID_IDENTITIES = new Set([ + "ordinary", + "admin", + "missing-groups", + "malformed-groups", + "unmapped", + "expired", +]); + +const identityClaims = Object.freeze({ + ordinary: { subject: "fixture-ordinary", displayName: "Fixture ordinary", groups: ["fixture-users"] }, + admin: { subject: "fixture-admin", displayName: "Fixture administrator", groups: ["fixture-admin"] }, + "missing-groups": { subject: "fixture-missing-groups", displayName: "Fixture missing groups" }, + "malformed-groups": { subject: "fixture-malformed-groups", displayName: "Fixture malformed groups", groups: ["fixture-users", 7] }, + unmapped: { subject: "fixture-unmapped", displayName: "Fixture unmapped", groups: ["fixture-unmapped"] }, + expired: { subject: "fixture-expired", displayName: "Fixture expired", groups: ["fixture-users"], expired: true }, +}); + +function safeError(code) { + return new Error(code); +} + +function ensurePrivateDirectory(directory) { + mkdirSync(directory, { recursive: true, mode: 0o700 }); + chmodSync(directory, 0o700); +} + +function createCertificate(directory) { + const keyFile = join(directory, "provider-key.pem"); + const certificateFile = join(directory, "provider-ca.pem"); + const result = spawnSync("openssl", [ + "req", "-x509", "-newkey", "rsa:2048", "-sha256", "-nodes", + "-keyout", keyFile, + "-out", certificateFile, + "-subj", "/CN=127.0.0.1", + "-addext", "subjectAltName=IP:127.0.0.1", + "-days", "1", + ], { stdio: "ignore" }); + if (result.status !== 0) throw safeError("oidc_fixture_certificate_generation_failed"); + chmodSync(keyFile, 0o600); + chmodSync(certificateFile, 0o600); + return { key: readFileSync(keyFile), cert: readFileSync(certificateFile), certificateFile }; +} + +function sendJson(reply, status, value) { + reply.writeHead(status, { + "cache-control": "no-store", + "content-type": "application/json; charset=utf-8", + }); + reply.end(JSON.stringify(value)); +} + +function redirect(reply, location) { + reply.writeHead(302, { "cache-control": "no-store", location }); + reply.end(); +} + +async function requestBody(request) { + let size = 0; + const chunks = []; + for await (const chunk of request) { + size += chunk.length; + if (size > MAX_BODY_BYTES) throw safeError("oidc_fixture_request_too_large"); + chunks.push(chunk); + } + return Buffer.concat(chunks).toString("utf8"); +} + +function jwt(privateKey, issuer, audience, nonce, identity) { + const claims = identityClaims[identity]; + const now = Math.floor(Date.now() / 1_000); + const payload = { + iss: issuer, + sub: claims.subject, + aud: audience, + exp: now + (claims.expired ? -30 : 60), + iat: now - 1, + nonce, + name: claims.displayName, + ...(Object.hasOwn(claims, "groups") ? { groups: claims.groups } : {}), + }; + const header = { alg: "RS256", typ: "JWT", kid: "fixture-rs256" }; + const protectedPart = Buffer.from(JSON.stringify(header)).toString("base64url"); + const payloadPart = Buffer.from(JSON.stringify(payload)).toString("base64url"); + const signingInput = `${protectedPart}.${payloadPart}`; + const signature = signRsa("RSA-SHA256", Buffer.from(signingInput), privateKey).toString("base64url"); + return `${signingInput}.${signature}`; +} + +function authorizationIdentity(identity) { + if (!VALID_IDENTITIES.has(identity)) throw safeError("oidc_fixture_identity_invalid"); + return identity; +} + +/** + * Start an HTTPS test provider. The returned telemetry intentionally excludes + * transient authorization codes, browser state, nonces, and token material. + */ +export async function startFakeOidcProvider(options = {}) { + const host = options.host ?? LOOPBACK_HOST; + if (host !== LOOPBACK_HOST) throw safeError("oidc_fixture_loopback_required"); + const ownsDirectory = options.directory === undefined; + const directory = options.directory ?? mkdtempSync(join(tmpdir(), "thothii-oidc-fixture-")); + ensurePrivateDirectory(directory); + const certificate = createCertificate(directory); + const { privateKey, publicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }); + const publicJwk = publicKey.export({ format: "jwk" }); + const jwks = { + keys: [{ ...publicJwk, alg: "RS256", kid: "fixture-rs256", use: "sig" }], + }; + const authorizations = new Map(); + const deviceCodes = new Map(); + let activeIdentity = authorizationIdentity(options.identity ?? "ordinary"); + let lastAuthorization = undefined; + let issuer = undefined; + let baseUrl = undefined; + + const server = createServer({ key: certificate.key, cert: certificate.cert }, async (request, reply) => { + try { + if (!issuer || !baseUrl || !request.url) { + sendJson(reply, 503, { error: "temporarily_unavailable" }); + return; + } + const url = new URL(request.url, baseUrl); + const path = url.pathname; + const discoveryPath = `${ISSUER_PATH}/.well-known/openid-configuration`; + const rfc8414Path = `/.well-known/openid-configuration${ISSUER_PATH}`; + if (request.method === "GET" && (path === discoveryPath || path === rfc8414Path)) { + sendJson(reply, 200, { + issuer, + authorization_endpoint: `${issuer}authorize`, + token_endpoint: `${issuer}token`, + jwks_uri: `${issuer}jwks`, + device_authorization_endpoint: `${issuer}device_authorization`, + response_types_supported: ["code"], + subject_types_supported: ["public"], + grant_types_supported: ["authorization_code", "urn:ietf:params:oauth:grant-type:device_code"], + token_endpoint_auth_methods_supported: ["client_secret_basic", "client_secret_post"], + code_challenge_methods_supported: ["S256"], + id_token_signing_alg_values_supported: ["RS256"], + }); + return; + } + if (request.method === "GET" && path === `${ISSUER_PATH}/jwks`) { + sendJson(reply, 200, jwks); + return; + } + if (request.method === "GET" && path === `${ISSUER_PATH}/authorize`) { + const redirectUri = url.searchParams.get("redirect_uri"); + const clientId = url.searchParams.get("client_id"); + const state = url.searchParams.get("state"); + const nonce = url.searchParams.get("nonce"); + const challenge = url.searchParams.get("code_challenge"); + if (url.searchParams.get("response_type") !== "code" || !redirectUri || !clientId || !state || !nonce + || !challenge || url.searchParams.get("code_challenge_method") !== "S256") { + sendJson(reply, 400, { error: "invalid_request" }); + return; + } + let callback; + try { + callback = new URL(redirectUri); + if (callback.protocol !== "http:" || callback.hostname !== LOOPBACK_HOST || callback.username || callback.password) { + throw safeError("oidc_fixture_redirect_invalid"); + } + } catch { + sendJson(reply, 400, { error: "invalid_request" }); + return; + } + const code = randomBytes(32).toString("base64url"); + authorizations.set(code, { + challenge, + clientId, + identity: activeIdentity, + nonce, + used: false, + }); + lastAuthorization = { identity: activeIdentity, codeChallengeMethod: "S256", pkceVerified: false }; + callback.searchParams.set("code", code); + callback.searchParams.set("state", state); + redirect(reply, callback.href); + return; + } + if (request.method === "POST" && path === `${ISSUER_PATH}/device_authorization`) { + const values = new URLSearchParams(await requestBody(request)); + const clientId = values.get("client_id"); + if (!clientId) { + sendJson(reply, 400, { error: "invalid_request" }); + return; + } + const deviceCode = randomBytes(32).toString("base64url"); + const userCode = "FIXTURE-CODE"; + deviceCodes.set(deviceCode, { clientId, identity: activeIdentity, nonce: "device", used: false }); + sendJson(reply, 200, { + device_code: deviceCode, + user_code: userCode, + verification_uri: `${issuer}device`, + verification_uri_complete: `${issuer}device?user_code=${userCode}`, + expires_in: 60, + interval: 1, + }); + return; + } + if (request.method === "POST" && path === `${ISSUER_PATH}/token`) { + const values = new URLSearchParams(await requestBody(request)); + const grantType = values.get("grant_type"); + let record; + if (grantType === "authorization_code") { + const code = values.get("code") ?? ""; + record = authorizations.get(code); + const verifier = values.get("code_verifier") ?? ""; + const verified = record !== undefined && !record.used + && createHash("sha256").update(verifier).digest("base64url") === record.challenge; + if (!verified) { + sendJson(reply, 400, { error: "invalid_grant" }); + return; + } + record.used = true; + if (lastAuthorization) lastAuthorization = { ...lastAuthorization, pkceVerified: true }; + } else if (grantType === "urn:ietf:params:oauth:grant-type:device_code") { + const deviceCode = values.get("device_code") ?? ""; + record = deviceCodes.get(deviceCode); + if (record === undefined || record.used) { + sendJson(reply, 400, { error: "invalid_grant" }); + return; + } + record.used = true; + } else { + sendJson(reply, 400, { error: "unsupported_grant_type" }); + return; + } + sendJson(reply, 200, { + access_token: randomBytes(32).toString("base64url"), + token_type: "Bearer", + expires_in: 60, + id_token: jwt(privateKey, issuer, record.clientId, record.nonce, record.identity), + }); + return; + } + if (request.method === "GET" && path === "/api/v3/core/groups/") { + if (!request.headers.authorization?.startsWith("Bearer ")) { + sendJson(reply, 401, { detail: "authentication required" }); + return; + } + const name = url.searchParams.get("name") ?? ""; + const present = name === "fixture-users" || name === "fixture-admin"; + sendJson(reply, 200, { + pagination: { next: null }, + results: present ? [{ name }] : [], + }); + return; + } + sendJson(reply, 404, { error: "not_found" }); + } catch { + if (!reply.headersSent) sendJson(reply, 400, { error: "invalid_request" }); + else reply.end(); + } + }); + + try { + await new Promise((resolveListen, rejectListen) => { + const onError = (error) => rejectListen(error); + server.once("error", onError); + server.listen({ host, port: options.port ?? 0 }, () => { + server.off("error", onError); + resolveListen(); + }); + }); + } catch (error) { + server.close(); + if (ownsDirectory) rmSync(directory, { recursive: true, force: true }); + throw error; + } + const address = server.address(); + if (!address || typeof address === "string") { + await new Promise((resolveClose) => server.close(resolveClose)); + if (ownsDirectory) rmSync(directory, { recursive: true, force: true }); + throw safeError("oidc_fixture_listen_failed"); + } + baseUrl = `https://${LOOPBACK_HOST}:${address.port}`; + issuer = `${baseUrl}${ISSUER_PATH}/`; + + return { + baseUrl, + issuer, + caFile: certificate.certificateFile, + setIdentity(identity) { + activeIdentity = authorizationIdentity(identity); + }, + lastAuthorization() { + return lastAuthorization === undefined ? undefined : { ...lastAuthorization }; + }, + async close() { + await new Promise((resolveClose) => server.close(resolveClose)); + if (ownsDirectory) rmSync(directory, { recursive: true, force: true }); + }, + }; +} + +const currentFile = fileURLToPath(import.meta.url); +if (process.argv[1] && resolve(process.argv[1]) === currentFile) { + const provider = await startFakeOidcProvider(); + process.stdout.write('{"status":"ready"}\n'); + const close = async () => { + await provider.close(); + process.exit(0); + }; + process.once("SIGINT", () => { void close(); }); + process.once("SIGTERM", () => { void close(); }); +} diff --git a/docs/testing/authentication-manual-acceptance.md b/docs/testing/authentication-manual-acceptance.md index 84931ac8..308fef23 100644 --- a/docs/testing/authentication-manual-acceptance.md +++ b/docs/testing/authentication-manual-acceptance.md @@ -4,14 +4,18 @@ This is a release-gate checklist, not evidence. Use one ordinary PSD test identi PSD test identity supplied through the approved test-identity process. Record only sanitized pass/fail results, timestamps, build identity, and diagnostic codes. Do not record names, internal URLs, directory/LDAP details, tokens, passwords, hashes, cookies, or realistic secret examples. +Keep the retained result under `.artifacts/manual-acceptance/authentication//` with a +sanitized digest. Do not retain raw browser traces, Compose environments, provider exports, or +unbounded logs. If the approved identities or access are unavailable, record **PENDING** rather +than inferring a PASS. ## Preconditions and ordering -1. Resolve the two parked Task 13 restore preconditions before certification: acquire the lifecycle - lock before any target-dependent preflight and stage/revalidate archive bytes and hashes inside - that lock immediately before extraction; replace convention-only - `createWithDependenciesLockHeld` with an opaque installation-bound transaction capability or - closure so lock-held primitives cannot be called without the capability. +1. Confirm retained Task 13 evidence for the restore prerequisites before certification: the + lifecycle lock is acquired before target-dependent preflight, archive bytes and hashes are + staged/revalidated inside that lock immediately before extraction, and checkpointing requires + an opaque installation-bound transaction capability. Manual acceptance never substitutes for + those automated concurrency and mutation tests. 2. Run Workspace Validate first; it is the static authentication gate. Run `tht auth check` for live non-interactive diagnosis, then `tht auth check --interactive` where Device Authorization is available, then Workspace Test for aggregate live validation. @@ -42,9 +46,13 @@ URLs, directory/LDAP details, tokens, passwords, hashes, cookies, or realistic s | Provider outage | Live check reports `oidc_discovery_unreachable`; browser login fails closed without exposing credentials. | | Restore is completed | Sessions and OIDC state are absent; all users must reauthenticate. | -## Status at Task 14 +## Status at Task 15 -Documentation and deterministic contract checks are the Task 14 scope. Browser OIDC callback E2E, -native Windows behavioral execution, the two parked restore-lock preconditions above, PSD/manual -identities, and any external L2 execution remain pending Task 15/release gates. Do not mark this -matrix PASS until those gates have actual retained evidence. +The hermetic browser suite now covers the loopback provider discovery/JWKS/device/group-list +surface and the complete OIDC Authorization Code + PKCE callback, including direct `groups` +fail-closed cases. It also covers local ordinary, remembered/restart, logout, and administrator +flows. This deterministic evidence does not replace the manual PSD/AuthentiK acceptance. + +Native Windows behavioral execution, approved PSD/AuthentiK identities and access, interactive +device acceptance, and external L2 remain **PENDING** until actual retained evidence exists. Do +not mark the feature or this matrix release-complete while any required gate remains pending. diff --git a/frontend/e2e/auth.spec.ts b/frontend/e2e/auth.spec.ts new file mode 100644 index 00000000..8cd8889d --- /dev/null +++ b/frontend/e2e/auth.spec.ts @@ -0,0 +1,155 @@ +import { expect, test, type Page } from "@playwright/test"; +import { createAuthenticationStack } from "./fixtures/auth-stack.mjs"; + +test.describe.configure({ mode: "serial" }); +// The only HTTPS navigation in this file is the test-scoped loopback provider. +test.use({ ignoreHTTPSErrors: true }); + +let stack: Awaited>; + +test.beforeAll(async () => { + stack = await createAuthenticationStack(); +}); + +test.afterAll(async () => { + await stack?.close(); +}); + +test("the loopback fixture exposes signed OIDC discovery, device authorization, and AuthentiK group lookup", async () => { + await expect(stack.providerSurface()).resolves.toEqual({ + discovery: true, + jwks: true, + deviceAuthorization: true, + deviceToken: true, + groupList: true, + }); +}); + +async function expectShell(page: Page): Promise { + await expect(page.getByTestId("app-shell")).toBeVisible({ timeout: 30_000 }); +} + +async function signInLocally(page: Page, account: "ordinary" | "admin", remember = false): Promise { + await page.getByLabel("Username").fill(stack.localAccount(account).username); + await page.getByLabel("Password").fill(stack.localAccount(account).password); + const rememberControl = page.getByRole("checkbox", { name: /remember me/i }); + if (remember) await rememberControl.check(); + await page.getByRole("button", { name: "Sign in", exact: true }).click(); + await expectShell(page); +} + +async function browserSession(page: Page): Promise<{ status: number; body: Record }> { + return page.evaluate(async () => { + const response = await fetch("/api/me", { credentials: "same-origin" }); + return { status: response.status, body: await response.json() as Record }; + }); +} + +async function expectNoWebStorageTokens(page: Page): Promise { + const entries = await page.evaluate(() => { + const values = (storage: Storage) => Array.from({ length: storage.length }, (_unused, index) => { + const key = storage.key(index) ?? ""; + return [key, storage.getItem(key) ?? ""]; + }); + return [...values(localStorage), ...values(sessionStorage)]; + }); + expect(entries.filter(([key, value]) => /(?:access|refresh|id)?[_-]?token|bearer|jwt/i.test(`${key}\n${value}`))).toEqual([]); +} + +async function signInWithOidc(page: Page): Promise { + await page.getByRole("button", { name: /continue with single sign-on/i }).click(); +} + +async function expectOidcCallbackDenied(page: Page): Promise { + await expect(page.locator("body")).toContainText("OIDC sign-in could not be completed", { timeout: 30_000 }); + await expect(page.getByTestId("app-shell")).toHaveCount(0); + await expectNoWebStorageTokens(page); +} + +test("local ordinary and remembered sessions survive restart, logout, and keep tokens out of Web Storage", async ({ page }) => { + await stack.useLocalMode(); + await page.goto(stack.publicUrl); + await expect(page.getByRole("heading", { name: "Sign in to ThothII" })).toBeVisible(); + + await signInLocally(page, "ordinary"); + expect((await browserSession(page)).body.roles).toEqual(["user"]); + await expectNoWebStorageTokens(page); + + await stack.restartBackend(); + await page.reload(); + await expectShell(page); + + await page.getByRole("button", { name: "Log out", exact: true }).click(); + await expect(page.getByRole("heading", { name: "Sign in to ThothII" })).toBeVisible(); + + await signInLocally(page, "ordinary", true); + const remembered = (await page.context().cookies(stack.publicUrl)).find((cookie) => cookie.name === "thothii_session"); + expect(remembered?.httpOnly).toBe(true); + expect(remembered?.expires ?? -1).toBeGreaterThan(Date.now() / 1_000); + + await stack.restartBackend(); + await page.reload(); + await expectShell(page); + await expectNoWebStorageTokens(page); + + await page.getByRole("button", { name: "Log out", exact: true }).click(); + await expect(page.getByRole("heading", { name: "Sign in to ThothII" })).toBeVisible(); + expect((await page.context().cookies(stack.publicUrl)).some((cookie) => cookie.name === "thothii_session")).toBe(false); +}); + +test("local administrator receives the administrator role", async ({ page }) => { + await stack.useLocalMode(); + await page.goto(stack.publicUrl); + await signInLocally(page, "admin"); + expect((await browserSession(page)).body.roles).toEqual(["admin"]); + await expectNoWebStorageTokens(page); +}); + +test("OIDC Authorization Code plus PKCE redirects back and maps ordinary and administrator groups", async ({ page }) => { + await stack.useOidcMode("ordinary"); + await page.goto(stack.publicUrl); + await signInWithOidc(page); + await expectShell(page); + expect((await browserSession(page)).body.roles).toEqual(["user"]); + expect(stack.lastAuthorization()).toMatchObject({ codeChallengeMethod: "S256", pkceVerified: true }); + await expectNoWebStorageTokens(page); + + await page.getByRole("button", { name: "Log out", exact: true }).click(); + await expect(page.getByRole("heading", { name: "Sign in to ThothII" })).toBeVisible(); + stack.setOidcIdentity("admin"); + await signInWithOidc(page); + await expectShell(page); + expect((await browserSession(page)).body.roles).toEqual(["admin"]); + await expectNoWebStorageTokens(page); +}); + +test("OIDC unmapped, missing, and malformed groups fail closed; an expired token can recover", async ({ page }) => { + await stack.useOidcMode("unmapped"); + await page.goto(stack.publicUrl); + await signInWithOidc(page); + await expect(page.getByRole("heading", { name: "Access not permitted" })).toBeVisible({ timeout: 30_000 }); + await expectNoWebStorageTokens(page); + + await page.context().clearCookies(); + stack.setOidcIdentity("missing-groups"); + await page.goto(stack.publicUrl); + await signInWithOidc(page); + await expectOidcCallbackDenied(page); + + stack.setOidcIdentity("malformed-groups"); + await page.goto(stack.publicUrl); + await signInWithOidc(page); + await expectOidcCallbackDenied(page); + + stack.setOidcIdentity("expired"); + await page.goto(stack.publicUrl); + await signInWithOidc(page); + await expectOidcCallbackDenied(page); + + stack.setOidcIdentity("ordinary"); + await page.goto(stack.publicUrl); + await signInWithOidc(page); + await expectShell(page); + expect((await browserSession(page)).body.roles).toEqual(["user"]); + await expectNoWebStorageTokens(page); +}); diff --git a/frontend/e2e/f1.spec.ts b/frontend/e2e/f1.spec.ts index 16d0dedf..919f14e8 100644 --- a/frontend/e2e/f1.spec.ts +++ b/frontend/e2e/f1.spec.ts @@ -1,4 +1,5 @@ -import { test, expect } from "@playwright/test"; +import { expect, test, type Page } from "@playwright/test"; +import { createAuthenticationStack } from "./fixtures/auth-stack.mjs"; /** * E2E F1 loop — hermetic (no VPN, no real Pi, no real Python). @@ -9,20 +10,42 @@ import { test, expect } from "@playwright/test"; * which emits the f1_disambiguation.json scenario (select widget with * "interpretazione A" / "interpretazione B"). * - * Flow: open app → open NewSessionDialog → fill question → submit (Crea) + * Flow: authenticate locally → focus the new-session composer → fill question → submit (Send) * → wait for F1 select widget → click an option → assert no error shown. */ -test("F1 loop: new question → F1 widget → respond", async ({ page }) => { - await page.goto("/"); +test.describe.configure({ mode: "serial" }); - // Open the new-session dialog. - await page.getByRole("button", { name: /nuova/i }).click(); +let stack: Awaited>; + +test.beforeAll(async () => { + stack = await createAuthenticationStack({ withF1Workspace: true }); + await stack.useLocalMode(); +}); + +test.afterAll(async () => { + await stack?.close(); +}); + +async function signInLocally(page: Page): Promise { + const account = stack.localAccount("ordinary"); + await page.getByLabel("Username").fill(account.username); + await page.getByLabel("Password").fill(account.password); + await page.getByRole("button", { name: "Sign in", exact: true }).click(); + await expect(page.getByTestId("app-shell")).toBeVisible({ timeout: 30_000 }); +} + +test("F1 loop: new question → F1 widget → respond", async ({ page }) => { + await page.goto(stack.publicUrl); + await signInLocally(page); + + // Focus the composer for a new session. + await page.getByRole("button", { name: "New session", exact: true }).click(); // Fill in the question. - await page.getByLabel(/domanda/i).fill("quante cardioversioni nel 2024"); + await page.getByLabel("New question").fill("quante cardioversioni nel 2024"); // Submit — the backend creates the session (fake-tht) and spawns Pi (fake-pi). - await page.getByRole("button", { name: /^crea$/i }).click(); + await page.getByRole("button", { name: "Send", exact: true }).click(); // Wait for the F1 disambiguation select widget to appear. // The fake-pi emits extension_ui_request → backend bridges to SSE → frontend @@ -41,5 +64,5 @@ test("F1 loop: new question → F1 widget → respond", async ({ page }) => { await expect( page.getByRole("button", { name: /interpretazione A/i }), ).not.toBeVisible({ timeout: 10_000 }); - await expect(page.locator("body")).not.toContainText(/errore/i); + await expect(page.locator("body")).not.toContainText(/error/i); }); diff --git a/frontend/e2e/fixtures/auth-stack.mjs b/frontend/e2e/fixtures/auth-stack.mjs new file mode 100644 index 00000000..9fd1fc20 --- /dev/null +++ b/frontend/e2e/fixtures/auth-stack.mjs @@ -0,0 +1,521 @@ +import { spawn } from "node:child_process"; +import { argon2 } from "node:crypto"; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { createServer as createHttpServer } from "node:http"; +import { request as httpsRequest } from "node:https"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { startFakeOidcProvider } from "../../../backend/test/fixtures/oidc-provider.mjs"; + +const __dir = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(__dir, "../../.."); +const frontendRoot = join(repositoryRoot, "frontend"); +const backendRoot = join(repositoryRoot, "backend"); +const harnessRoot = join(repositoryRoot, "harness"); +const thtRoot = join(repositoryRoot, "tools", "tht"); +const fakePi = join(frontendRoot, "e2e", "fixtures", "fake-pi.mjs"); +const fakeTht = join(frontendRoot, "e2e", "fixtures", "fake-tht.mjs"); + +function safeError(code) { + return new Error(code); +} + +function buildAuthenticationStorageBridge(output) { + const result = spawn("go", ["build", "-o", output, "./cmd/tht"], { + cwd: thtRoot, + stdio: "ignore", + }); + return new Promise((resolveBuild, rejectBuild) => { + result.once("error", () => rejectBuild(safeError("e2e_auth_storage_build_failed"))); + result.once("exit", (code) => code === 0 ? resolveBuild() : rejectBuild(safeError("e2e_auth_storage_build_failed"))); + }); +} + +function secureDirectory(path) { + mkdirSync(path, { recursive: true, mode: 0o700 }); + chmodSync(path, 0o700); +} + +function runFixtureCommand(command, args, cwd) { + const child = spawn(command, args, { cwd, stdio: "ignore" }); + return new Promise((resolveCommand, rejectCommand) => { + child.once("error", () => rejectCommand(safeError("e2e_workspace_fixture_command_failed"))); + child.once("exit", (code) => code === 0 + ? resolveCommand() + : rejectCommand(safeError("e2e_workspace_fixture_command_failed"))); + }); +} + +const F1_WORKSPACE_ID = "fixture-workspace"; +const F1_WORKSPACE_DESCRIPTOR = `workspace: + schema_version: 3 + id: fixture-workspace + name: Fixture workspace + language: en +dwh: + engine: postgres + database: fixture + schema: fixture + supported_transports: [postgres_direct] +semantic_index: + vector_store: + engine: qdrant + collection: fixture-workspace + dimensions: 1024 + distance: cosine + embedding: + provider: ollama_internal + model: qwen3-embedding:0.6b + dimensions: 1024 +llm_policy: + allowed: [zai/glm-5.2] + default: zai/glm-5.2 +`; + +async function prepareF1Workspace(root) { + const source = join(root, "workspace-source"); + const remote = join(root, "workspace-remote.git"); + secureDirectory(source); + secureDirectory(join(source, F1_WORKSPACE_ID)); + writeSecure(join(source, "thoth-workspaces.yaml"), [ + "schema_version: 1", + "workspaces:", + ` - id: ${F1_WORKSPACE_ID}`, + " name: Fixture workspace", + "", + ].join("\n")); + writeSecure(join(source, F1_WORKSPACE_ID, "workspace.yaml"), F1_WORKSPACE_DESCRIPTOR); + + await runFixtureCommand("git", ["init", "--bare", "--initial-branch=main", remote], root); + chmodSync(remote, 0o700); + await runFixtureCommand("git", ["init", "--initial-branch=main"], source); + await runFixtureCommand("git", ["config", "user.name", "ThothII E2E Fixture"], source); + await runFixtureCommand("git", ["config", "user.email", "thothii-e2e@example.invalid"], source); + await runFixtureCommand("git", ["add", "-A"], source); + await runFixtureCommand("git", ["commit", "-m", "Create deterministic fixture workspace"], source); + await runFixtureCommand("git", ["remote", "add", "origin", remote], source); + await runFixtureCommand("git", ["push", "origin", "main"], source); + return { id: F1_WORKSPACE_ID, remote }; +} + +function writeSecure(path, value) { + writeFileSync(path, value, { encoding: "utf8", mode: 0o600 }); + chmodSync(path, 0o600); +} + +async function testPasswordHash(password) { + const salt = Buffer.from("thothii-e2e-salt"); + const message = Buffer.from(password, "utf8"); + let digest; + try { + digest = await new Promise((resolveDigest, rejectDigest) => { + argon2("argon2id", { + message, + nonce: salt, + memory: 65_536, + passes: 3, + parallelism: 1, + tagLength: 32, + }, (error, derived) => error || !derived ? rejectDigest(error ?? safeError("e2e_password_hash_failed")) : resolveDigest(derived)); + }); + return `$argon2id$v=19$m=65536,t=3,p=1$${salt.toString("base64").replaceAll("=", "")}$${digest.toString("base64").replaceAll("=", "")}`; + } finally { + message.fill(0); + salt.fill(0); + digest?.fill(0); + } +} + +function pause(milliseconds) { + return new Promise((resolvePause) => setTimeout(resolvePause, milliseconds)); +} + +function providerJson(url, caFile, options = {}) { + return new Promise((resolveResponse, rejectResponse) => { + const body = options.body ?? ""; + const request = httpsRequest(url, { + method: options.method ?? "GET", + ca: readFileSync(caFile), + headers: { + accept: "application/json", + ...(body.length === 0 ? {} : { + "content-length": String(Buffer.byteLength(body)), + "content-type": "application/x-www-form-urlencoded", + }), + ...options.headers, + }, + }, (response) => { + const chunks = []; + let size = 0; + response.on("data", (chunk) => { + size += chunk.length; + if (size > 64 * 1024) request.destroy(safeError("e2e_provider_response_too_large")); + else chunks.push(chunk); + }); + response.once("error", () => rejectResponse(safeError("e2e_provider_response_failed"))); + response.once("end", () => { + try { + resolveResponse({ status: response.statusCode ?? 0, body: JSON.parse(Buffer.concat(chunks).toString("utf8")) }); + } catch { + rejectResponse(safeError("e2e_provider_response_invalid")); + } + }); + }); + request.once("error", () => rejectResponse(safeError("e2e_provider_request_failed"))); + request.end(body); + }); +} + +async function freeLoopbackPort() { + const server = createServer(); + await new Promise((resolveListen, rejectListen) => { + server.once("error", rejectListen); + server.listen({ host: "127.0.0.1", port: 0 }, resolveListen); + }); + const address = server.address(); + await new Promise((resolveClose) => server.close(resolveClose)); + if (!address || typeof address === "string") throw safeError("e2e_loopback_port_unavailable"); + return address.port; +} + +const F1_QDRANT_INDEXES = Object.freeze([ + "content_hash", "document_id", "kind", "record_key", + "record_kind", "vector_generation", "workspace_id", "workspace_revision", +]); + +async function startFakeQdrant() { + const payloadSchema = Object.fromEntries(F1_QDRANT_INDEXES.map((field) => [field, { data_type: "keyword" }])); + const server = createHttpServer((request, response) => { + const path = new URL(request.url ?? "/", "http://loopback.invalid").pathname; + if (request.method !== "GET" || path !== `/collections/${F1_WORKSPACE_ID}`) { + response.writeHead(404).end(); + return; + } + response.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({ + result: { + config: { params: { vectors: { size: 1024, distance: "Cosine" } } }, + payload_schema: payloadSchema, + }, + })); + }); + await new Promise((resolveListen, rejectListen) => { + server.once("error", rejectListen); + server.listen({ host: "127.0.0.1", port: 0 }, resolveListen); + }); + const address = server.address(); + if (!address || typeof address === "string") { + await new Promise((resolveClose) => server.close(resolveClose)); + throw safeError("e2e_qdrant_loopback_port_unavailable"); + } + return { + baseUrl: `http://127.0.0.1:${address.port}/`, + close: () => new Promise((resolveClose) => server.close(resolveClose)), + }; +} + +function managedProcess(command, args, options) { + const child = spawn(command, args, { + cwd: options.cwd, + env: options.env, + stdio: ["ignore", "pipe", "pipe"], + }); + let exited = false; + let diagnostic = ""; + const captureDiagnostic = (chunk) => { + const sanitized = String(chunk) + .replace(/https?:\/\/[^\s)]+/g, "[url]") + .replace(/(?:THT_[A-Z_]+|PI_[A-Z_]+|AUTH_MODE)=\S+/g, "$1=[redacted]") + .replace(/[A-Za-z0-9_-]{43,}/g, "[redacted]"); + diagnostic = `${diagnostic}${sanitized}`.slice(-800); + }; + child.once("exit", () => { exited = true; }); + child.stdout?.on("data", captureDiagnostic); + child.stderr?.on("data", captureDiagnostic); + return { + child, + exited: () => exited, + diagnostic: () => diagnostic.replace(/\s+/g, " ").trim(), + async close() { + if (exited) return; + child.kill("SIGTERM"); + for (let attempt = 0; attempt < 50 && !exited; attempt += 1) await pause(20); + if (!exited) child.kill("SIGKILL"); + for (let attempt = 0; attempt < 50 && !exited; attempt += 1) await pause(20); + }, + }; +} + +async function waitForOk(url, processHandle) { + for (let attempt = 0; attempt < 300; attempt += 1) { + if (processHandle.exited()) { + const detail = processHandle.diagnostic(); + throw safeError(detail ? `e2e_service_stopped_during_startup:${detail}` : "e2e_service_stopped_during_startup"); + } + try { + const response = await fetch(url, { redirect: "error" }); + if (response.ok) return; + } catch { + // The process is expected to race its listener setup. + } + await pause(100); + } + throw safeError("e2e_service_startup_timeout"); +} + +function cleanBackendEnvironment(overrides) { + const env = { ...process.env }; + for (const name of [ + "AUTH_MODE", "THT_AUTH_CONFIG_FILE", "THT_AUTH_STATE_ROOT", "THT_SECRETS_FILE", "NODE_EXTRA_CA_CERTS", + "SETTINGS_FILE", "THT_MAINTENANCE_FILE", "THT_WORKSPACE_REGISTRY_ROOT", "THT_WORKSPACE_SECRET_STORE_ROOT", + "THT_WORKSPACE_SECRET_RUNTIME_ROOT", "THT_WORKSPACE_GIT_REMOTE", "THT_WORKSPACE_GIT_BRANCH", + "THT_WORKSPACE_SECRET_ROOTS", "THT_WORKSPACE_INSTALLATION_ID", "THT_LEGACY_WORKSPACE_MODE", + "THT_DWH_PRECHECK", "THT_INTERNAL_QDRANT_URL", "THT_CONFIG", + ]) delete env[name]; + for (const name of Object.keys(env)) { + if (name.startsWith("THT_WS_")) delete env[name]; + } + return { ...env, ...overrides }; +} + +export async function createAuthenticationStack({ withF1Workspace = false } = {}) { + const root = mkdtempSync(join(realpathSync(tmpdir()), "thothii-auth-e2e-")); + secureDirectory(root); + const stateRoot = join(root, "auth-state"); + const registryRoot = join(root, "workspace-registry"); + const workspaceSecretRoot = join(root, "workspace-secrets"); + const workspaceRuntimeRoot = join(root, "workspace-runtime"); + const fixtureSecretRoot = join(root, "fixture-runtime-secrets"); + const providerRoot = join(root, "provider"); + const authConfigFile = join(root, "auth.yaml"); + const usersFile = join(root, "users.yaml"); + const secretsFile = join(root, "test.secrets"); + const settingsFile = join(root, "settings.json"); + const maintenanceFile = join(root, "maintenance.json"); + const authStorageBinary = join(root, "tht-auth-storage"); + const fixtureDwhPasswordFile = join(fixtureSecretRoot, "fixture-dwh-password"); + const fixtureDwhCaFile = join(fixtureSecretRoot, "fixture-dwh-ca.pem"); + for (const path of [stateRoot, registryRoot, workspaceSecretRoot, workspaceRuntimeRoot, fixtureSecretRoot, providerRoot]) secureDirectory(path); + for (const child of ["sessions", "oidc"]) secureDirectory(join(stateRoot, child)); + + const [frontendPort, backendPort] = await Promise.all([freeLoopbackPort(), freeLoopbackPort()]); + const publicUrl = `http://127.0.0.1:${frontendPort}`; + const backendUrl = `http://127.0.0.1:${backendPort}`; + const workspace = withF1Workspace ? await prepareF1Workspace(root) : undefined; + const provider = await startFakeOidcProvider({ directory: providerRoot }); + await buildAuthenticationStorageBridge(authStorageBinary); + const localPassword = "e2e-local-password"; + const passwordHash = await testPasswordHash(localPassword); + const accounts = Object.freeze({ + ordinary: Object.freeze({ username: "ordinary", password: localPassword }), + admin: Object.freeze({ username: "administrator", password: localPassword }), + }); + writeSecure(usersFile, JSON.stringify({ + version: 1, + users: [ + { + id: "11111111-1111-4111-8111-111111111111", + username: accounts.ordinary.username, + displayName: "Fixture ordinary", + passwordHash, + roles: ["user"], + enabled: true, + authRevision: 1, + }, + { + id: "22222222-2222-4222-8222-222222222222", + username: accounts.admin.username, + displayName: "Fixture administrator", + passwordHash, + roles: ["admin"], + enabled: true, + authRevision: 1, + }, + ], + })); + writeSecure(secretsFile, [ + "THT_OIDC_CLIENT_SECRET=e2e-client-secret-not-a-production-secret", + "THT_AUTHENTIK_API_TOKEN=e2e-group-catalog-token-not-a-production-secret", + "", + ].join("\n")); + if (workspace) { + writeSecure(fixtureDwhPasswordFile, "fixture-password-not-a-secret\n"); + writeSecure(fixtureDwhCaFile, "fixture-ca-not-a-production-certificate\n"); + writeSecure(settingsFile, JSON.stringify({ + workspace: workspace.id, + provider: "zai", + model: "glm-5.2", + thinking: "medium", + })); + } + + let mode = undefined; + let backend = undefined; + let qdrant = undefined; + const frontend = managedProcess(join(frontendRoot, "node_modules", ".bin", "vite"), [ + "--host", "127.0.0.1", "--port", String(frontendPort), "--strictPort", + ], { + cwd: frontendRoot, + env: { + ...process.env, + THT_FRONTEND_API_UPSTREAM: backendUrl, + }, + }); + try { + await waitForOk(publicUrl, frontend); + qdrant = workspace ? await startFakeQdrant() : undefined; + + const localConfig = () => ({ + version: 1, + mode: "local", + publicUrl, + session: { + regularTtlSeconds: 600, + regularIdleSeconds: 600, + rememberTtlSeconds: 2_592_000, + rememberIdleSeconds: 604_800, + }, + local: { usersFile: "users.yaml" }, + }); + const oidcConfig = () => ({ + version: 1, + mode: "oidc", + publicUrl, + session: { + regularTtlSeconds: 600, + regularIdleSeconds: 600, + oidcTtlSeconds: 60, + }, + oidc: { + issuer: provider.issuer, + clientId: "thothii-e2e-client", + clientSecretRef: "THT_OIDC_CLIENT_SECRET", + scopes: ["openid", "profile", "groups"], + groupsClaim: "groups", + }, + groupCatalog: { + driver: "authentik", + baseUrl: provider.baseUrl, + apiTokenRef: "THT_AUTHENTIK_API_TOKEN", + }, + authorization: { + groupRoles: { + "fixture-users": ["user"], + "fixture-admin": ["admin"], + }, + }, + }); + + async function startBackend() { + if (mode === undefined) throw safeError("e2e_auth_mode_not_configured"); + backend = managedProcess(join(backendRoot, "node_modules", ".bin", "tsx"), ["src/server.ts"], { + cwd: backendRoot, + env: cleanBackendEnvironment({ + NODE_ENV: "test", + HOST: "127.0.0.1", + PORT: String(backendPort), + PI_BIN: fakePi, + THT_BIN: fakeTht, + THT_AUTH_STORAGE_BIN: authStorageBinary, + THT_HARNESS_DIR: harnessRoot, + THT_AUTH_CONFIG_FILE: authConfigFile, + THT_AUTH_STATE_ROOT: stateRoot, + THT_SECRETS_FILE: secretsFile, + NODE_EXTRA_CA_CERTS: provider.caFile, + SETTINGS_FILE: settingsFile, + THT_MAINTENANCE_FILE: maintenanceFile, + THT_WORKSPACE_REGISTRY_ROOT: registryRoot, + THT_WORKSPACE_SECRET_STORE_ROOT: workspaceSecretRoot, + THT_WORKSPACE_SECRET_RUNTIME_ROOT: workspaceRuntimeRoot, + THT_WORKSPACE_INSTALLATION_ID: "e2e", + THT_DATA_ROOT: join(root, "data"), + ...(qdrant ? { THT_INTERNAL_QDRANT_URL: qdrant.baseUrl } : {}), + ...(workspace ? { + THT_WORKSPACE_GIT_REMOTE: workspace.remote, + THT_WORKSPACE_GIT_BRANCH: "main", + THT_WORKSPACE_SECRET_ROOTS: fixtureSecretRoot, + THT_WS_FIXTURE_WORKSPACE_DWH_TRANSPORT: "postgres_direct", + THT_WS_FIXTURE_WORKSPACE_DWH_HOST: "127.0.0.1", + THT_WS_FIXTURE_WORKSPACE_DWH_PORT: "5432", + THT_WS_FIXTURE_WORKSPACE_DWH_USER: "fixture", + THT_WS_FIXTURE_WORKSPACE_DWH_PASSWORD_FILE: fixtureDwhPasswordFile, + THT_WS_FIXTURE_WORKSPACE_DWH_TLS_CA_FILE: fixtureDwhCaFile, + } : {}), + }), + }); + await waitForOk(`${backendUrl}/health`, backend); + } + + async function restartBackend() { + await backend?.close(); + backend = undefined; + await startBackend(); + } + + return { + publicUrl, + localAccount(account) { + const found = accounts[account]; + if (!found) throw safeError("e2e_local_account_unknown"); + return found; + }, + lastAuthorization() { + return provider.lastAuthorization(); + }, + async providerSurface() { + const discovery = await providerJson(`${provider.issuer}.well-known/openid-configuration`, provider.caFile); + const jwks = await providerJson(`${provider.issuer}jwks`, provider.caFile); + const device = await providerJson(`${provider.issuer}device_authorization`, provider.caFile, { + method: "POST", + body: "client_id=thothii-e2e-client", + }); + const deviceCode = typeof device.body?.device_code === "string" ? device.body.device_code : ""; + const deviceToken = deviceCode.length === 0 ? { status: 0, body: {} } : await providerJson(`${provider.issuer}token`, provider.caFile, { + method: "POST", + body: `grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Adevice_code&device_code=${encodeURIComponent(deviceCode)}`, + }); + const groups = await providerJson(`${provider.baseUrl}/api/v3/core/groups/?name=fixture-users`, provider.caFile, { + headers: { authorization: "Bearer e2e-fixture" }, + }); + return { + discovery: discovery.status === 200 && discovery.body?.issuer === provider.issuer, + jwks: jwks.status === 200 && jwks.body?.keys?.[0]?.alg === "RS256" && jwks.body?.keys?.[0]?.use === "sig", + deviceAuthorization: device.status === 200 && typeof device.body?.device_code === "string" + && typeof device.body?.verification_uri === "string", + deviceToken: deviceToken.status === 200 && typeof deviceToken.body?.id_token === "string", + groupList: groups.status === 200 && groups.body?.results?.[0]?.name === "fixture-users", + }; + }, + setOidcIdentity(identity) { + provider.setIdentity(identity); + }, + async useLocalMode() { + writeSecure(authConfigFile, JSON.stringify(localConfig())); + mode = "local"; + await restartBackend(); + }, + async useOidcMode(identity) { + provider.setIdentity(identity); + writeSecure(authConfigFile, JSON.stringify(oidcConfig())); + mode = "oidc"; + await restartBackend(); + }, + restartBackend, + async close() { + await backend?.close(); + await frontend.close(); + await provider.close(); + await qdrant?.close(); + rmSync(root, { recursive: true, force: true }); + }, + }; + } catch (error) { + await backend?.close(); + await frontend.close(); + await provider.close(); + await qdrant?.close(); + rmSync(root, { recursive: true, force: true }); + throw error; + } +} diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index 42a43771..580ba972 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -1,50 +1,14 @@ import { defineConfig, devices } from "@playwright/test"; -import { resolve } from "node:path"; - -const __dir = new URL(".", import.meta.url).pathname; -const fakePi = resolve(__dir, "e2e/fixtures/fake-pi.mjs"); -const fakeTht = resolve(__dir, "e2e/fixtures/fake-tht.mjs"); -const harnessDir = resolve(__dir, "..", "harness"); export default defineConfig({ testDir: "./e2e", timeout: 60_000, retries: process.env.CI ? 2 : 0, - use: { - baseURL: "http://localhost:5199", - }, + workers: 1, projects: [ { name: "chromium", use: { ...devices["Desktop Chrome"] }, }, ], - webServer: [ - { - // Backend — wired to both fakes; no VPN/Pi/Python required. - command: "npm run dev", - cwd: resolve(__dir, "..", "backend"), - url: "http://localhost:8799/health", - timeout: 30_000, - reuseExistingServer: !process.env.CI, - env: { - PI_BIN: fakePi, - THT_BIN: fakeTht, - THT_HARNESS_DIR: harnessDir, - PORT: "8799", - AUTH_MODE: "none", - }, - }, - { - // Frontend dev server pointing at the hermetic backend. - command: "npm run dev -- --port 5199", - cwd: __dir, - url: "http://localhost:5199", - timeout: 30_000, - reuseExistingServer: !process.env.CI, - env: { - VITE_BACKEND_URL: "http://localhost:8799", - }, - }, - ], }); diff --git a/harness/tests/test_local_compose_contract.py b/harness/tests/test_local_compose_contract.py index 8066766c..6c04f4c5 100644 --- a/harness/tests/test_local_compose_contract.py +++ b/harness/tests/test_local_compose_contract.py @@ -13,11 +13,14 @@ def test_local_compose_uses_the_generic_external_endpoint_contract(): } # workspace-maintenance is profile-gated: it must not be part of the default local startup. assert compose["services"]["workspace-maintenance"].get("profiles") == ["workspace-maintenance"] - assert local["services"]["core"]["environment"]["AUTH_MODE"] == "none" + # Authentication is controlled by the mounted configuration; the local profile + # must not silently re-enable the legacy unauthenticated development mode. + assert "AUTH_MODE" not in local["services"]["core"]["environment"] assert local["services"]["core"]["ports"] == ["127.0.0.1:${THOTH_CORE_HTTP_PORT:-8787}:8787"] assert local["services"]["frontend"]["ports"] == ["127.0.0.1:${THOTH_HTTP_PORT:-8080}:8080"] environment = compose["services"]["core"]["environment"] + assert {"THT_AUTH_CONFIG_FILE", "THT_AUTH_STATE_ROOT"} <= set(environment) for name in ("THT_DWH_REST_URL", "THT_LLM_URL"): assert name in environment assert environment["THT_INTERNAL_QDRANT_URL"] == "http://qdrant:6333" diff --git a/scripts/authentication-smoke.sh b/scripts/authentication-smoke.sh new file mode 100755 index 00000000..77e74e7c --- /dev/null +++ b/scripts/authentication-smoke.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P) +expected_node=${THT_EXPECTED_NODE_VERSION:-v24.16.0} +actual_node=$(node --version) + +if [[ "$actual_node" != "$expected_node" ]]; then + printf 'authentication smoke requires Node %s; found %s\n' "$expected_node" "$actual_node" >&2 + exit 2 +fi + +for command in go openssl; do + command -v "$command" >/dev/null 2>&1 || { + printf 'authentication smoke requires %s\n' "$command" >&2 + exit 2 + } +done + +playwright="$root/frontend/node_modules/.bin/playwright" +[[ -x "$playwright" ]] || { + echo "authentication smoke requires frontend dependencies (run npm ci in frontend)" >&2 + exit 2 +} + +temporary_root=$(mktemp -d "${TMPDIR:-/tmp}/thothii-authentication-smoke.XXXXXX") +trap 'rm -rf "$temporary_root"' EXIT HUP INT TERM +log="$temporary_root/playwright.log" +output="$temporary_root/playwright" +mkdir -p "$output" +sentinel="task15-sentinel-$(openssl rand -hex 24)" +export THT_TASK15_SENTINEL="$sentinel" + +sanitize_failure_log() { + sed -E \ + -e "s/${sentinel}/[redacted]/g" \ + -e 's#https?://[^[:space:])]+#[url]#g' \ + -e 's/(THT_[A-Z_]+|PI_[A-Z_]+|AUTH_MODE)=[^[:space:]]+/\1=[redacted]/g' \ + "$log" | tail -n 100 >&2 +} + +if ! ( + cd "$root/frontend" + THT_E2E_AUTH_STACK=1 "$playwright" test e2e/auth.spec.ts --workers=1 --output="$output" +) >"$log" 2>&1; then + echo "authentication smoke: hermetic browser suite failed" >&2 + sanitize_failure_log + exit 1 +fi + +if rg -a -Fq -- "$sentinel" "$log" "$output"; then + echo "authentication smoke: sentinel appeared in retained test output" >&2 + exit 1 +fi + +printf 'authentication smoke: hermetic OIDC/browser suite passed on Node %s\n' "$actual_node" diff --git a/scripts/test-compose-provider-readiness.sh b/scripts/test-compose-provider-readiness.sh index 3b25268c..eb6b408f 100755 --- a/scripts/test-compose-provider-readiness.sh +++ b/scripts/test-compose-provider-readiness.sh @@ -18,10 +18,56 @@ trap cleanup EXIT HUP INT TERM printf '%s\n' '{"zai":{"type":"api_key","key":"fixture-native-auth-key"}}' >"$tmp/pi-auth.json" printf '%s\n' 'THT_MODEL_API_KEY=fixture-model-api-key' >"$tmp/thothii.secrets" chmod 0600 "$tmp/pi-auth.json" "$tmp/thothii.secrets" +auth_config="$tmp/auth" +mkdir "$auth_config" +chmod 0700 "$auth_config" +printf '%s\n' \ + 'version: 1' \ + 'mode: local' \ + 'publicUrl: http://127.0.0.1:8080' \ + 'local:' \ + ' usersFile: users.yaml' \ + >"$auth_config/auth.yaml" +node - "$auth_config/users.yaml" <<'NODE' +const { argon2 } = require("node:crypto"); +const { writeFileSync } = require("node:fs"); + +const message = Buffer.from("fixture-local-password", "utf8"); +const nonce = Buffer.from([...Array(16).keys()]); +argon2("argon2id", { + message, + nonce, + memory: 65_536, + parallelism: 1, + tagLength: 32, + passes: 3, +}, (error, digest) => { + message.fill(0); + nonce.fill(0); + if (error || !digest) throw error ?? new Error("fixture password hash failed"); + const salt = Buffer.from([...Array(16).keys()]).toString("base64").replaceAll("=", ""); + const hash = digest.toString("base64").replaceAll("=", ""); + writeFileSync(process.argv[2], [ + "version: 1", + "users:", + " - id: 00000000-0000-4000-8000-000000000001", + " username: fixture-user", + " displayName: Fixture user", + ` passwordHash: $argon2id$v=19$m=65536,t=3,p=1$${salt}$${hash}`, + " roles:", + " - user", + " enabled: true", + " authRevision: 1", + "", + ].join("\\n"), { mode: 0o600 }); +}); +NODE +chmod 0600 "$auth_config/auth.yaml" "$auth_config/users.yaml" printf '%s\n' \ 'THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git' \ "PI_AUTH_FILE=$tmp/pi-auth.json" \ "THT_SECRETS_FILE=$tmp/thothii.secrets" \ + "THT_AUTH_CONFIG_ROOT=$auth_config" \ 'THOTH_CORE_HTTP_PORT=0' \ 'THOTH_HTTP_PORT=0' \ >"$tmp/local.env" diff --git a/scripts/test-pi-user-auth-compose.sh b/scripts/test-pi-user-auth-compose.sh index 10594825..47daf350 100755 --- a/scripts/test-pi-user-auth-compose.sh +++ b/scripts/test-pi-user-auth-compose.sh @@ -8,12 +8,18 @@ trap 'rm -rf "$tmp"' EXIT HUP INT TERM auth_file="$tmp/auth.json" printf '%s\n' '{}' >"$auth_file" chmod 0600 "$auth_file" +auth_config="$tmp/auth" +mkdir "$auth_config" +chmod 0700 "$auth_config" +printf '%s\n' 'mode: local' >"$auth_config/auth.yaml" +chmod 0600 "$auth_config/auth.yaml" secrets_file="$tmp/thothii.secrets" printf '%s\n' 'THT_MODEL_API_KEY=fixture-model-api-key' >"$secrets_file" chmod 0600 "$secrets_file" rendered=$(THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git \ - PI_AUTH_FILE="$auth_file" THT_SECRETS_FILE="$secrets_file" docker compose config) + PI_AUTH_FILE="$auth_file" THT_SECRETS_FILE="$secrets_file" \ + THT_AUTH_CONFIG_ROOT="$auth_config" docker compose config) printf '%s\n' "$rendered" | grep -q "source: $auth_file" printf '%s\n' "$rendered" | grep -q 'target: /home/thoth/.pi/agent/auth.json' printf '%s\n' "$rendered" | grep -A4 'target: /home/thoth/.pi/agent/auth.json' \ @@ -33,6 +39,7 @@ fi dev_rendered=$(THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git \ PI_AUTH_FILE="$auth_file" THT_SECRETS_FILE="$secrets_file" \ + THT_AUTH_CONFIG_ROOT="$auth_config" \ docker compose --env-file deploy/env/local.env.example -f docker-compose.dev.yml config) printf '%s\n' "$dev_rendered" | grep -q "source: $auth_file" printf '%s\n' "$dev_rendered" | grep -q 'target: /home/thoth/.pi/agent/auth.json' diff --git a/scripts/test-preprocess-compose-config.sh b/scripts/test-preprocess-compose-config.sh index b350d31c..bdceeb7f 100755 --- a/scripts/test-preprocess-compose-config.sh +++ b/scripts/test-preprocess-compose-config.sh @@ -5,13 +5,18 @@ cd "$(dirname "$0")/.." tmp_bundle=$(mktemp) tmp_auth=$(mktemp) -trap 'rm -f "$tmp_bundle" "$tmp_auth"' EXIT HUP INT TERM +tmp_auth_config=$(mktemp -d) +trap 'rm -f "$tmp_bundle" "$tmp_auth"; rm -rf "$tmp_auth_config"' EXIT HUP INT TERM printf '%s\n' 'THT_MODEL_API_KEY=test-model' >"$tmp_bundle" chmod 0600 "$tmp_bundle" printf '%s\n' '{}' >"$tmp_auth" chmod 0600 "$tmp_auth" +chmod 0700 "$tmp_auth_config" +printf '%s\n' 'mode: local' >"$tmp_auth_config/auth.yaml" +chmod 0600 "$tmp_auth_config/auth.yaml" export THT_SECRETS_FILE="$tmp_bundle" export PI_AUTH_FILE="$tmp_auth" +export THT_AUTH_CONFIG_ROOT="$tmp_auth_config" export THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git config_json=$(docker compose -f compose.yaml -f deploy/compose.preprocess.yaml --profile preprocess config --format json) diff --git a/scripts/test-server-pi-state-topology.sh b/scripts/test-server-pi-state-topology.sh index 59f07e22..8f8f2d78 100755 --- a/scripts/test-server-pi-state-topology.sh +++ b/scripts/test-server-pi-state-topology.sh @@ -9,7 +9,12 @@ fixture="$(mktemp -d "$tmp_parent/thoth-server-pi-state.XXXXXX")" trap 'rm -rf "$fixture"' EXIT HUP INT TERM pi_state="$fixture/empty pi state" +auth_config="$fixture/auth" mkdir -p "$pi_state" +mkdir -p "$auth_config" +chmod 0700 "$auth_config" +printf 'mode: local\n' >"$auth_config/auth.yaml" +chmod 0600 "$auth_config/auth.yaml" "$root/scripts/prepare-server-pi-state.sh" "$pi_state" "$(id -u)" "$(id -g)" for target in auth.json models.json settings.json; do @@ -40,6 +45,7 @@ THOTH_SERVER_BIND=127.0.0.1 THOTH_HTTP_PORT=0 PI_AUTH_FILE=$fixture/pi-auth.json THT_SECRETS_FILE=$fixture/thothii.secrets +THT_AUTH_CONFIG_ROOT=$auth_config THT_DATA_ROOT=$fixture/data THT_PI_STATE_ROOT=$pi_state THT_WORKSPACE_REGISTRY_ROOT=$fixture/workspace-registry diff --git a/tools/tht/internal/backup/create.go b/tools/tht/internal/backup/create.go index 73999343..807cca1e 100644 --- a/tools/tht/internal/backup/create.go +++ b/tools/tht/internal/backup/create.go @@ -127,23 +127,32 @@ func createWithDependencies(ctx context.Context, installation config.Installatio if request.IncludeSecrets && !request.Confirm { return Result{}, ErrConfirmationRequired } - lock, err := lifecycle.Acquire(installation) + transaction, err := lifecycle.AcquireTransaction(installation) if err != nil { return Result{}, err } defer func() { - if releaseErr := lock.Release(); releaseErr != nil { + if releaseErr := transaction.Release(); releaseErr != nil { result = Result{} resultErr = errors.Join(resultErr, fmt.Errorf("release backup lifecycle lock: %w", releaseErr)) } }() - return createWithDependenciesLockHeld(ctx, installation, request, dependencies) + return createWithDependenciesTransaction(ctx, transaction, installation, request, dependencies) } -// createWithDependenciesLockHeld performs backup creation while the caller owns the installation -// lifecycle lock. It must never acquire a lifecycle lock itself: Restore uses this primitive to -// create its recovery checkpoint inside its already-locked transaction. -func createWithDependenciesLockHeld(ctx context.Context, installation config.Installation, request CreateRequest, dependencies dependencies) (result Result, resultErr error) { +// createWithDependenciesTransaction performs backup creation only with an active, opaque +// installation-bound lifecycle capability. Restore passes the capability it acquired for the +// enclosing transaction, so a recovery checkpoint cannot run without the same lifecycle lock. +func createWithDependenciesTransaction(ctx context.Context, transaction *lifecycle.Transaction, installation config.Installation, request CreateRequest, dependencies dependencies) (result Result, resultErr error) { + if err := transaction.Verify(installation); err != nil { + return Result{}, err + } + if dependencies.runner == nil || dependencies.now == nil || dependencies.homeDir == nil || dependencies.revision == nil || dependencies.sleep == nil || dependencies.reserveOutput == nil || dependencies.publishReserved == nil { + return Result{}, errors.New("backup dependencies are incomplete") + } + if request.IncludeSecrets && !request.Confirm { + return Result{}, ErrConfirmationRequired + } if err := rejectInlineSecretValues(installation.EnvFile); err != nil { return Result{}, err } diff --git a/tools/tht/internal/backup/create_test.go b/tools/tht/internal/backup/create_test.go index b4c21c39..d43f8f66 100644 --- a/tools/tht/internal/backup/create_test.go +++ b/tools/tht/internal/backup/create_test.go @@ -537,6 +537,33 @@ func TestCreateHonorsTheSharedInstallationLifecycleLock(t *testing.T) { } } +func TestCreateTransactionCapabilityRefusesUnlockedOrForeignInstallation(t *testing.T) { + fixture := newBackupFixture(t, "local") + runner := newBackupRunner(fixture.installation, false) + request := CreateRequest{Output: filepath.Join(t.TempDir(), "transaction.zip")} + + if _, err := createWithDependenciesTransaction(context.Background(), nil, fixture.installation, request, testDependencies(t, runner)); !errors.Is(err, lifecycle.ErrTransactionInactive) { + t.Fatalf("nil transaction error = %v, want ErrTransactionInactive", err) + } + if len(runner.calls) != 0 { + t.Fatalf("Docker runner was called without a transaction capability: %v", runner.calls) + } + + otherRoot := t.TempDir() + other := config.Installation{ProjectDirectory: otherRoot, Path: filepath.Join(otherRoot, "thothii-installation.yaml")} + transaction, err := lifecycle.AcquireTransaction(other) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = transaction.Release() }) + if _, err := createWithDependenciesTransaction(context.Background(), transaction, fixture.installation, request, testDependencies(t, runner)); !errors.Is(err, lifecycle.ErrTransactionInstallation) { + t.Fatalf("foreign transaction error = %v, want ErrTransactionInstallation", err) + } + if len(runner.calls) != 0 { + t.Fatalf("Docker runner was called with a foreign transaction capability: %v", runner.calls) + } +} + func TestCreateRefusesMutableNonRunningServiceStates(t *testing.T) { for _, state := range []string{"paused", "restarting", "created", "removing"} { t.Run(state, func(t *testing.T) { diff --git a/tools/tht/internal/backup/preflight.go b/tools/tht/internal/backup/preflight.go index a8169582..04410d62 100644 --- a/tools/tht/internal/backup/preflight.go +++ b/tools/tht/internal/backup/preflight.go @@ -91,6 +91,14 @@ type verifiedArchive struct { limits PreflightLimits } +// stagedArchive holds an installation-private, immutable copy of the exact bytes accepted by +// Preflight. The original archive remains retained only for provenance revalidation. +type stagedArchive struct { + file *os.File + path string + directory string +} + type inspectedArchiveEntry struct { metadata ArchiveEntryMetadata member *zip.File @@ -176,12 +184,17 @@ func Preflight(ctx context.Context, installation config.Installation, request Pr if err != nil { return PreflightResult{}, err } + stagingBytes := uint64(openedInfo.Size()) + if stagingBytes > ^uint64(0)-requiredBytes { + return PreflightResult{}, errors.New("restore staging requirement exceeds supported size") + } + requiredWithStaging := requiredBytes + stagingBytes freeBytes, err := dependencies.FreeBytes(installation.ProjectDirectory) if err != nil { return PreflightResult{}, fmt.Errorf("check free disk space: %w", err) } - if freeBytes < requiredBytes { - return PreflightResult{}, fmt.Errorf("insufficient free disk space for restore: need %d bytes, have %d", requiredBytes, freeBytes) + if freeBytes < requiredWithStaging { + return PreflightResult{}, fmt.Errorf("insufficient free disk space for restore: need %d bytes, have %d", requiredWithStaging, freeBytes) } if err := contextError(ctx); err != nil { return PreflightResult{}, err @@ -214,6 +227,10 @@ func Preflight(ctx context.Context, installation config.Installation, request Pr // it immediately before a restore transaction and use the returned retained handle, never reopen // ArchivePath. It refuses a path replacement or in-place content change. func (result PreflightResult) RevalidateArchive() (*os.File, error) { + return result.revalidateArchive(context.Background()) +} + +func (result PreflightResult) revalidateArchive(ctx context.Context) (*os.File, error) { if result.archive == nil || result.archive.file == nil { return nil, errors.New("backup archive has not been retained by preflight") } @@ -228,13 +245,115 @@ func (result PreflightResult) RevalidateArchive() (*os.File, error) { if err != nil || !os.SameFile(result.archive.info, heldInfo) || heldInfo.Size() != result.ArchiveSize { return nil, errors.New("backup archive changed after preflight") } - digest, err := digestArchive(context.Background(), result.archive.file, result.archive.limits.MaxArchiveBytes) + digest, err := digestArchive(ctx, result.archive.file, result.archive.limits.MaxArchiveBytes) if err != nil || digest != result.archive.digest { return nil, errors.New("backup archive changed after preflight") } return result.archive.file, nil } +// StageArchive revalidates the retained archive and copies its exact bytes into a private file +// immediately before extraction. Later writes to the source archive cannot affect extraction. +func (result PreflightResult) StageArchive(ctx context.Context) (_ *stagedArchive, resultErr error) { + source, err := result.revalidateArchive(ctx) + if err != nil { + return nil, err + } + directory, err := os.MkdirTemp("", "tht-restore-stage-") + if err != nil { + return nil, errors.New("create private restore staging directory") + } + if err := os.Chmod(directory, 0o700); err != nil { + _ = os.Remove(directory) + return nil, errors.New("protect private restore staging directory") + } + path := filepath.Join(directory, "archive.zip") + file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + _ = os.Remove(directory) + return nil, errors.New("create private restore staging archive") + } + staged := &stagedArchive{file: file, path: path, directory: directory} + completed := false + defer func() { + if !completed { + _ = staged.Close() + } + }() + if _, err := source.Seek(0, io.SeekStart); err != nil { + return nil, errors.New("seek verified backup archive for staging") + } + digest := sha256.New() + buffer := make([]byte, 128*1024) + var total int64 + for { + if err := contextError(ctx); err != nil { + return nil, err + } + count, readErr := source.Read(buffer) + if count > 0 { + if int64(count) > result.ArchiveSize-total { + return nil, errors.New("backup archive changed after preflight") + } + written, writeErr := file.Write(buffer[:count]) + if writeErr != nil || written != count { + return nil, errors.New("write private restore staging archive") + } + if _, writeErr := digest.Write(buffer[:count]); writeErr != nil { + return nil, errors.New("hash private restore staging archive") + } + total += int64(count) + } + if errors.Is(readErr, io.EOF) { + break + } + if readErr != nil { + return nil, errors.New("read verified backup archive for staging") + } + } + if total != result.ArchiveSize || digestForHash(digest) != result.archive.digest { + return nil, errors.New("backup archive changed after preflight") + } + if err := file.Sync(); err != nil { + return nil, errors.New("sync private restore staging archive") + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return nil, errors.New("rewind private restore staging archive") + } + completed = true + return staged, nil +} + +// Close removes only the staging file and directory created by StageArchive. +func (staged *stagedArchive) Close() error { + if staged == nil { + return nil + } + var failed bool + if staged.file != nil { + if err := staged.file.Close(); err != nil { + failed = true + } + staged.file = nil + } + if staged.path != "" { + if err := os.Remove(staged.path); err != nil && !errors.Is(err, os.ErrNotExist) { + failed = true + } + staged.path = "" + } + if staged.directory != "" { + if err := os.Remove(staged.directory); err != nil && !errors.Is(err, os.ErrNotExist) { + failed = true + } + staged.directory = "" + } + if failed { + return errors.New("destroy private restore staging archive") + } + return nil +} + // CloseArchive releases the retained read-only archive handle after the caller finishes the // restore transaction or decides not to proceed. func (result PreflightResult) CloseArchive() error { diff --git a/tools/tht/internal/backup/preflight_test.go b/tools/tht/internal/backup/preflight_test.go index c3505f74..fa2c88e1 100644 --- a/tools/tht/internal/backup/preflight_test.go +++ b/tools/tht/internal/backup/preflight_test.go @@ -7,6 +7,7 @@ import ( "crypto/sha256" "encoding/hex" "errors" + "io" "os" "path/filepath" "strings" @@ -356,6 +357,79 @@ func TestPreflightRevalidationRefusesAnArchivePathThatWasReplaced(t *testing.T) } } +func TestPreflightStagesArchiveIntoImmutablePrivateBytes(t *testing.T) { + installation := preflightTestInstallation(t) + archive := filepath.Join(t.TempDir(), "checked.zip") + writePreflightArchive(t, archive, preflightArchiveSpec{ + entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("before")}}, + }) + + result, err := Preflight(context.Background(), installation, PreflightRequest{Archive: archive, Confirm: true}, permissivePreflightDependencies()) + if err != nil { + t.Fatal(err) + } + defer result.CloseArchive() + staged, err := result.StageArchive(context.Background()) + if err != nil { + t.Fatal(err) + } + defer staged.Close() + + writePreflightArchive(t, archive, preflightArchiveSpec{ + entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("after!")}}, + }) + reader, err := zip.NewReader(staged.file, result.ArchiveSize) + if err != nil { + t.Fatal(err) + } + if len(reader.File) < 2 { + t.Fatalf("staged archive members = %d, want manifest and payload", len(reader.File)) + } + var payload *zip.File + for _, member := range reader.File { + if member.Name == "configuration/operator.env" { + payload = member + break + } + } + if payload == nil { + t.Fatal("staged archive is missing the configured payload") + } + stream, err := payload.Open() + if err != nil { + t.Fatal(err) + } + defer stream.Close() + body, err := io.ReadAll(stream) + if err != nil { + t.Fatal(err) + } + if string(body) != "before" { + t.Fatalf("staged payload = %q, want preflighted bytes", body) + } +} + +func TestPreflightStagingRejectsInPlaceArchiveHashMutation(t *testing.T) { + installation := preflightTestInstallation(t) + archive := filepath.Join(t.TempDir(), "checked.zip") + writePreflightArchive(t, archive, preflightArchiveSpec{ + entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("before")}}, + }) + + result, err := Preflight(context.Background(), installation, PreflightRequest{Archive: archive, Confirm: true}, permissivePreflightDependencies()) + if err != nil { + t.Fatal(err) + } + defer result.CloseArchive() + writePreflightArchive(t, archive, preflightArchiveSpec{ + entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("after!")}}, + }) + + if _, err := result.StageArchive(context.Background()); err == nil || !strings.Contains(err.Error(), "changed") { + t.Fatalf("StageArchive() error = %v, want changed archive refusal", err) + } +} + func TestPreflightRejectsArchiveEntryWithModeDifferentFromManifest(t *testing.T) { installation := preflightTestInstallation(t) archive := filepath.Join(t.TempDir(), "mode-mismatch.zip") diff --git a/tools/tht/internal/backup/restore.go b/tools/tht/internal/backup/restore.go index e90022dc..8c15dff3 100644 --- a/tools/tht/internal/backup/restore.go +++ b/tools/tht/internal/backup/restore.go @@ -9,6 +9,7 @@ import ( "time" "github.com/aritmolab/thothii/tools/tht/internal/config" + "github.com/aritmolab/thothii/tools/tht/internal/lifecycle" ) var ErrRestoreConfirmationRequired = errors.New("restore requires --yes") @@ -29,19 +30,17 @@ type RestoreResult struct { Verified bool } -type restoreLock interface{ Release() error } type restoreVerify func(context.Context, config.Installation, archiveRunner) error type restoreDependencies struct { preflight func(context.Context, config.Installation, PreflightRequest) (PreflightResult, error) - // checkpointLocked creates the secret-aware recovery archive while the caller already owns - // the installation lifecycle lock. It must not call public Create, which would re-acquire the - // non-reentrant lock and deadlock the restore transaction. - checkpointLocked func(context.Context, config.Installation, CreateRequest) (Result, error) + // checkpoint requires the opaque capability created by lifecycle acquisition. It must not call + // public Create, which would re-acquire the non-reentrant lock and deadlock the transaction. + checkpoint func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) prepareRecovery func(context.Context, config.Installation, string) (PreflightResult, error) recover func(context.Context, config.Installation, PreflightResult, bool) error cleanupCheckpoint func(string) error - acquireLock func(config.Installation) (restoreLock, error) + acquireTransaction func(config.Installation) (*lifecycle.Transaction, error) runner archiveRunner sleep func(duration time.Duration) restoreFile func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error @@ -84,36 +83,32 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati if request.Archive == "" { return RestoreResult{}, errors.New("restore archive is required") } - if deps.preflight == nil || deps.checkpointLocked == nil || deps.prepareRecovery == nil || deps.recover == nil || deps.cleanupCheckpoint == nil || deps.acquireLock == nil || deps.runner == nil || deps.restoreFile == nil || deps.restoreVolume == nil || deps.resetAuthenticationState == nil || deps.verify == nil { + if deps.preflight == nil || deps.checkpoint == nil || deps.prepareRecovery == nil || deps.recover == nil || deps.cleanupCheckpoint == nil || deps.acquireTransaction == nil || deps.runner == nil || deps.restoreFile == nil || deps.restoreVolume == nil || deps.resetAuthenticationState == nil || deps.verify == nil { return RestoreResult{}, errors.New("restore dependencies are incomplete") } - preflight, err := deps.preflight(ctx, installation, PreflightRequest{Archive: request.Archive, Confirm: true, AllowExternalSecrets: true}) - if err != nil { - return RestoreResult{}, err - } - archive, err := preflight.RevalidateArchive() - if err != nil { - _ = preflight.CloseArchive() - return RestoreResult{}, err - } // Lock ordering is lifecycle lock -> Compose/operator maintenance barrier. The core operator // command never acquires the host lifecycle lock, so this order cannot form a lock cycle with // Docker Compose or the durable maintenance marker. - lock, err := deps.acquireLock(installation) + transaction, err := deps.acquireTransaction(installation) if err != nil { - _ = preflight.CloseArchive() return result, err } defer func() { - if releaseErr := lock.Release(); releaseErr != nil { + if releaseErr := transaction.Release(); releaseErr != nil { result = RestoreResult{} resultErr = errors.Join(resultErr, fmt.Errorf("release restore lifecycle lock: %w", releaseErr)) } }() + // Target-dependent preflight is intentionally inside the lifecycle transaction. This binds + // target ownership, volume, image, and free-space checks to the later mutation. + preflight, err := deps.preflight(ctx, installation, PreflightRequest{Archive: request.Archive, Confirm: true, AllowExternalSecrets: true}) + if err != nil { + return RestoreResult{}, err + } defer preflight.CloseArchive() - checkpoint, err := deps.checkpointLocked(ctx, installation, CreateRequest{IncludeSecrets: true, Confirm: true}) + checkpoint, err := deps.checkpoint(ctx, transaction, installation, CreateRequest{IncludeSecrets: true, Confirm: true}) if err != nil { return result, fmt.Errorf("create recovery checkpoint: %w", err) } @@ -217,8 +212,18 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati return result, err } } + staged, err := preflight.StageArchive(ctx) + if err != nil { + return result, err + } + defer func() { + if closeErr := staged.Close(); closeErr != nil { + result = RestoreResult{} + resultErr = errors.Join(resultErr, closeErr) + } + }() state.mutated = true - if err := restoreVerifiedEntries(ctx, installation, preflight, archive, deps.restoreFile, deps.restoreVolume); err != nil { + if err := restoreVerifiedEntries(ctx, installation, preflight, staged.file, deps.restoreFile, deps.restoreVolume); err != nil { return result, err } if err := deps.resetAuthenticationState(ctx, installation, deps.runner); err != nil { diff --git a/tools/tht/internal/backup/restore_host.go b/tools/tht/internal/backup/restore_host.go index e55f364b..64466aee 100644 --- a/tools/tht/internal/backup/restore_host.go +++ b/tools/tht/internal/backup/restore_host.go @@ -41,21 +41,19 @@ func productionRestoreDependencies(installation config.Installation) restoreDepe }, }) }, - checkpointLocked: func(ctx context.Context, target config.Installation, request CreateRequest) (Result, error) { + checkpoint: func(ctx context.Context, transaction *lifecycle.Transaction, target config.Installation, request CreateRequest) (Result, error) { path, err := restoreCheckpointPath(target, time.Now().UTC()) if err != nil { return Result{}, err } request.Output = path - return createWithDependenciesLockHeld(ctx, target, request, productionCreateDependencies(target)) + return createWithDependenciesTransaction(ctx, transaction, target, request, productionCreateDependencies(target)) }, - cleanupCheckpoint: cleanupRecoveryCheckpoint, - acquireLock: func(target config.Installation) (restoreLock, error) { - return lifecycle.Acquire(target) - }, - runner: runner, - sleep: time.Sleep, - restoreFile: restoreFilePayload, + cleanupCheckpoint: cleanupRecoveryCheckpoint, + acquireTransaction: lifecycle.AcquireTransaction, + runner: runner, + sleep: time.Sleep, + restoreFile: restoreFilePayload, restoreVolume: func(ctx context.Context, _ config.Installation, volume VolumeMetadata, input io.Reader) error { result, err := runner.Stream(ctx, volumeRestoreCommand(volume.Name), input, io.Discard) if err != nil || result.ExitCode != 0 { @@ -90,8 +88,7 @@ func cleanupRecoveryCheckpoint(path string) error { return nil } -func recoverRestoreTransaction(ctx context.Context, installation config.Installation, recovery PreflightResult, wasRunning bool, deps restoreDependencies) error { - var resultErr error +func recoverRestoreTransaction(ctx context.Context, installation config.Installation, recovery PreflightResult, wasRunning bool, deps restoreDependencies) (resultErr error) { if err := runCompose(ctx, installation, deps.runner, "stop"); err != nil { resultErr = errors.Join(resultErr, err) // The first stop may already have taken effect before Docker lost its response. Retry the @@ -100,11 +97,16 @@ func recoverRestoreTransaction(ctx context.Context, installation config.Installa return errors.Join(resultErr, retryErr) } } - archive, err := recovery.RevalidateArchive() + staged, err := recovery.StageArchive(ctx) if err != nil { return errors.Join(resultErr, err) } - if err := restoreVerifiedEntries(ctx, installation, recovery, archive, deps.restoreFile, deps.restoreVolume); err != nil { + defer func() { + if closeErr := staged.Close(); closeErr != nil { + resultErr = errors.Join(resultErr, closeErr) + } + }() + if err := restoreVerifiedEntries(ctx, installation, recovery, staged.file, deps.restoreFile, deps.restoreVolume); err != nil { return errors.Join(resultErr, err) } // Recovery restores only configuration/secret files and durable application volumes. Runtime diff --git a/tools/tht/internal/backup/restore_test.go b/tools/tht/internal/backup/restore_test.go index ddba356d..251bd665 100644 --- a/tools/tht/internal/backup/restore_test.go +++ b/tools/tht/internal/backup/restore_test.go @@ -120,7 +120,7 @@ func TestRestoreStoppedInstallationRunsCheckpointRestoreAndVerification(t *testi var events []string var checkpointRequest CreateRequest deps := restoreTestDependencies(t, runner) - deps.checkpointLocked = func(_ context.Context, _ config.Installation, request CreateRequest) (Result, error) { + deps.checkpoint = func(_ context.Context, _ *lifecycle.Transaction, _ config.Installation, request CreateRequest) (Result, error) { events = append(events, "checkpoint") checkpointRequest = request return Result{Path: "/tmp/checkpoint.zip"}, nil @@ -133,9 +133,9 @@ func TestRestoreStoppedInstallationRunsCheckpointRestoreAndVerification(t *testi events = append(events, "cleanup-checkpoint") return nil } - deps.acquireLock = func(config.Installation) (restoreLock, error) { + deps.acquireTransaction = func(target config.Installation) (*lifecycle.Transaction, error) { events = append(events, "lock") - return fakeRestoreLock{release: func() { events = append(events, "unlock") }}, nil + return lifecycle.AcquireTransaction(target) } deps.restoreFile = func(_ context.Context, _ config.Installation, entry ArchiveEntryMetadata, _ io.Reader) error { events = append(events, "file:"+entry.Path) @@ -159,9 +159,12 @@ func TestRestoreStoppedInstallationRunsCheckpointRestoreAndVerification(t *testi if !checkpointRequest.IncludeSecrets || !checkpointRequest.Confirm { t.Fatalf("checkpoint request = %#v, want private confirmed secret-aware checkpoint", checkpointRequest) } - if got, want := events, []string{"lock", "checkpoint", "prepare-recovery", "file:configuration/operator.env", "health", "doctor", "pi", "workspace", "cleanup-checkpoint", "unlock"}; !equalStrings(got, want) { + if got, want := events, []string{"lock", "checkpoint", "prepare-recovery", "file:configuration/operator.env", "health", "doctor", "pi", "workspace", "cleanup-checkpoint"}; !equalStrings(got, want) { t.Fatalf("restore events = %v, want %v", got, want) } + if err := lifecycleLockFreeAfterTerminalRestore(installation); err != nil { + t.Fatal(err) + } } func TestRestoreClosesTargetArchiveBeforeReleasingLifecycleLock(t *testing.T) { @@ -179,22 +182,85 @@ func TestRestoreClosesTargetArchiveBeforeReleasingLifecycleLock(t *testing.T) { target = result return result, err } - unlockBeforeTargetClose := false - deps.acquireLock = func(config.Installation) (restoreLock, error) { - return fakeRestoreLock{release: func() { - if target.archive != nil && target.archive.file != nil { - if _, err := target.archive.file.Stat(); err == nil { - unlockBeforeTargetClose = true - } - } - }}, nil - } + deps.acquireTransaction = lifecycle.AcquireTransaction if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err != nil { t.Fatal(err) } - if unlockBeforeTargetClose { - t.Fatal("restore released its lifecycle lock before closing the target archive") + if target.archive != nil && target.archive.file != nil { + if _, err := target.archive.file.Stat(); err == nil { + t.Fatal("restore did not close the target archive") + } + } + if err := lifecycleLockFreeAfterTerminalRestore(installation); err != nil { + t.Fatal(err) + } +} + +func TestRestoreAcquiresLifecycleLockBeforeTargetDependentPreflight(t *testing.T) { + installation := preflightTestInstallation(t) + archive := filepath.Join(t.TempDir(), "restore.zip") + writePreflightArchive(t, archive, preflightArchiveSpec{ + entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("safe")}}, + }) + + runner := newBackupRunner(installation, false) + deps := restoreTestDependencies(t, runner) + deps.acquireTransaction = lifecycle.AcquireTransaction + deps.preflight = func(ctx context.Context, target config.Installation, request PreflightRequest) (PreflightResult, error) { + probe, err := lifecycle.Acquire(target) + if err == nil { + _ = probe.Release() + return PreflightResult{}, errors.New("target-dependent preflight ran before lifecycle lock acquisition") + } + if !errors.Is(err, lifecycle.ErrLocked) { + return PreflightResult{}, fmt.Errorf("probe lifecycle lock: %w", err) + } + return Preflight(ctx, target, request, permissivePreflightDependencies()) + } + + if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err != nil { + t.Fatalf("Restore() error = %v, want preflight protected by lifecycle lock", err) + } + if err := lifecycleLockFreeAfterTerminalRestore(installation); err != nil { + t.Fatal(err) + } +} + +func TestRestoreStagesArchiveAfterCheckpointAndRejectsMutation(t *testing.T) { + installation := preflightTestInstallation(t) + archive := filepath.Join(t.TempDir(), "restore.zip") + writePreflightArchive(t, archive, preflightArchiveSpec{ + entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("before")}}, + }) + + runner := newBackupRunner(installation, false) + deps := restoreTestDependencies(t, runner) + deps.acquireTransaction = lifecycle.AcquireTransaction + deps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) { + writePreflightArchive(t, archive, preflightArchiveSpec{ + entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("after!")}}, + }) + return Result{Path: filepath.Join(t.TempDir(), "checkpoint.zip")}, nil + } + var restored [][]byte + deps.restoreFile = func(_ context.Context, _ config.Installation, _ ArchiveEntryMetadata, stream io.Reader) error { + body, err := io.ReadAll(stream) + if err != nil { + return err + } + restored = append(restored, body) + return nil + } + + if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err == nil || !strings.Contains(err.Error(), "changed") { + t.Fatalf("Restore() error = %v, want archive mutation refusal", err) + } + if len(restored) != 0 { + t.Fatalf("restore applied mutated archive payloads: %q", restored) + } + if err := lifecycleLockFreeAfterTerminalRestore(installation); err != nil { + t.Fatal(err) } } @@ -297,8 +363,8 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t caller, cancel := context.WithCancel(context.Background()) defer cancel() deps := restoreTestDependencies(t, runner) - deps.acquireLock = func(target config.Installation) (restoreLock, error) { return lifecycle.Acquire(target) } - deps.checkpointLocked = func(context.Context, config.Installation, CreateRequest) (Result, error) { + deps.acquireTransaction = lifecycle.AcquireTransaction + deps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) { gate("checkpoint") return Result{Path: filepath.Join(t.TempDir(), "checkpoint.zip")}, nil } @@ -369,8 +435,8 @@ func competingRestoreAndBackupEntry(installation config.Installation, archive st checkpointCalled := false restoreDeps := restoreTestDependencies(t, newBackupRunner(installation, false)) - restoreDeps.acquireLock = func(target config.Installation) (restoreLock, error) { return lifecycle.Acquire(target) } - restoreDeps.checkpointLocked = func(context.Context, config.Installation, CreateRequest) (Result, error) { + restoreDeps.acquireTransaction = lifecycle.AcquireTransaction + restoreDeps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) { checkpointCalled = true return Result{Path: filepath.Join(t.TempDir(), "competing-checkpoint.zip")}, nil } @@ -408,8 +474,8 @@ func TestRestoreCannotApplyAStaleCheckpointOverAnInterleavedRestore(t *testing.T continueCheckpoint := make(chan struct{}) firstRunner := newBackupRunner(installation, true) firstDeps := restoreTestDependencies(t, firstRunner) - firstDeps.acquireLock = func(target config.Installation) (restoreLock, error) { return lifecycle.Acquire(target) } - firstDeps.checkpointLocked = func(context.Context, config.Installation, CreateRequest) (Result, error) { + firstDeps.acquireTransaction = lifecycle.AcquireTransaction + firstDeps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) { checkpointState = targetState close(checkpointEntered) <-continueCheckpoint @@ -439,8 +505,8 @@ func TestRestoreCannotApplyAStaleCheckpointOverAnInterleavedRestore(t *testing.T interleavedCheckpoint := false interleavedMutation := false interleavedDeps := restoreTestDependencies(t, newBackupRunner(installation, false)) - interleavedDeps.acquireLock = func(target config.Installation) (restoreLock, error) { return lifecycle.Acquire(target) } - interleavedDeps.checkpointLocked = func(context.Context, config.Installation, CreateRequest) (Result, error) { + interleavedDeps.acquireTransaction = lifecycle.AcquireTransaction + interleavedDeps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) { interleavedCheckpoint = true return Result{Path: filepath.Join(t.TempDir(), "interleaved-checkpoint.zip")}, nil } @@ -542,7 +608,7 @@ func TestRestorePreflightFailureDoesNotMutateTarget(t *testing.T) { return PreflightResult{}, preflightErr } checkpointCalls := 0 - deps.checkpointLocked = func(context.Context, config.Installation, CreateRequest) (Result, error) { + deps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) { checkpointCalls++ return Result{}, nil } @@ -567,7 +633,7 @@ func TestRestoreCheckpointFailureDoesNotMutateTarget(t *testing.T) { runner := newBackupRunner(installation, true) deps := restoreTestDependencies(t, runner) checkpointErr := errors.New("checkpoint unavailable") - deps.checkpointLocked = func(context.Context, config.Installation, CreateRequest) (Result, error) { + deps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) { return Result{}, checkpointErr } restoredFiles := 0 @@ -591,7 +657,7 @@ func TestRestoreFileFailureRollsBackSecretAwareCheckpointBeforeCleanup(t *testin runner := newBackupRunner(installation, true) deps := restoreTestDependencies(t, runner) fileErr := errors.New("cannot restore operator configuration") - deps.checkpointLocked = func(context.Context, config.Installation, CreateRequest) (Result, error) { + deps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) { return Result{Path: "/tmp/recovery.zip"}, nil } var events []string @@ -1095,7 +1161,7 @@ func TestRestoreReleasesBarrierOnlyAfterVerifiedRecoveryFromLostResponse(t *test } runner := &commandFailureRunner{fakeBackupRunner: backing, failures: []*commandFailure{&failure}} deps := restoreTestDependencies(t, runner) - deps.checkpointLocked = func(context.Context, config.Installation, CreateRequest) (Result, error) { + deps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) { return Result{Path: "/tmp/recovery.zip"}, nil } deps.prepareRecovery = func(context.Context, config.Installation, string) (PreflightResult, error) { @@ -1322,10 +1388,6 @@ func (runner *lifecycleGateRunner) SessionInventoryScope() string { return runner.fakeBackupRunner.SessionInventoryScope() } -type fakeRestoreLock struct { - release func() -} - type authenticationStateResetRunner struct { args []string result compose.Result @@ -1431,13 +1493,6 @@ func TestVerifyRestoreWorkspaceRejectsInvalidOperatorResults(t *testing.T) { } } -func (lock fakeRestoreLock) Release() error { - if lock.release != nil { - lock.release() - } - return nil -} - func equalStrings(got, want []string) bool { if len(got) != len(want) { return false @@ -1456,18 +1511,18 @@ func restoreTestDependencies(t *testing.T, runner archiveRunner) restoreDependen preflight: func(ctx context.Context, installation config.Installation, request PreflightRequest) (PreflightResult, error) { return Preflight(ctx, installation, request, permissivePreflightDependencies()) }, - checkpointLocked: func(context.Context, config.Installation, CreateRequest) (Result, error) { + checkpoint: func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) { return Result{Path: "/tmp/default-checkpoint.zip"}, nil }, prepareRecovery: func(context.Context, config.Installation, string) (PreflightResult, error) { return PreflightResult{}, nil }, - recover: func(context.Context, config.Installation, PreflightResult, bool) error { return nil }, - cleanupCheckpoint: func(string) error { return nil }, - acquireLock: func(config.Installation) (restoreLock, error) { return fakeRestoreLock{}, nil }, - runner: runner, - sleep: func(time.Duration) {}, - restoreFile: func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return nil }, + recover: func(context.Context, config.Installation, PreflightResult, bool) error { return nil }, + cleanupCheckpoint: func(string) error { return nil }, + acquireTransaction: lifecycle.AcquireTransaction, + runner: runner, + sleep: func(time.Duration) {}, + restoreFile: func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return nil }, restoreVolume: func(context.Context, config.Installation, VolumeMetadata, io.Reader) error { return nil }, diff --git a/tools/tht/internal/lifecycle/lock.go b/tools/tht/internal/lifecycle/lock.go index f919e32b..7d6299bb 100644 --- a/tools/tht/internal/lifecycle/lock.go +++ b/tools/tht/internal/lifecycle/lock.go @@ -16,8 +16,10 @@ import ( ) var ( - ErrLocked = errors.New("another lifecycle operation is already running for this installation") - ErrOwnership = errors.New("lifecycle lock ownership changed; refusing to remove it") + ErrLocked = errors.New("another lifecycle operation is already running for this installation") + ErrOwnership = errors.New("lifecycle lock ownership changed; refusing to remove it") + ErrTransactionInactive = errors.New("lifecycle transaction capability is not active") + ErrTransactionInstallation = errors.New("lifecycle transaction capability belongs to another installation") ) const lockFileName = "lifecycle.lock.owner.json" @@ -36,6 +38,14 @@ type Lock struct { released bool } +// Transaction is an opaque, installation-bound capability for work that must run while a +// lifecycle lock remains owned. Its fields are deliberately private so callers can obtain one +// only through AcquireTransaction. +type Transaction struct { + lock *Lock + controlDirectory string +} + // Acquire obtains the shared lock used by backup, restore, Pi lifecycle and product updates. func Acquire(installation config.Installation) (*Lock, error) { directory := installation.ControlDirectory() @@ -76,6 +86,39 @@ func Acquire(installation config.Installation) (*Lock, error) { return &Lock{path: path, token: token}, nil } +// AcquireTransaction obtains a lifecycle lock and returns the capability required by callers +// that perform nested work inside the same non-reentrant transaction. +func AcquireTransaction(installation config.Installation) (*Transaction, error) { + lock, err := Acquire(installation) + if err != nil { + return nil, err + } + return &Transaction{ + lock: lock, + controlDirectory: filepath.Clean(installation.ControlDirectory()), + }, nil +} + +// Verify refuses a nil, released, replaced, or foreign-installation capability before a nested +// lifecycle operation can begin. +func (transaction *Transaction) Verify(installation config.Installation) error { + if transaction == nil || transaction.lock == nil { + return ErrTransactionInactive + } + if transaction.controlDirectory != filepath.Clean(installation.ControlDirectory()) { + return ErrTransactionInstallation + } + return transaction.lock.verifyHeld() +} + +// Release relinquishes the lifecycle lock associated with this transaction capability. +func (transaction *Transaction) Release() error { + if transaction == nil || transaction.lock == nil { + return nil + } + return transaction.lock.Release() +} + // Path returns the installation-private owner-file path for diagnostics and tests. func (lock *Lock) Path() string { if lock == nil { @@ -111,3 +154,26 @@ func (lock *Lock) Release() error { lock.released = true return nil } + +func (lock *Lock) verifyHeld() error { + if lock == nil { + return ErrTransactionInactive + } + lock.mu.Lock() + defer lock.mu.Unlock() + if lock.released { + return ErrTransactionInactive + } + contents, err := os.ReadFile(lock.path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return ErrOwnership + } + return fmt.Errorf("read lifecycle lock owner: %w", err) + } + var current owner + if json.Unmarshal(contents, ¤t) != nil || current.Token == "" || current.Token != lock.token { + return ErrOwnership + } + return nil +} diff --git a/tools/tht/internal/lifecycle/lock_test.go b/tools/tht/internal/lifecycle/lock_test.go index 196e97ef..2fb4babc 100644 --- a/tools/tht/internal/lifecycle/lock_test.go +++ b/tools/tht/internal/lifecycle/lock_test.go @@ -48,3 +48,27 @@ func TestLifecycleLockReleaseDoesNotRemoveAnotherOwnersFile(t *testing.T) { t.Fatalf("foreign lock was removed: %v", err) } } + +func TestTransactionCapabilityIsInstallationBoundAndExpiresOnRelease(t *testing.T) { + root := t.TempDir() + installation := config.Installation{ProjectDirectory: root, Path: filepath.Join(root, "thothii-installation.yaml")} + transaction, err := AcquireTransaction(installation) + if err != nil { + t.Fatal(err) + } + if err := transaction.Verify(installation); err != nil { + t.Fatalf("Verify() active capability error = %v", err) + } + + otherRoot := t.TempDir() + other := config.Installation{ProjectDirectory: otherRoot, Path: filepath.Join(otherRoot, "thothii-installation.yaml")} + if err := transaction.Verify(other); !errors.Is(err, ErrTransactionInstallation) { + t.Fatalf("Verify() for another installation error = %v, want ErrTransactionInstallation", err) + } + if err := transaction.Release(); err != nil { + t.Fatal(err) + } + if err := transaction.Verify(installation); !errors.Is(err, ErrTransactionInactive) { + t.Fatalf("Verify() after Release() error = %v, want ErrTransactionInactive", err) + } +}