diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index b94c7600..aa0611f0 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -103,7 +103,7 @@ jobs: - name: Install Chromium for Playwright working-directory: frontend run: npx playwright install --with-deps chromium - - name: Run authentication browser smoke + - name: Run authentication and authenticated F1 browser smoke run: bash scripts/authentication-smoke.sh linux-docker: diff --git a/frontend/e2e/auth.spec.ts b/frontend/e2e/auth.spec.ts index c21e199e..6d94437b 100644 --- a/frontend/e2e/auth.spec.ts +++ b/frontend/e2e/auth.spec.ts @@ -22,6 +22,7 @@ test("the loopback fixture exposes signed OIDC discovery, device authorization, deviceAuthorization: true, deviceToken: true, groupList: true, + runtimeCredential: process.env.THT_TASK15_SENTINEL !== undefined, }); }); diff --git a/frontend/e2e/fixtures/auth-stack.mjs b/frontend/e2e/fixtures/auth-stack.mjs index c12454ba..1a730fde 100644 --- a/frontend/e2e/fixtures/auth-stack.mjs +++ b/frontend/e2e/fixtures/auth-stack.mjs @@ -18,8 +18,8 @@ const thtRoot = join(repositoryRoot, "tools", "tht"); const fakePi = join(frontendRoot, "e2e", "fixtures", "fake-pi.mjs"); const fakeTht = join(frontendRoot, "e2e", "fixtures", "fake-tht.mjs"); const FIXTURE_CLIENT_ID = "thothii-e2e-client"; -const FIXTURE_CLIENT_SECRET = "e2e-client-secret-not-a-production-secret"; -const FIXTURE_API_TOKEN = "e2e-group-catalog-token-not-a-production-secret"; +const DEFAULT_FIXTURE_CLIENT_SECRET = "e2e-client-secret-not-a-production-secret"; +const DEFAULT_FIXTURE_API_TOKEN = "e2e-group-catalog-token-not-a-production-secret"; const WRONG_CLIENT_ID = "thothii-e2e-unregistered-client"; const WRONG_CLIENT_SECRET = "e2e-wrong-client-secret-not-production"; const WRONG_API_TOKEN = "e2e-wrong-api-token-not-production"; @@ -31,6 +31,21 @@ function safeError(code) { return new Error(code); } +function resolveFixtureCredentials() { + const runtimeCredential = process.env.THT_TASK15_SENTINEL; + if (runtimeCredential === undefined) { + return Object.freeze({ + clientSecret: DEFAULT_FIXTURE_CLIENT_SECRET, + apiToken: DEFAULT_FIXTURE_API_TOKEN, + runtime: false, + }); + } + if (runtimeCredential.length < 24 || runtimeCredential.length > 512 || /[\r\n\0]/u.test(runtimeCredential)) { + throw safeError("e2e_runtime_fixture_credential_invalid"); + } + return Object.freeze({ clientSecret: runtimeCredential, apiToken: runtimeCredential, runtime: true }); +} + function buildAuthenticationStorageBridge(output) { const result = spawn("go", ["build", "-o", output, "./cmd/tht"], { cwd: thtRoot, @@ -322,7 +337,7 @@ function cleanBackendEnvironment(overrides) { "SETTINGS_FILE", "THT_MAINTENANCE_FILE", "THT_WORKSPACE_REGISTRY_ROOT", "THT_WORKSPACE_SECRET_STORE_ROOT", "THT_WORKSPACE_SECRET_RUNTIME_ROOT", "THT_WORKSPACE_GIT_REMOTE", "THT_WORKSPACE_GIT_BRANCH", "THT_WORKSPACE_SECRET_ROOTS", "THT_WORKSPACE_INSTALLATION_ID", "THT_LEGACY_WORKSPACE_MODE", - "THT_DWH_PRECHECK", "THT_INTERNAL_QDRANT_URL", "THT_CONFIG", + "THT_DWH_PRECHECK", "THT_INTERNAL_QDRANT_URL", "THT_CONFIG", "THT_TASK15_SENTINEL", ]) delete env[name]; for (const name of Object.keys(env)) { if (name.startsWith("THT_WS_")) delete env[name]; @@ -331,6 +346,7 @@ function cleanBackendEnvironment(overrides) { } export async function createAuthenticationStack({ withF1Workspace = false } = {}) { + const credentials = resolveFixtureCredentials(); const root = mkdtempSync(join(realpathSync(tmpdir()), "thothii-auth-e2e-")); secureDirectory(root); const stateRoot = join(root, "auth-state"); @@ -358,10 +374,10 @@ export async function createAuthenticationStack({ withF1Workspace = false } = {} directory: providerRoot, registration: { clientId: FIXTURE_CLIENT_ID, - clientSecret: FIXTURE_CLIENT_SECRET, + clientSecret: credentials.clientSecret, redirectUri: new URL("/api/auth/oidc/callback", publicUrl).href, }, - apiToken: FIXTURE_API_TOKEN, + apiToken: credentials.apiToken, }); await buildAuthenticationStorageBridge(authStorageBinary); const localPassword = "e2e-local-password"; @@ -395,8 +411,8 @@ export async function createAuthenticationStack({ withF1Workspace = false } = {} })); function writeOidcSecrets(variant) { if (!OIDC_CREDENTIAL_VARIANTS.has(variant)) throw safeError("e2e_oidc_credential_variant_invalid"); - const selectedClientSecret = variant === "wrong-client-secret" ? WRONG_CLIENT_SECRET : FIXTURE_CLIENT_SECRET; - const selectedApiToken = variant === "wrong-api-token" ? WRONG_API_TOKEN : FIXTURE_API_TOKEN; + const selectedClientSecret = variant === "wrong-client-secret" ? WRONG_CLIENT_SECRET : credentials.clientSecret; + const selectedApiToken = variant === "wrong-api-token" ? WRONG_API_TOKEN : credentials.apiToken; writeSecure(secretsFile, [ `THT_OIDC_CLIENT_SECRET=${selectedClientSecret}`, `THT_AUTHENTIK_API_TOKEN=${selectedApiToken}`, @@ -539,7 +555,7 @@ export async function createAuthenticationStack({ withF1Workspace = false } = {} method: "POST", body: new URLSearchParams({ client_id: FIXTURE_CLIENT_ID, - client_secret: FIXTURE_CLIENT_SECRET, + client_secret: credentials.clientSecret, }).toString(), }); const deviceCode = typeof device.body?.device_code === "string" ? device.body.device_code : ""; @@ -548,12 +564,12 @@ export async function createAuthenticationStack({ withF1Workspace = false } = {} body: new URLSearchParams({ grant_type: "urn:ietf:params:oauth:grant-type:device_code", client_id: FIXTURE_CLIENT_ID, - client_secret: FIXTURE_CLIENT_SECRET, + client_secret: credentials.clientSecret, device_code: deviceCode, }).toString(), }); const groups = await providerJson(`${provider.baseUrl}/api/v3/core/groups/?name=fixture-users`, provider.caFile, { - headers: { authorization: `Bearer ${FIXTURE_API_TOKEN}` }, + headers: { authorization: `Bearer ${credentials.apiToken}` }, }); return { discovery: discovery.status === 200 && discovery.body?.issuer === provider.issuer, @@ -562,6 +578,7 @@ export async function createAuthenticationStack({ withF1Workspace = false } = {} && typeof device.body?.verification_uri === "string", deviceToken: deviceToken.status === 200 && typeof deviceToken.body?.id_token === "string", groupList: groups.status === 200 && groups.body?.results?.[0]?.name === "fixture-users", + runtimeCredential: credentials.runtime, }; }, setOidcIdentity(identity) { diff --git a/scripts/authentication-smoke.sh b/scripts/authentication-smoke.sh index 77e74e7c..3b49bc04 100755 --- a/scripts/authentication-smoke.sh +++ b/scripts/authentication-smoke.sh @@ -30,6 +30,7 @@ output="$temporary_root/playwright" mkdir -p "$output" sentinel="task15-sentinel-$(openssl rand -hex 24)" export THT_TASK15_SENTINEL="$sentinel" +readonly browser_specs=(e2e/auth.spec.ts e2e/f1.spec.ts) sanitize_failure_log() { sed -E \ @@ -41,7 +42,7 @@ sanitize_failure_log() { if ! ( cd "$root/frontend" - THT_E2E_AUTH_STACK=1 "$playwright" test e2e/auth.spec.ts --workers=1 --output="$output" + THT_E2E_AUTH_STACK=1 "$playwright" test "${browser_specs[@]}" --workers=1 --output="$output" ) >"$log" 2>&1; then echo "authentication smoke: hermetic browser suite failed" >&2 sanitize_failure_log @@ -53,4 +54,4 @@ if rg -a -Fq -- "$sentinel" "$log" "$output"; then exit 1 fi -printf 'authentication smoke: hermetic OIDC/browser suite passed on Node %s\n' "$actual_node" +printf 'authentication smoke: hermetic OIDC/F1 browser suite passed on Node %s\n' "$actual_node" diff --git a/scripts/unified-deployment-smoke.sh b/scripts/unified-deployment-smoke.sh index f8902188..acccff6e 100755 --- a/scripts/unified-deployment-smoke.sh +++ b/scripts/unified-deployment-smoke.sh @@ -986,9 +986,30 @@ task13_core_id() { } task13_assert_maintenance_auth_isolation() { + local rendered + rendered="$TASK13_TMP/maintenance-auth-isolation.json" + if ! task13_compose config --format json >"$rendered" 2>>"$TASK13_LOG"; then + task13_log_failure "render workspace maintenance mount isolation" + fi + node - "$rendered" <<'NODE' +const config = JSON.parse(require("node:fs").readFileSync(process.argv[2], "utf8")); +const maintenance = config.services?.["workspace-maintenance"]; +if (!maintenance || !Array.isArray(maintenance.volumes)) process.exit(1); +if (maintenance.volumes.some((mount) => mount?.target === "/run/thothii-auth" || mount?.target === "/data/auth")) { + process.exit(1); +} +NODE + task13_compose_logged "seed core authentication isolation sentinel" exec -T core sh -ceu \ + ': > /data/auth/task13-maintenance-isolation-sentinel' task13_compose_logged "workspace maintenance auth isolation" \ --profile workspace-maintenance run --rm --no-deps --entrypoint sh workspace-maintenance -ceu \ - 'test ! -e /run/thothii-auth && test ! -e /data/auth' + 'test ! -e /run/thothii-auth + test -d /data/auth + test ! -e /data/auth/task13-maintenance-isolation-sentinel' + task13_compose_logged "verify core authentication isolation sentinel" exec -T core sh -ceu \ + 'test -f /data/auth/task13-maintenance-isolation-sentinel' + task13_compose_logged "remove core authentication isolation sentinel" exec -T core sh -ceu \ + 'rm -f /data/auth/task13-maintenance-isolation-sentinel' } task13_assert_local_auth_lifecycle() { diff --git a/tools/tht/internal/backup/preflight.go b/tools/tht/internal/backup/preflight.go index 04410d62..ff299c6b 100644 --- a/tools/tht/internal/backup/preflight.go +++ b/tools/tht/internal/backup/preflight.go @@ -82,6 +82,8 @@ type PreflightResult struct { Manifest Manifest Entries []ArchiveEntryMetadata archive *verifiedArchive + stagingRoot string + freeBytes func(string) (uint64, error) } type verifiedArchive struct { @@ -189,7 +191,8 @@ func Preflight(ctx context.Context, installation config.Installation, request Pr return PreflightResult{}, errors.New("restore staging requirement exceeds supported size") } requiredWithStaging := requiredBytes + stagingBytes - freeBytes, err := dependencies.FreeBytes(installation.ProjectDirectory) + stagingCapacityPath := installation.ControlDirectory() + freeBytes, err := dependencies.FreeBytes(stagingCapacityPath) if err != nil { return PreflightResult{}, fmt.Errorf("check free disk space: %w", err) } @@ -219,7 +222,9 @@ func Preflight(ctx context.Context, installation config.Installation, request Pr return PreflightResult{ ArchivePath: archivePath, ArchiveSize: openedInfo.Size(), RequiredBytes: requiredBytes, Manifest: manifest, Entries: metadata, - archive: &verifiedArchive{file: archiveFile, info: openedInfo, digest: finalDigest, limits: limits}, + archive: &verifiedArchive{file: archiveFile, info: openedInfo, digest: finalDigest, limits: limits}, + stagingRoot: filepath.Join(stagingCapacityPath, "restore-staging"), + freeBytes: dependencies.FreeBytes, }, nil } @@ -259,7 +264,20 @@ func (result PreflightResult) StageArchive(ctx context.Context) (_ *stagedArchiv if err != nil { return nil, err } - directory, err := os.MkdirTemp("", "tht-restore-stage-") + if result.stagingRoot == "" || result.freeBytes == nil { + return nil, errors.New("backup archive has no controlled staging reservation") + } + if err := ensurePrivateStagingRoot(result.stagingRoot); err != nil { + return nil, fmt.Errorf("create private restore staging root: %w", err) + } + freeBytes, err := result.freeBytes(result.stagingRoot) + if err != nil { + return nil, errors.New("check private restore staging capacity") + } + if result.ArchiveSize < 0 || freeBytes < uint64(result.ArchiveSize) { + return nil, errors.New("insufficient free disk space for private restore staging archive") + } + directory, err := os.MkdirTemp(result.stagingRoot, "archive-") if err != nil { return nil, errors.New("create private restore staging directory") } @@ -324,6 +342,27 @@ func (result PreflightResult) StageArchive(ctx context.Context) (_ *stagedArchiv return staged, nil } +func ensurePrivateStagingRoot(root string) error { + if !filepath.IsAbs(root) || filepath.Clean(root) != root { + return errors.New("restore staging root is invalid") + } + if err := os.Mkdir(root, 0o700); err != nil && !errors.Is(err, os.ErrExist) { + return errors.New("restore staging root is unavailable") + } + info, err := os.Lstat(root) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("restore staging root is unsafe") + } + if err := os.Chmod(root, 0o700); err != nil { + return errors.New("restore staging root cannot be protected") + } + info, err = os.Lstat(root) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm() != 0o700 { + return errors.New("restore staging root protection is invalid") + } + return nil +} + // Close removes only the staging file and directory created by StageArchive. func (staged *stagedArchive) Close() error { if staged == nil { diff --git a/tools/tht/internal/backup/preflight_test.go b/tools/tht/internal/backup/preflight_test.go index fa2c88e1..86d19466 100644 --- a/tools/tht/internal/backup/preflight_test.go +++ b/tools/tht/internal/backup/preflight_test.go @@ -57,6 +57,49 @@ func TestPreflightReturnsValidatedMetadataAndCallsAllTargetChecksWithoutExtracti } } +func TestPreflightStagesOnTheAccountedInstallationFilesystemInsteadOfTMPDIR(t *testing.T) { + installation := preflightTestInstallation(t) + if err := os.MkdirAll(installation.ControlDirectory(), 0o700); err != nil { + t.Fatal(err) + } + archive := filepath.Join(t.TempDir(), "valid.zip") + writePreflightArchive(t, archive, preflightArchiveSpec{ + entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("safe")}}, + }) + + blockedTMPDIR := filepath.Join(t.TempDir(), "not-a-directory") + if err := os.WriteFile(blockedTMPDIR, []byte("blocked"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("TMPDIR", blockedTMPDIR) + + var capacityTargets []string + dependencies := permissivePreflightDependencies() + dependencies.FreeBytes = func(target string) (uint64, error) { + capacityTargets = append(capacityTargets, target) + return 1 << 30, nil + } + result, err := Preflight(context.Background(), installation, PreflightRequest{Archive: archive, Confirm: true}, dependencies) + if err != nil { + t.Fatal(err) + } + defer result.CloseArchive() + + staged, err := result.StageArchive(context.Background()) + if err != nil { + t.Fatal(err) + } + defer staged.Close() + + if len(capacityTargets) == 0 || capacityTargets[0] != installation.ControlDirectory() { + t.Fatalf("free-space targets = %q, want installation control directory %q", capacityTargets, installation.ControlDirectory()) + } + stagingRoot := filepath.Join(installation.ControlDirectory(), "restore-staging") + if relative, err := filepath.Rel(stagingRoot, staged.directory); err != nil || relative == "." || strings.HasPrefix(relative, "..") { + t.Fatalf("staging directory = %q, want a child of %q", staged.directory, stagingRoot) + } +} + func TestPreflightRejectsAdversarialArchiveEntriesAndManifestIdentity(t *testing.T) { installation := preflightTestInstallation(t) tests := []struct { @@ -359,6 +402,9 @@ func TestPreflightRevalidationRefusesAnArchivePathThatWasReplaced(t *testing.T) func TestPreflightStagesArchiveIntoImmutablePrivateBytes(t *testing.T) { installation := preflightTestInstallation(t) + if err := os.MkdirAll(installation.ControlDirectory(), 0o700); err != nil { + t.Fatal(err) + } archive := filepath.Join(t.TempDir(), "checked.zip") writePreflightArchive(t, archive, preflightArchiveSpec{ entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("before")}}, diff --git a/tools/tht/internal/backup/restore.go b/tools/tht/internal/backup/restore.go index 8c15dff3..d68131de 100644 --- a/tools/tht/internal/backup/restore.go +++ b/tools/tht/internal/backup/restore.go @@ -38,7 +38,7 @@ type restoreDependencies struct { // public Create, which would re-acquire the non-reentrant lock and deadlock the transaction. checkpoint func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) prepareRecovery func(context.Context, config.Installation, string) (PreflightResult, error) - recover func(context.Context, config.Installation, PreflightResult, bool) error + recover func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error cleanupCheckpoint func(string) error acquireTransaction func(config.Installation) (*lifecycle.Transaction, error) runner archiveRunner @@ -119,11 +119,42 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati } defer recovery.CloseArchive() + // Both immutable archives are staged while the lifecycle transaction is held and before + // maintenance, service stops, or destination writes. Keeping both files open reserves their + // combined staging capacity, so checkpoint recovery never needs a new allocation after a + // candidate mutation has begun. + candidateStage, err := preflight.StageArchive(ctx) + if err != nil { + cleanupErr := deps.cleanupCheckpoint(checkpoint.Path) + return result, errors.Join(err, cleanupErr) + } + defer func() { + if closeErr := candidateStage.Close(); closeErr != nil { + result = RestoreResult{} + resultErr = errors.Join(resultErr, closeErr) + } + }() + recoveryStage, err := recovery.StageArchive(ctx) + if err != nil { + cleanupErr := deps.cleanupCheckpoint(checkpoint.Path) + return result, errors.Join(err, cleanupErr) + } + defer func() { + if closeErr := recoveryStage.Close(); closeErr != nil { + result = RestoreResult{} + resultErr = errors.Join(resultErr, closeErr) + } + }() + if err := ensureCombinedRestoreCapacity(preflight, recovery); err != nil { + cleanupErr := deps.cleanupCheckpoint(checkpoint.Path) + return result, errors.Join(err, cleanupErr) + } + state := restoreTransactionState{} defer func() { if state.recoveryRequired(resultErr) { recoveryContext, cancel := boundedCleanupContext() - recoveryErr := deps.recover(recoveryContext, installation, recovery, state.wasRunning) + recoveryErr := deps.recover(recoveryContext, installation, recovery, recoveryStage, state.wasRunning) cancel() if recoveryErr != nil { resultErr = errors.Join(resultErr, fmt.Errorf("restore recovery checkpoint: %w", recoveryErr)) @@ -212,18 +243,8 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati return result, err } } - staged, err := preflight.StageArchive(ctx) - if err != nil { - return result, err - } - defer func() { - if closeErr := staged.Close(); closeErr != nil { - result = RestoreResult{} - resultErr = errors.Join(resultErr, closeErr) - } - }() state.mutated = true - if err := restoreVerifiedEntries(ctx, installation, preflight, staged.file, deps.restoreFile, deps.restoreVolume); err != nil { + if err := restoreVerifiedEntries(ctx, installation, preflight, candidateStage.file, deps.restoreFile, deps.restoreVolume); err != nil { return result, err } if err := deps.resetAuthenticationState(ctx, installation, deps.runner); err != nil { @@ -244,6 +265,24 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati return result, nil } +func ensureCombinedRestoreCapacity(candidate, recovery PreflightResult) error { + if candidate.freeBytes == nil || candidate.stagingRoot == "" || candidate.stagingRoot != recovery.stagingRoot { + return errors.New("candidate and recovery archives do not share controlled restore staging") + } + if recovery.RequiredBytes > ^uint64(0)-candidate.RequiredBytes { + return errors.New("combined candidate and recovery restore capacity exceeds supported size") + } + required := candidate.RequiredBytes + recovery.RequiredBytes + freeBytes, err := candidate.freeBytes(candidate.stagingRoot) + if err != nil { + return errors.New("check combined candidate and recovery staging capacity") + } + if freeBytes < required { + return fmt.Errorf("insufficient free disk space for combined candidate and recovery staging: need %d bytes, have %d", required, freeBytes) + } + return nil +} + func verifyRestoreTransaction(ctx context.Context, installation config.Installation, deps restoreDependencies) error { for _, name := range []string{"health", "doctor", "pi", "workspace"} { check := deps.verify[name] diff --git a/tools/tht/internal/backup/restore_host.go b/tools/tht/internal/backup/restore_host.go index 64466aee..68edab64 100644 --- a/tools/tht/internal/backup/restore_host.go +++ b/tools/tht/internal/backup/restore_host.go @@ -75,8 +75,8 @@ func productionRestoreDependencies(installation config.Installation) restoreDepe deps.prepareRecovery = func(ctx context.Context, target config.Installation, path string) (PreflightResult, error) { return deps.preflight(ctx, target, PreflightRequest{Archive: path, Confirm: true, AllowExternalSecrets: true}) } - deps.recover = func(ctx context.Context, target config.Installation, recovery PreflightResult, wasRunning bool) error { - return recoverRestoreTransaction(ctx, target, recovery, wasRunning, deps) + deps.recover = func(ctx context.Context, target config.Installation, recovery PreflightResult, staged *stagedArchive, wasRunning bool) error { + return recoverRestoreTransaction(ctx, target, recovery, staged, wasRunning, deps) } return deps } @@ -88,7 +88,10 @@ func cleanupRecoveryCheckpoint(path string) error { return nil } -func recoverRestoreTransaction(ctx context.Context, installation config.Installation, recovery PreflightResult, wasRunning bool, deps restoreDependencies) (resultErr error) { +func recoverRestoreTransaction(ctx context.Context, installation config.Installation, recovery PreflightResult, staged *stagedArchive, wasRunning bool, deps restoreDependencies) (resultErr error) { + if staged == nil || staged.file == nil { + return errors.New("recovery checkpoint was not staged before restore mutation") + } if err := runCompose(ctx, installation, deps.runner, "stop"); err != nil { resultErr = errors.Join(resultErr, err) // The first stop may already have taken effect before Docker lost its response. Retry the @@ -97,15 +100,6 @@ func recoverRestoreTransaction(ctx context.Context, installation config.Installa return errors.Join(resultErr, retryErr) } } - staged, err := recovery.StageArchive(ctx) - if err != nil { - return errors.Join(resultErr, err) - } - defer func() { - if closeErr := staged.Close(); closeErr != nil { - resultErr = errors.Join(resultErr, closeErr) - } - }() if err := restoreVerifiedEntries(ctx, installation, recovery, staged.file, deps.restoreFile, deps.restoreVolume); err != nil { return errors.Join(resultErr, err) } diff --git a/tools/tht/internal/backup/restore_test.go b/tools/tht/internal/backup/restore_test.go index 251bd665..c9cd6bab 100644 --- a/tools/tht/internal/backup/restore_test.go +++ b/tools/tht/internal/backup/restore_test.go @@ -120,14 +120,15 @@ func TestRestoreStoppedInstallationRunsCheckpointRestoreAndVerification(t *testi var events []string var checkpointRequest CreateRequest deps := restoreTestDependencies(t, runner) + prepareRecovery := deps.prepareRecovery deps.checkpoint = func(_ context.Context, _ *lifecycle.Transaction, _ config.Installation, request CreateRequest) (Result, error) { events = append(events, "checkpoint") checkpointRequest = request return Result{Path: "/tmp/checkpoint.zip"}, nil } - deps.prepareRecovery = func(context.Context, config.Installation, string) (PreflightResult, error) { + deps.prepareRecovery = func(ctx context.Context, target config.Installation, path string) (PreflightResult, error) { events = append(events, "prepare-recovery") - return PreflightResult{}, nil + return prepareRecovery(ctx, target, path) } deps.cleanupCheckpoint = func(string) error { events = append(events, "cleanup-checkpoint") @@ -167,6 +168,49 @@ func TestRestoreStoppedInstallationRunsCheckpointRestoreAndVerification(t *testi } } +func TestRestoreRejectsCombinedCandidateAndRecoveryStagingCapacityBeforeMutation(t *testing.T) { + installation := preflightTestInstallation(t) + candidateArchive := restoreArchive(t) + recoveryArchive := restoreArchive(t) + runner := newBackupRunner(installation, false) + dependencies := restoreTestDependencies(t, runner) + + capacityChecks := 0 + preflightDependencies := permissivePreflightDependencies() + preflightDependencies.FreeBytes = func(string) (uint64, error) { + capacityChecks++ + if capacityChecks == 5 { + return 0, nil + } + return 1 << 30, nil + } + dependencies.preflight = func(ctx context.Context, target config.Installation, request PreflightRequest) (PreflightResult, error) { + return Preflight(ctx, target, request, preflightDependencies) + } + dependencies.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) { + return Result{Path: recoveryArchive}, nil + } + dependencies.prepareRecovery = func(ctx context.Context, target config.Installation, path string) (PreflightResult, error) { + return Preflight(ctx, target, PreflightRequest{Archive: path, Confirm: true, AllowExternalSecrets: true}, preflightDependencies) + } + mutated := false + dependencies.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { + mutated = true + return nil + } + + _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: candidateArchive, Confirm: true}, dependencies) + if err == nil || !strings.Contains(err.Error(), "staging") { + t.Fatalf("Restore() error = %v, want recovery staging capacity rejection", err) + } + if mutated { + t.Fatal("restore mutated the installation before reserving candidate and recovery staging capacity") + } + if capacityChecks != 5 { + t.Fatalf("free-space checks = %d, want candidate/recovery preflight and staging checks", capacityChecks) + } +} + func TestRestoreClosesTargetArchiveBeforeReleasingLifecycleLock(t *testing.T) { installation := preflightTestInstallation(t) archive := filepath.Join(t.TempDir(), "restore.zip") @@ -301,7 +345,7 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t gate("target-failure") return targetFailure } - deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error { + deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error { gate("recovery") runner.running, runner.coreRunning = true, true return nil @@ -318,7 +362,7 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t gate("target-failure") return targetFailure } - deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error { + deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error { gate("recovery-failure") return recoveryFailure } @@ -335,7 +379,7 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t cancel() return context.Canceled } - deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error { + deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error { gate("recovery") runner.running, runner.coreRunning = true, true return nil @@ -353,6 +397,11 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("safe")}}, }) runner := &lifecycleGateRunner{fakeBackupRunner: newBackupRunner(installation, true)} + recoveryArchive := filepath.Join(t.TempDir(), "recovery.zip") + writePreflightArchive(t, recoveryArchive, preflightArchiveSpec{ + installationID: fixture.installationID, + entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("checkpoint")}}, + }) stages := make(chan string) continueStage := make(chan struct{}) gate := func(stage string) { @@ -363,6 +412,9 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t caller, cancel := context.WithCancel(context.Background()) defer cancel() deps := restoreTestDependencies(t, runner) + deps.prepareRecovery = func(ctx context.Context, target config.Installation, _ string) (PreflightResult, error) { + return Preflight(ctx, target, PreflightRequest{Archive: recoveryArchive, Confirm: true, AllowExternalSecrets: true}, permissivePreflightDependencies()) + } deps.acquireTransaction = lifecycle.AcquireTransaction deps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) { gate("checkpoint") @@ -474,6 +526,14 @@ func TestRestoreCannotApplyAStaleCheckpointOverAnInterleavedRestore(t *testing.T continueCheckpoint := make(chan struct{}) firstRunner := newBackupRunner(installation, true) firstDeps := restoreTestDependencies(t, firstRunner) + recoveryArchive := filepath.Join(t.TempDir(), "first-recovery.zip") + writePreflightArchive(t, recoveryArchive, preflightArchiveSpec{ + installationID: fixture.installationID, + entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("checkpoint")}}, + }) + firstDeps.prepareRecovery = func(ctx context.Context, target config.Installation, _ string) (PreflightResult, error) { + return Preflight(ctx, target, PreflightRequest{Archive: recoveryArchive, Confirm: true, AllowExternalSecrets: true}, permissivePreflightDependencies()) + } firstDeps.acquireTransaction = lifecycle.AcquireTransaction firstDeps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) { checkpointState = targetState @@ -484,7 +544,7 @@ func TestRestoreCannotApplyAStaleCheckpointOverAnInterleavedRestore(t *testing.T firstDeps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return errors.New("first target mutation failed before changing state") } - firstDeps.recover = func(context.Context, config.Installation, PreflightResult, bool) error { + firstDeps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error { recoveryObserved = targetState targetState = checkpointState firstRunner.running, firstRunner.coreRunning = true, true @@ -661,7 +721,7 @@ func TestRestoreFileFailureRollsBackSecretAwareCheckpointBeforeCleanup(t *testin return Result{Path: "/tmp/recovery.zip"}, nil } var events []string - deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error { + deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error { events = append(events, "recover") return nil } @@ -701,7 +761,7 @@ func TestRestoreFailureAfterAuthenticationMutationRollsBackAndClearsRuntimeState events = append(events, "auth-runtime-reset-failed") return resetErr } - deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error { + deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error { events = append(events, "secret-aware-recovery-and-reauth-reset") return nil } @@ -727,7 +787,10 @@ func TestRestoreCleanupFailureDoesNotSuppressRollback(t *testing.T) { cleanupErr := errors.New("checkpoint cleanup failure") recovered := false deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return mutationErr } - deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error { recovered = true; return nil } + deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error { + recovered = true + return nil + } deps.cleanupCheckpoint = func(string) error { return cleanupErr } _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps) @@ -783,7 +846,7 @@ func TestRestoreStartFailureRecoversPreviouslyRunningTarget(t *testing.T) { backingRunner := newBackupRunner(installation, true) deps := restoreTestDependencies(t, failStartRestoreRunner{fakeBackupRunner: backingRunner}) recovered := false - deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error { + deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error { recovered = true backingRunner.running = true return nil @@ -806,7 +869,7 @@ func TestRestoreVerificationFailureRecoversPreviouslyRunningTarget(t *testing.T) verificationErr := errors.New("Pi is unavailable") deps.verify["pi"] = func(context.Context, config.Installation, archiveRunner) error { return verificationErr } recovered := false - deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error { + deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error { recovered = true return nil } @@ -884,7 +947,7 @@ func TestRestoreRecoversBehindBarrierForEveryVerificationFailure(t *testing.T) { } var recoveryBarrierActive bool var recoveryContext cleanupContextObservation - deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ bool) error { + deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ *stagedArchive, _ bool) error { recoveryContext = observeCleanupContext(ctx) recoveryBarrierActive = runner.maintenance runner.running, runner.coreRunning = true, true @@ -924,7 +987,7 @@ func TestRestoreDoesNotRollbackAfterFinalDeactivationResponseLoss(t *testing.T) } deps := restoreTestDependencies(t, runner) recoveryCalls := 0 - deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error { + deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error { recoveryCalls++ return nil } @@ -982,7 +1045,7 @@ func TestRestoreUsesBoundedRecoveryContextAfterPostMutationCancellation(t *testi } var recoveryContext cleanupContextObservation var recoveryBarrierActive bool - deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ bool) error { + deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ *stagedArchive, _ bool) error { recoveryContext = observeCleanupContext(ctx) recoveryBarrierActive = runner.maintenance runner.running, runner.coreRunning = true, true @@ -1067,7 +1130,8 @@ func TestRecoverRestoreTransactionVerifiesRecoveredStateBeforeReturning(t *testi } } - if err := recoverRestoreTransaction(context.Background(), installation, recovery, true, deps); err != nil { + staged := stageRecoveryForTest(t, installation, recovery) + if err := recoverRestoreTransaction(context.Background(), installation, recovery, staged, true, deps); err != nil { t.Fatal(err) } if got, want := checks, []string{"health", "doctor", "pi", "workspace"}; !equalStrings(got, want) { @@ -1096,7 +1160,8 @@ func TestRecoverRestoreTransactionFailsClosedForEveryVerification(t *testing.T) return verificationErr } - err = recoverRestoreTransaction(context.Background(), installation, recovery, true, deps) + staged := stageRecoveryForTest(t, installation, recovery) + err = recoverRestoreTransaction(context.Background(), installation, recovery, staged, true, deps) if !errors.Is(err, verificationErr) { t.Fatalf("recoverRestoreTransaction() error = %v, want %v", err, verificationErr) } @@ -1176,8 +1241,8 @@ func TestRestoreReleasesBarrierOnlyAfterVerifiedRecoveryFromLostResponse(t *test } return nil } - deps.recover = func(ctx context.Context, target config.Installation, checkpoint PreflightResult, wasRunning bool) error { - return recoverRestoreTransaction(ctx, target, checkpoint, wasRunning, deps) + deps.recover = func(ctx context.Context, target config.Installation, checkpoint PreflightResult, staged *stagedArchive, wasRunning bool) error { + return recoverRestoreTransaction(ctx, target, checkpoint, staged, wasRunning, deps) } _, err = restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps) @@ -1274,7 +1339,7 @@ func TestRestoreCleansMaintenanceAfterMutationAndRollbackFailures(t *testing.T) deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return mutationErr } - deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error { + deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error { if test.recoveryErr == nil { backing.running, backing.coreRunning = true, true } @@ -1505,8 +1570,26 @@ func equalStrings(got, want []string) bool { return true } +func stageRecoveryForTest(t *testing.T, installation config.Installation, recovery PreflightResult) *stagedArchive { + t.Helper() + if err := os.MkdirAll(installation.ControlDirectory(), 0o700); err != nil { + t.Fatal(err) + } + staged, err := recovery.StageArchive(context.Background()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := staged.Close(); err != nil { + t.Error(err) + } + }) + return staged +} + func restoreTestDependencies(t *testing.T, runner archiveRunner) restoreDependencies { t.Helper() + recoveryArchive := restoreArchive(t) return restoreDependencies{ preflight: func(ctx context.Context, installation config.Installation, request PreflightRequest) (PreflightResult, error) { return Preflight(ctx, installation, request, permissivePreflightDependencies()) @@ -1514,10 +1597,10 @@ func restoreTestDependencies(t *testing.T, runner archiveRunner) restoreDependen checkpoint: func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) { return Result{Path: "/tmp/default-checkpoint.zip"}, nil }, - prepareRecovery: func(context.Context, config.Installation, string) (PreflightResult, error) { - return PreflightResult{}, nil + prepareRecovery: func(ctx context.Context, installation config.Installation, _ string) (PreflightResult, error) { + return Preflight(ctx, installation, PreflightRequest{Archive: recoveryArchive, Confirm: true, AllowExternalSecrets: true}, permissivePreflightDependencies()) }, - recover: func(context.Context, config.Installation, PreflightResult, bool) error { return nil }, + recover: func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error { return nil }, cleanupCheckpoint: func(string) error { return nil }, acquireTransaction: lifecycle.AcquireTransaction, runner: runner,