package backup import ( "context" "errors" "io" "path/filepath" "strings" "testing" "time" "github.com/aritmolab/thothii/tools/tht/internal/compose" "github.com/aritmolab/thothii/tools/tht/internal/config" ) func TestRestoreStoppedInstallationRunsCheckpointRestoreAndVerification(t *testing.T) { installation := preflightTestInstallation(t) archive := filepath.Join(t.TempDir(), "restore.zip") writePreflightArchive(t, archive, preflightArchiveSpec{ entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("safe")}}, }) runner := newBackupRunner(installation, false) var events []string var checkpointRequest CreateRequest deps := restoreTestDependencies(t, runner) deps.checkpoint = func(_ context.Context, _ config.Installation, request CreateRequest) (Result, error) { events = append(events, "checkpoint") checkpointRequest = request return Result{Path: "/tmp/checkpoint.zip"}, nil } deps.acquireLock = func(config.Installation) (restoreLock, error) { events = append(events, "lock") return fakeRestoreLock{release: func() { events = append(events, "unlock") }}, nil } deps.restoreFile = func(_ context.Context, _ config.Installation, entry ArchiveEntryMetadata, _ io.Reader) error { events = append(events, "file:"+entry.Path) return nil } for _, name := range []string{"health", "doctor", "pi", "workspace"} { name := name deps.verify[name] = func(context.Context, config.Installation, archiveRunner) error { events = append(events, name) return nil } } result, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps) if err != nil { t.Fatal(err) } if result.Checkpoint != "/tmp/checkpoint.zip" || result.Restarted || !result.Verified { t.Fatalf("Restore() result = %#v", result) } if checkpointRequest.IncludeSecrets || checkpointRequest.Confirm { t.Fatalf("checkpoint request = %#v, want non-secret unconfirmed checkpoint", checkpointRequest) } if got, want := events, []string{"checkpoint", "lock", "file:configuration/operator.env", "health", "doctor", "pi", "workspace", "unlock"}; !equalStrings(got, want) { t.Fatalf("restore events = %v, want %v", got, want) } } func TestRestoreResetsAuthenticationStateBeforeRestart(t *testing.T) { installation := preflightTestInstallation(t) archive := restoreArchive(t) runner := newBackupRunner(installation, true) deps := restoreTestDependencies(t, runner) var events []string deps.restoreFile = func(_ context.Context, _ config.Installation, entry ArchiveEntryMetadata, _ io.Reader) error { events = append(events, "file:"+entry.Path) return nil } deps.resetAuthenticationState = func(context.Context, config.Installation, archiveRunner) error { events = append(events, "reset-auth-state") return nil } for _, name := range []string{"health", "doctor", "pi", "workspace"} { name := name deps.verify[name] = func(context.Context, config.Installation, archiveRunner) error { events = append(events, name) return nil } } result, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps) if err != nil { t.Fatal(err) } if !result.Restarted || !result.Verified { t.Fatalf("restore result = %#v", result) } if got, want := events, []string{"file:configuration/operator.env", "reset-auth-state", "health", "doctor", "pi", "workspace"}; !equalStrings(got, want) { t.Fatalf("restore events = %v, want %v", got, want) } } func TestResetAuthenticationStateCreatesOnlyPrivateEmptyStateDirectories(t *testing.T) { installation := preflightTestInstallation(t) runner := &authenticationStateResetRunner{} if err := resetAuthenticationState(context.Background(), installation, runner); err != nil { t.Fatal(err) } joined := strings.Join(runner.args, "\x00") for _, required := range []string{ "run", "--rm", "--no-deps", "--no-TTY", "--entrypoint", "sh", "core", "-ceu", "rm -rf /data/auth && install -d -m 0700 /data/auth /data/auth/sessions /data/auth/oidc", } { if !strings.Contains(joined, required) { t.Fatalf("authentication state reset command omits %q: %#v", required, runner.args) } } } func TestRestorePreflightFailureDoesNotMutateTarget(t *testing.T) { installation := preflightTestInstallation(t) runner := newBackupRunner(installation, true) deps := restoreTestDependencies(t, runner) preflightErr := errors.New("archive cannot be restored") deps.preflight = func(context.Context, config.Installation, PreflightRequest) (PreflightResult, error) { return PreflightResult{}, preflightErr } checkpointCalls := 0 deps.checkpoint = func(context.Context, config.Installation, CreateRequest) (Result, error) { checkpointCalls++ return Result{}, nil } restoredFiles := 0 deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { restoredFiles++ return nil } result, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: "unreadable.zip", Confirm: true}, deps) if !errors.Is(err, preflightErr) { t.Fatalf("restore error = %v, want preflight error", err) } if result != (RestoreResult{}) || checkpointCalls != 0 || restoredFiles != 0 || runner.stopCount != 0 || runner.startCount != 0 || !runner.running { t.Fatalf("preflight failure mutated target: result=%#v checkpoint=%d files=%d stops=%d starts=%d running=%t", result, checkpointCalls, restoredFiles, runner.stopCount, runner.startCount, runner.running) } } func TestRestoreCheckpointFailureDoesNotMutateTarget(t *testing.T) { installation := preflightTestInstallation(t) archive := restoreArchive(t) runner := newBackupRunner(installation, true) deps := restoreTestDependencies(t, runner) checkpointErr := errors.New("checkpoint unavailable") deps.checkpoint = func(context.Context, config.Installation, CreateRequest) (Result, error) { return Result{}, checkpointErr } restoredFiles := 0 deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { restoredFiles++ return nil } result, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps) if !errors.Is(err, checkpointErr) { t.Fatalf("restore error = %v, want checkpoint error", err) } if result != (RestoreResult{}) || restoredFiles != 0 || runner.stopCount != 0 || runner.startCount != 0 || !runner.running { t.Fatalf("checkpoint failure mutated target: result=%#v files=%d stops=%d starts=%d running=%t", result, restoredFiles, runner.stopCount, runner.startCount, runner.running) } } func TestRestoreFileFailureStopsMutatedTargetAndRetainsCheckpoint(t *testing.T) { installation := preflightTestInstallation(t) archive := restoreArchive(t) runner := newBackupRunner(installation, true) deps := restoreTestDependencies(t, runner) fileErr := errors.New("cannot restore operator configuration") deps.checkpoint = func(context.Context, config.Installation, CreateRequest) (Result, error) { return Result{Path: "/tmp/recovery.zip"}, nil } deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return fileErr } result, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps) if !errors.Is(err, fileErr) { t.Fatalf("restore error = %v, want file error", err) } if result.Checkpoint != "/tmp/recovery.zip" { t.Fatalf("recovery checkpoint = %q, want retained path", result.Checkpoint) } if runner.running || runner.stopCount != 2 { t.Fatalf("mutated target was not stopped: running=%t stops=%d", runner.running, runner.stopCount) } } func TestRestoreStartFailureStopsRunningTarget(t *testing.T) { installation := preflightTestInstallation(t) archive := restoreArchive(t) backingRunner := newBackupRunner(installation, true) deps := restoreTestDependencies(t, failStartRestoreRunner{fakeBackupRunner: backingRunner}) _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps) if err == nil || !strings.Contains(err.Error(), "start refused") { t.Fatalf("restore error = %v, want restart failure", err) } if backingRunner.running || backingRunner.stopCount != 2 || backingRunner.startCount != 0 { t.Fatalf("failed restart left target available: running=%t stops=%d starts=%d", backingRunner.running, backingRunner.stopCount, backingRunner.startCount) } } func TestRestoreVerificationFailureStopsRunningTarget(t *testing.T) { installation := preflightTestInstallation(t) archive := restoreArchive(t) runner := newBackupRunner(installation, true) deps := restoreTestDependencies(t, runner) verificationErr := errors.New("Pi is unavailable") deps.verify["pi"] = func(context.Context, config.Installation, archiveRunner) error { return verificationErr } _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps) if !errors.Is(err, verificationErr) { t.Fatalf("restore error = %v, want verification failure", err) } if runner.running || runner.stopCount != 2 || runner.startCount != 1 { t.Fatalf("verification failure left target available: running=%t stops=%d starts=%d", runner.running, runner.stopCount, runner.startCount) } } func TestRestoreRefusesActiveSessionsWithoutDrain(t *testing.T) { installation := preflightTestInstallation(t) archive := restoreArchive(t) runner := newBackupRunner(installation, true) runner.sessionResponses = []string{`[{"status":"running","archived":false}]`} deps := restoreTestDependencies(t, runner) restoredFiles := 0 deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { restoredFiles++ return nil } _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps) if !errors.Is(err, ErrActiveSessions) { t.Fatalf("restore error = %v, want active-session refusal", err) } if restoredFiles != 0 || runner.stopCount != 0 || runner.startCount != 0 || !runner.running { t.Fatalf("active-session refusal mutated target: files=%d stops=%d starts=%d running=%t", restoredFiles, runner.stopCount, runner.startCount, runner.running) } } func restoreArchive(t *testing.T) string { t.Helper() archive := filepath.Join(t.TempDir(), "restore.zip") writePreflightArchive(t, archive, preflightArchiveSpec{ entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("safe")}}, }) return archive } type failStartRestoreRunner struct { *fakeBackupRunner } func (runner failStartRestoreRunner) Run(ctx context.Context, args []string, stdin io.Reader) (compose.Result, error) { if strings.HasSuffix(strings.Join(args, " "), " start") { return compose.Result{}, errors.New("start refused") } return runner.fakeBackupRunner.Run(ctx, args, stdin) } type fakeRestoreLock struct { release func() } type authenticationStateResetRunner struct{ args []string } func (runner *authenticationStateResetRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) { runner.args = append([]string(nil), args...) return compose.Result{}, nil } func (runner *authenticationStateResetRunner) Stream(context.Context, []string, io.Reader, io.Writer) (compose.Result, error) { return compose.Result{}, errors.New("authentication state reset must not stream a volume archive") } func (*authenticationStateResetRunner) SessionInventoryScope() string { return "mine" } func (lock fakeRestoreLock) Release() error { if lock.release != nil { lock.release() } return nil } func equalStrings(got, want []string) bool { if len(got) != len(want) { return false } for index := range got { if got[index] != want[index] { return false } } return true } func restoreTestDependencies(t *testing.T, runner archiveRunner) restoreDependencies { t.Helper() return restoreDependencies{ preflight: func(ctx context.Context, installation config.Installation, request PreflightRequest) (PreflightResult, error) { return Preflight(ctx, installation, request, permissivePreflightDependencies()) }, checkpoint: func(context.Context, config.Installation, CreateRequest) (Result, error) { return Result{Path: "/tmp/default-checkpoint.zip"}, nil }, acquireLock: func(config.Installation) (restoreLock, error) { return fakeRestoreLock{}, nil }, runner: runner, sleep: func(time.Duration) {}, restoreFile: func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return nil }, restoreVolume: func(context.Context, config.Installation, VolumeMetadata, io.Reader) error { return nil }, resetAuthenticationState: func(context.Context, config.Installation, archiveRunner) error { return nil }, verify: map[string]restoreVerify{ "health": func(context.Context, config.Installation, archiveRunner) error { return nil }, "doctor": func(context.Context, config.Installation, archiveRunner) error { return nil }, "pi": func(context.Context, config.Installation, archiveRunner) error { return nil }, "workspace": func(context.Context, config.Installation, archiveRunner) error { return nil }, }, } }