259 lines
10 KiB
Go
259 lines
10 KiB
Go
package backup
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/aritmolab/thothii/tools/tht/internal/compose"
|
|
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
|
)
|
|
|
|
func TestRestoreStoppedInstallationRunsCheckpointRestoreAndVerification(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
archive := filepath.Join(t.TempDir(), "restore.zip")
|
|
writePreflightArchive(t, archive, preflightArchiveSpec{
|
|
entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("safe")}},
|
|
})
|
|
runner := newBackupRunner(installation, false)
|
|
var events []string
|
|
var checkpointRequest CreateRequest
|
|
deps := restoreTestDependencies(t, runner)
|
|
deps.checkpoint = func(_ context.Context, _ config.Installation, request CreateRequest) (Result, error) {
|
|
events = append(events, "checkpoint")
|
|
checkpointRequest = request
|
|
return Result{Path: "/tmp/checkpoint.zip"}, nil
|
|
}
|
|
deps.acquireLock = func(config.Installation) (restoreLock, error) {
|
|
events = append(events, "lock")
|
|
return fakeRestoreLock{release: func() { events = append(events, "unlock") }}, nil
|
|
}
|
|
deps.restoreFile = func(_ context.Context, _ config.Installation, entry ArchiveEntryMetadata, _ io.Reader) error {
|
|
events = append(events, "file:"+entry.Path)
|
|
return nil
|
|
}
|
|
for _, name := range []string{"health", "doctor", "pi", "workspace"} {
|
|
name := name
|
|
deps.verify[name] = func(context.Context, config.Installation, archiveRunner) error {
|
|
events = append(events, name)
|
|
return nil
|
|
}
|
|
}
|
|
|
|
result, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.Checkpoint != "/tmp/checkpoint.zip" || result.Restarted || !result.Verified {
|
|
t.Fatalf("Restore() result = %#v", result)
|
|
}
|
|
if checkpointRequest.IncludeSecrets || checkpointRequest.Confirm {
|
|
t.Fatalf("checkpoint request = %#v, want non-secret unconfirmed checkpoint", checkpointRequest)
|
|
}
|
|
if got, want := events, []string{"checkpoint", "lock", "file:configuration/operator.env", "health", "doctor", "pi", "workspace", "unlock"}; !equalStrings(got, want) {
|
|
t.Fatalf("restore events = %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
func TestRestorePreflightFailureDoesNotMutateTarget(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
runner := newBackupRunner(installation, true)
|
|
deps := restoreTestDependencies(t, runner)
|
|
preflightErr := errors.New("archive cannot be restored")
|
|
deps.preflight = func(context.Context, config.Installation, PreflightRequest) (PreflightResult, error) {
|
|
return PreflightResult{}, preflightErr
|
|
}
|
|
checkpointCalls := 0
|
|
deps.checkpoint = func(context.Context, config.Installation, CreateRequest) (Result, error) {
|
|
checkpointCalls++
|
|
return Result{}, nil
|
|
}
|
|
restoredFiles := 0
|
|
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
|
|
restoredFiles++
|
|
return nil
|
|
}
|
|
|
|
result, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: "unreadable.zip", Confirm: true}, deps)
|
|
if !errors.Is(err, preflightErr) {
|
|
t.Fatalf("restore error = %v, want preflight error", err)
|
|
}
|
|
if result != (RestoreResult{}) || checkpointCalls != 0 || restoredFiles != 0 || runner.stopCount != 0 || runner.startCount != 0 || !runner.running {
|
|
t.Fatalf("preflight failure mutated target: result=%#v checkpoint=%d files=%d stops=%d starts=%d running=%t", result, checkpointCalls, restoredFiles, runner.stopCount, runner.startCount, runner.running)
|
|
}
|
|
}
|
|
|
|
func TestRestoreCheckpointFailureDoesNotMutateTarget(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
archive := restoreArchive(t)
|
|
runner := newBackupRunner(installation, true)
|
|
deps := restoreTestDependencies(t, runner)
|
|
checkpointErr := errors.New("checkpoint unavailable")
|
|
deps.checkpoint = func(context.Context, config.Installation, CreateRequest) (Result, error) {
|
|
return Result{}, checkpointErr
|
|
}
|
|
restoredFiles := 0
|
|
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
|
|
restoredFiles++
|
|
return nil
|
|
}
|
|
|
|
result, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
|
|
if !errors.Is(err, checkpointErr) {
|
|
t.Fatalf("restore error = %v, want checkpoint error", err)
|
|
}
|
|
if result != (RestoreResult{}) || restoredFiles != 0 || runner.stopCount != 0 || runner.startCount != 0 || !runner.running {
|
|
t.Fatalf("checkpoint failure mutated target: result=%#v files=%d stops=%d starts=%d running=%t", result, restoredFiles, runner.stopCount, runner.startCount, runner.running)
|
|
}
|
|
}
|
|
|
|
func TestRestoreFileFailureStopsMutatedTargetAndRetainsCheckpoint(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
archive := restoreArchive(t)
|
|
runner := newBackupRunner(installation, true)
|
|
deps := restoreTestDependencies(t, runner)
|
|
fileErr := errors.New("cannot restore operator configuration")
|
|
deps.checkpoint = func(context.Context, config.Installation, CreateRequest) (Result, error) {
|
|
return Result{Path: "/tmp/recovery.zip"}, nil
|
|
}
|
|
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
|
|
return fileErr
|
|
}
|
|
|
|
result, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
|
|
if !errors.Is(err, fileErr) {
|
|
t.Fatalf("restore error = %v, want file error", err)
|
|
}
|
|
if result.Checkpoint != "/tmp/recovery.zip" {
|
|
t.Fatalf("recovery checkpoint = %q, want retained path", result.Checkpoint)
|
|
}
|
|
if runner.running || runner.stopCount != 2 {
|
|
t.Fatalf("mutated target was not stopped: running=%t stops=%d", runner.running, runner.stopCount)
|
|
}
|
|
}
|
|
|
|
func TestRestoreStartFailureStopsRunningTarget(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
archive := restoreArchive(t)
|
|
backingRunner := newBackupRunner(installation, true)
|
|
deps := restoreTestDependencies(t, failStartRestoreRunner{fakeBackupRunner: backingRunner})
|
|
|
|
_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
|
|
if err == nil || !strings.Contains(err.Error(), "start refused") {
|
|
t.Fatalf("restore error = %v, want restart failure", err)
|
|
}
|
|
if backingRunner.running || backingRunner.stopCount != 2 || backingRunner.startCount != 0 {
|
|
t.Fatalf("failed restart left target available: running=%t stops=%d starts=%d", backingRunner.running, backingRunner.stopCount, backingRunner.startCount)
|
|
}
|
|
}
|
|
|
|
func TestRestoreVerificationFailureStopsRunningTarget(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
archive := restoreArchive(t)
|
|
runner := newBackupRunner(installation, true)
|
|
deps := restoreTestDependencies(t, runner)
|
|
verificationErr := errors.New("Pi is unavailable")
|
|
deps.verify["pi"] = func(context.Context, config.Installation, archiveRunner) error { return verificationErr }
|
|
|
|
_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
|
|
if !errors.Is(err, verificationErr) {
|
|
t.Fatalf("restore error = %v, want verification failure", err)
|
|
}
|
|
if runner.running || runner.stopCount != 2 || runner.startCount != 1 {
|
|
t.Fatalf("verification failure left target available: running=%t stops=%d starts=%d", runner.running, runner.stopCount, runner.startCount)
|
|
}
|
|
}
|
|
|
|
func TestRestoreRefusesActiveSessionsWithoutDrain(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
archive := restoreArchive(t)
|
|
runner := newBackupRunner(installation, true)
|
|
runner.sessionResponses = []string{`[{"status":"running","archived":false}]`}
|
|
deps := restoreTestDependencies(t, runner)
|
|
restoredFiles := 0
|
|
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
|
|
restoredFiles++
|
|
return nil
|
|
}
|
|
|
|
_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
|
|
if !errors.Is(err, ErrActiveSessions) {
|
|
t.Fatalf("restore error = %v, want active-session refusal", err)
|
|
}
|
|
if restoredFiles != 0 || runner.stopCount != 0 || runner.startCount != 0 || !runner.running {
|
|
t.Fatalf("active-session refusal mutated target: files=%d stops=%d starts=%d running=%t", restoredFiles, runner.stopCount, runner.startCount, runner.running)
|
|
}
|
|
}
|
|
|
|
func restoreArchive(t *testing.T) string {
|
|
t.Helper()
|
|
archive := filepath.Join(t.TempDir(), "restore.zip")
|
|
writePreflightArchive(t, archive, preflightArchiveSpec{
|
|
entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("safe")}},
|
|
})
|
|
return archive
|
|
}
|
|
|
|
type failStartRestoreRunner struct {
|
|
*fakeBackupRunner
|
|
}
|
|
|
|
func (runner failStartRestoreRunner) Run(ctx context.Context, args []string, stdin io.Reader) (compose.Result, error) {
|
|
if strings.HasSuffix(strings.Join(args, " "), " start") {
|
|
return compose.Result{}, errors.New("start refused")
|
|
}
|
|
return runner.fakeBackupRunner.Run(ctx, args, stdin)
|
|
}
|
|
|
|
type fakeRestoreLock struct {
|
|
release func()
|
|
}
|
|
|
|
func (lock fakeRestoreLock) Release() error {
|
|
if lock.release != nil {
|
|
lock.release()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func equalStrings(got, want []string) bool {
|
|
if len(got) != len(want) {
|
|
return false
|
|
}
|
|
for index := range got {
|
|
if got[index] != want[index] {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func restoreTestDependencies(t *testing.T, runner archiveRunner) restoreDependencies {
|
|
t.Helper()
|
|
return restoreDependencies{
|
|
preflight: func(ctx context.Context, installation config.Installation, request PreflightRequest) (PreflightResult, error) {
|
|
return Preflight(ctx, installation, request, permissivePreflightDependencies())
|
|
},
|
|
checkpoint: func(context.Context, config.Installation, CreateRequest) (Result, error) {
|
|
return Result{Path: "/tmp/default-checkpoint.zip"}, nil
|
|
},
|
|
acquireLock: func(config.Installation) (restoreLock, error) { return fakeRestoreLock{}, nil },
|
|
runner: runner,
|
|
sleep: func(time.Duration) {},
|
|
restoreFile: func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return nil },
|
|
restoreVolume: func(context.Context, config.Installation, VolumeMetadata, io.Reader) error {
|
|
return nil
|
|
},
|
|
verify: map[string]restoreVerify{
|
|
"health": func(context.Context, config.Installation, archiveRunner) error { return nil },
|
|
"doctor": func(context.Context, config.Installation, archiveRunner) error { return nil },
|
|
"pi": func(context.Context, config.Installation, archiveRunner) error { return nil },
|
|
"workspace": func(context.Context, config.Installation, archiveRunner) error { return nil },
|
|
},
|
|
}
|
|
}
|