fix(auth): harden backup restore lifecycle cleanup
This commit is contained in:
@@ -557,6 +557,136 @@ func TestCreateRefusesMutableNonRunningServiceStates(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateCleansMutationsWhenDockerLosesTheResponse(t *testing.T) {
|
||||
activationResponseLost := errors.New("activation response lost")
|
||||
stopResponseLost := errors.New("stop response lost")
|
||||
for _, scenario := range []struct {
|
||||
name string
|
||||
failure *commandFailure
|
||||
wantErr error
|
||||
cancelCaller bool
|
||||
wantStopCount int
|
||||
wantStartCount int
|
||||
wantCleanupCalls int
|
||||
}{
|
||||
{
|
||||
name: "maintenance activation",
|
||||
wantErr: activationResponseLost,
|
||||
failure: &commandFailure{
|
||||
match: func(command string) bool { return strings.Contains(command, " maintenance-activate") },
|
||||
err: activationResponseLost,
|
||||
remaining: 1,
|
||||
},
|
||||
wantCleanupCalls: 1,
|
||||
},
|
||||
{
|
||||
name: "stop response loss",
|
||||
wantErr: stopResponseLost,
|
||||
wantStopCount: 1,
|
||||
wantStartCount: 1,
|
||||
failure: &commandFailure{
|
||||
match: func(command string) bool { return strings.HasSuffix(command, " stop") },
|
||||
err: stopResponseLost,
|
||||
effect: func() {
|
||||
// A mutating command may have completed before its response was lost.
|
||||
},
|
||||
remaining: 1,
|
||||
},
|
||||
wantCleanupCalls: 2,
|
||||
},
|
||||
{
|
||||
name: "caller cancellation after stop",
|
||||
wantErr: context.Canceled,
|
||||
cancelCaller: true,
|
||||
wantStopCount: 1,
|
||||
wantStartCount: 1,
|
||||
failure: &commandFailure{
|
||||
match: func(command string) bool { return strings.HasSuffix(command, " stop") },
|
||||
err: context.Canceled,
|
||||
remaining: 1,
|
||||
},
|
||||
wantCleanupCalls: 2,
|
||||
},
|
||||
} {
|
||||
scenario := scenario
|
||||
t.Run(scenario.name, func(t *testing.T) {
|
||||
fixture := newBackupFixture(t, "local")
|
||||
backing := newBackupRunner(fixture.installation, true)
|
||||
caller, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
failure := *scenario.failure
|
||||
originalEffect := failure.effect
|
||||
failure.effect = func() {
|
||||
if strings.Contains(scenario.name, "activation") {
|
||||
backing.maintenance = true
|
||||
} else {
|
||||
backing.stopCount++
|
||||
backing.running, backing.coreRunning = false, false
|
||||
}
|
||||
if originalEffect != nil {
|
||||
originalEffect()
|
||||
}
|
||||
if scenario.cancelCaller {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
runner := &commandFailureRunner{fakeBackupRunner: backing, failures: []*commandFailure{&failure}}
|
||||
|
||||
_, err := createWithDependencies(caller, fixture.installation, CreateRequest{Output: filepath.Join(t.TempDir(), "backup.zip")}, testDependencies(t, runner))
|
||||
if !errors.Is(err, scenario.wantErr) {
|
||||
t.Fatalf("Create() error = %v, want %v", err, scenario.wantErr)
|
||||
}
|
||||
if backing.running != true || backing.coreRunning != true || backing.maintenance != false {
|
||||
t.Fatalf("cleanup state = running:%t core:%t maintenance:%t, want running and admitted", backing.running, backing.coreRunning, backing.maintenance)
|
||||
}
|
||||
if backing.stopCount != scenario.wantStopCount || backing.startCount != scenario.wantStartCount {
|
||||
t.Fatalf("lifecycle commands = stop:%d start:%d, want stop:%d start:%d", backing.stopCount, backing.startCount, scenario.wantStopCount, scenario.wantStartCount)
|
||||
}
|
||||
if len(runner.cleanupCommandContexts) != scenario.wantCleanupCalls {
|
||||
t.Fatalf("cleanup command contexts = %d, want %d", len(runner.cleanupCommandContexts), scenario.wantCleanupCalls)
|
||||
}
|
||||
assertIndependentBoundedCleanupContexts(t, runner.cleanupCommandContexts)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateJoinsPrimaryAndCleanupFailuresAfterPartialRestart(t *testing.T) {
|
||||
fixture := newBackupFixture(t, "local")
|
||||
backing := newBackupRunner(fixture.installation, true)
|
||||
startResponseLost := errors.New("start response lost")
|
||||
deactivationResponseLost := errors.New("deactivation response lost")
|
||||
runner := &commandFailureRunner{
|
||||
fakeBackupRunner: backing,
|
||||
failures: []*commandFailure{
|
||||
{
|
||||
match: func(command string) bool { return strings.HasSuffix(command, " start") },
|
||||
err: startResponseLost,
|
||||
effect: func() {
|
||||
backing.startCount++
|
||||
backing.running, backing.coreRunning = true, true
|
||||
},
|
||||
remaining: 1,
|
||||
},
|
||||
{
|
||||
match: func(command string) bool { return strings.Contains(command, " maintenance-deactivate") },
|
||||
err: deactivationResponseLost,
|
||||
effect: func() {
|
||||
backing.maintenance = false
|
||||
},
|
||||
remaining: 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: filepath.Join(t.TempDir(), "backup.zip")}, testDependencies(t, runner))
|
||||
if !errors.Is(err, startResponseLost) || !errors.Is(err, deactivationResponseLost) {
|
||||
t.Fatalf("Create() error = %v, want joined start and deactivation failures", err)
|
||||
}
|
||||
if backing.startCount != 2 || !backing.running || backing.maintenance {
|
||||
t.Fatalf("partial-success cleanup state = starts:%d running:%t maintenance:%t", backing.startCount, backing.running, backing.maintenance)
|
||||
}
|
||||
}
|
||||
|
||||
type backupFixture struct {
|
||||
root string
|
||||
installationID string
|
||||
@@ -656,6 +786,80 @@ type fakeBackupRunner struct {
|
||||
serviceStates map[string]string
|
||||
}
|
||||
|
||||
type commandFailure struct {
|
||||
match func(string) bool
|
||||
err error
|
||||
effect func()
|
||||
skip int
|
||||
remaining int
|
||||
}
|
||||
|
||||
type commandFailureRunner struct {
|
||||
*fakeBackupRunner
|
||||
failures []*commandFailure
|
||||
cleanupCommandContexts []cleanupContextObservation
|
||||
streamFailure func(context.Context) error
|
||||
}
|
||||
|
||||
type cleanupContextObservation struct {
|
||||
err error
|
||||
deadline time.Time
|
||||
hasDeadline bool
|
||||
}
|
||||
|
||||
func observeCleanupContext(ctx context.Context) cleanupContextObservation {
|
||||
deadline, hasDeadline := ctx.Deadline()
|
||||
return cleanupContextObservation{err: ctx.Err(), deadline: deadline, hasDeadline: hasDeadline}
|
||||
}
|
||||
|
||||
func (r *commandFailureRunner) Run(ctx context.Context, args []string, stdin io.Reader) (compose.Result, error) {
|
||||
command := strings.Join(args, " ")
|
||||
if strings.HasSuffix(command, " start") || strings.Contains(command, " maintenance-deactivate") {
|
||||
r.cleanupCommandContexts = append(r.cleanupCommandContexts, observeCleanupContext(ctx))
|
||||
}
|
||||
for _, failure := range r.failures {
|
||||
if failure.remaining != 0 && failure.match(command) {
|
||||
if failure.skip > 0 {
|
||||
failure.skip--
|
||||
continue
|
||||
}
|
||||
if failure.remaining > 0 {
|
||||
failure.remaining--
|
||||
}
|
||||
if failure.effect != nil {
|
||||
failure.effect()
|
||||
}
|
||||
return compose.Result{}, failure.err
|
||||
}
|
||||
}
|
||||
return r.fakeBackupRunner.Run(ctx, args, stdin)
|
||||
}
|
||||
|
||||
func (r *commandFailureRunner) Stream(ctx context.Context, args []string, stdin io.Reader, stdout io.Writer) (compose.Result, error) {
|
||||
if r.streamFailure != nil {
|
||||
return compose.Result{}, r.streamFailure(ctx)
|
||||
}
|
||||
return r.fakeBackupRunner.Stream(ctx, args, stdin, stdout)
|
||||
}
|
||||
|
||||
func assertIndependentBoundedCleanupContexts(t *testing.T, contexts []cleanupContextObservation) {
|
||||
t.Helper()
|
||||
if len(contexts) == 0 {
|
||||
t.Fatal("expected cleanup commands")
|
||||
}
|
||||
for _, cleanupContext := range contexts {
|
||||
if cleanupContext.err != nil {
|
||||
t.Fatalf("cleanup used a cancelled context: %v", cleanupContext.err)
|
||||
}
|
||||
if !cleanupContext.hasDeadline {
|
||||
t.Fatal("cleanup context has no deadline")
|
||||
}
|
||||
if remaining := time.Until(cleanupContext.deadline); remaining <= 0 || remaining > 10*time.Minute {
|
||||
t.Fatalf("unexpected cleanup deadline remaining: %s", remaining)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newBackupRunner(installation config.Installation, running bool) *fakeBackupRunner {
|
||||
return &fakeBackupRunner{installation: installation, running: running, coreRunning: running}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user