fix: finalize durable Pi lifecycle

2026-08-04 21:34:08 +02:00
parent 5ba2821a1b
commit a368889838
20 changed files with 1016 additions and 147 deletions
@@ -95,10 +95,14 @@ func updateWithHooks(ctx context.Context, runner Runner, request Request, hooks
 		}
 		request.Image = canonical
 	}
-	if old, err := readState(request.StatePath); err == nil && old.Phase != PhaseVerified && old.Phase != PhaseRolledBack && old.Phase != PhaseNoop {
+	if old, err := readState(request.StatePath); err == nil && stateNeedsRecovery(old) {
 		return Result{StatePath: request.StatePath}, ErrInterruptedUpdate
 	} else if err != nil && !errors.Is(err, os.ErrNotExist) {
 		return Result{StatePath: request.StatePath}, err
+	} else if err == nil && !old.MutationStarted {
+		if cleanupErr := hooks.removeFile(lifecycleOverridePath(request.StatePath, old.Transaction)); cleanupErr != nil {
+			return Result{StatePath: request.StatePath}, errors.New("safe prior preparation state could not be cleaned up")
+		}
 	}
 	if err := setMaintenance(ctx, runner, true); err != nil {
 		return Result{StatePath: request.StatePath}, err
@@ -183,24 +187,30 @@ func updateWithHooks(ctx context.Context, runner Runner, request Request, hooks
 	lifecycle := composeOverrideRunner{Runner: runner, path: overridePath}

 	state.Phase = PhaseBuilding
-	clearMaintenance = false
 	if err := hooks.writeState(request.StatePath, state); err != nil {
-		result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
+		result, retErr, clearMaintenance = failPreparation(request.StatePath, overridePath, state, err, hooks)
 		return result, retErr
 	}
 	if err := prepareCandidate(ctx, lifecycle, request, candidateReference); err != nil {
-		result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
+		result, retErr, clearMaintenance = failPreparation(request.StatePath, overridePath, state, err, hooks)
 		return result, retErr
 	}
 	running, err = activeSessions(ctx, runner)
 	if err != nil {
-		result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
+		result, retErr, clearMaintenance = failPreparation(request.StatePath, overridePath, state, err, hooks)
 		return result, retErr
 	}
 	if running {
-		result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, ErrActiveSessions, hooks)
+		result, retErr, clearMaintenance = failPreparation(request.StatePath, overridePath, state, ErrActiveSessions, hooks)
 		return result, retErr
 	}
+	state.MutationStarted = true
+	if err := hooks.writeState(request.StatePath, state); err != nil {
+		state.MutationStarted = false
+		result, retErr, clearMaintenance = failPreparation(request.StatePath, overridePath, state, err, hooks)
+		return result, retErr
+	}
+	clearMaintenance = false
 	if err := recreateCore(ctx, lifecycle); err != nil {
 		result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
 		return result, retErr
@@ -223,18 +233,48 @@ func updateWithHooks(ctx context.Context, runner Runner, request Request, hooks
 		result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
 		return result, retErr
 	}
-	state.Phase, state.Error = PhaseVerified, ""
+	state.Phase, state.Error = PhasePromoting, ""
 	if err := hooks.writeState(request.StatePath, state); err != nil {
 		result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
 		return result, retErr
 	}
-	if err := hooks.removeFile(overridePath); err != nil {
-		return Result{Phase: PhaseFailed, StatePath: request.StatePath}, errors.New("verified update override cleanup failed: maintenance recovery required")
+	if err := promoteLifecycleOverride(overridePath, currentImageOverridePath(request.StatePath), candidateReference); err != nil {
+		result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
+		return result, retErr
+	}
+	state.Phase = PhaseVerified
+	if err := hooks.writeState(request.StatePath, state); err != nil {
+		result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
+		return result, retErr
 	}
 	clearMaintenance = true
 	return Result{Phase: PhaseVerified, StatePath: request.StatePath}, nil
 }

+func stateNeedsRecovery(state State) bool {
+	switch state.Phase {
+	case PhaseVerified, PhaseRolledBack, PhaseNoop:
+		return false
+	case PhaseFailed:
+		return state.MutationStarted
+	default:
+		return state.MutationStarted
+	}
+}
+
+func failPreparation(statePath, overridePath string, state State, cause error, hooks lifecycleHooks) (Result, error, bool) {
+	state.Phase = PhaseFailed
+	state.MutationStarted = false
+	state.Error = "candidate preparation failed before core mutation"
+	writeErr := hooks.writeState(statePath, state)
+	removeErr := hooks.removeFile(overridePath)
+	message := "candidate preparation failed before core mutation"
+	if writeErr != nil || removeErr != nil {
+		message += "; safe preparation cleanup was incomplete"
+	}
+	return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("%s: %w", message, cause), true
+}
+
 // Rollback restores the image recorded in durable update state. It is safe for interrupted runs.
 func Rollback(ctx context.Context, runner Runner, statePath string, confirm bool) (result Result, retErr error) {
 	return rollbackWithHooks(ctx, runner, statePath, confirm, defaultLifecycleHooks)
@@ -290,13 +330,13 @@ func rollbackWithHooks(ctx context.Context, runner Runner, statePath string, con
 		}
 		return Result{Phase: PhaseFailed, StatePath: statePath}, err
 	}
+	if err := promoteLifecycleOverride(overridePath, currentImageOverridePath(statePath), state.Previous.Reference); err != nil {
+		return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("rollback restored the core but durable current-image promotion failed: recovery required")
+	}
 	state.Phase, state.Error = PhaseRolledBack, ""
 	if err := hooks.writeState(statePath, state); err != nil {
 		return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("rollback restored the core but recovery state could not be persisted")
 	}
-	if err := hooks.removeFile(overridePath); err != nil {
-		return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("rollback override cleanup failed: maintenance recovery required")
-	}
 	clearMaintenance = true
 	return Result{Phase: PhaseRolledBack, StatePath: statePath}, nil
 }
@@ -323,14 +363,14 @@ func compensate(ctx context.Context, runner Runner, statePath, overridePath stri
 		}
 		return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("update failed; automatic rollback also failed: recovery required"), false
 	}
+	if err := promoteLifecycleOverride(overridePath, currentImageOverridePath(statePath), state.Previous.Reference); err != nil {
+		return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("previous core image was restored but durable selector promotion failed: recovery required"), false
+	}
 	state.Phase, state.Error = PhaseRolledBack, ""
 	if writeErr := hooks.writeState(statePath, state); writeErr != nil {
 		return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("previous core image was restored but recovery state write failed: recovery required"), false
 	}
-	if err := hooks.removeFile(overridePath); err != nil {
-		return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("previous core image was restored but override cleanup failed: recovery required"), false
-	}
-	return Result{Phase: PhaseRolledBack, StatePath: statePath}, fmt.Errorf("update failed; previous core image was restored"), true
+	return Result{Phase: PhaseRolledBack, StatePath: statePath}, fmt.Errorf("update failed; previous core image was restored: %w", cause), true
 }

 func sourceValue(request Request) string {
@@ -600,6 +640,10 @@ func lifecycleOverridePath(statePath, transaction string) string {
 	return filepath.Join(filepath.Dir(statePath), "pi-lifecycle-"+transaction+".yaml")
 }

+func currentImageOverridePath(statePath string) string {
+	return filepath.Join(filepath.Dir(statePath), "current-image.yaml")
+}
+
 func writeLifecycleOverride(path, image string) error {
 	quoted, err := json.Marshal(image)
 	if err != nil {
@@ -612,6 +656,41 @@ func writeLifecycleOverride(path, image string) error {
 	return nil
 }

+func promoteLifecycleOverride(source, destination, expectedImage string) error {
+	if err := durableReplace(source, destination, filepath.Dir(destination)); err != nil {
+		selected, readErr := readLifecycleOverride(destination)
+		if readErr == nil && selected == expectedImage {
+			return nil
+		}
+		return errors.New("lifecycle image override could not be promoted durably")
+	}
+	selected, err := readLifecycleOverride(destination)
+	if err != nil || selected != expectedImage {
+		return errors.New("promoted lifecycle image override could not be verified")
+	}
+	return nil
+}
+
+func readLifecycleOverride(path string) (string, error) {
+	contents, err := os.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+	for _, line := range strings.Split(string(contents), "\n") {
+		line = strings.TrimSpace(line)
+		if !strings.HasPrefix(line, "image:") {
+			continue
+		}
+		encoded := strings.TrimSpace(strings.TrimPrefix(line, "image:"))
+		var image string
+		if json.Unmarshal([]byte(encoded), &image) != nil || image == "" || strings.ContainsAny(image, "\r\n") {
+			return "", errors.New("lifecycle image override is invalid")
+		}
+		return image, nil
+	}
+	return "", errors.New("lifecycle image override has no core image")
+}
+
 // RecoverMaintenance clears a stale durable gate only after the running core and terminal
 // recovery metadata prove that no rollback is still required.
 func RecoverMaintenance(ctx context.Context, runner Runner, statePath string, confirm bool) error {
@@ -625,11 +704,26 @@ func RecoverMaintenance(ctx context.Context, runner Runner, statePath string, co
 	defer lock.Release()
 	state, stateErr := readState(statePath)
 	if stateErr == nil {
-		if state.Phase != PhaseVerified && state.Phase != PhaseRolledBack && state.Phase != PhaseNoop {
+		transactionOverride := lifecycleOverridePath(statePath, state.Transaction)
+		switch {
+		case state.Phase == PhasePromoting:
+			if err := recoverPromotion(ctx, runner, statePath, transactionOverride, &state); err != nil {
+				return err
+			}
+		case !state.MutationStarted && state.Phase != PhaseVerified && state.Phase != PhaseRolledBack && state.Phase != PhaseNoop:
+			state.Phase, state.Error = PhaseFailed, "candidate preparation interrupted before core mutation"
+			if err := writeState(statePath, state); err != nil {
+				return errors.New("maintenance recovery could not finalize safe preparation state")
+			}
+			if err := durableRemove(transactionOverride); err != nil {
+				return errors.New("maintenance recovery could not remove the safe preparation override")
+			}
+		case stateNeedsRecovery(state):
 			return ErrInterruptedUpdate
-		}
-		if err := durableRemove(lifecycleOverridePath(statePath, state.Transaction)); err != nil {
-			return errors.New("maintenance recovery could not remove the lifecycle override")
+		default:
+			if err := durableRemove(transactionOverride); err != nil {
+				return errors.New("maintenance recovery could not remove the lifecycle override")
+			}
 		}
 	} else if !errors.Is(stateErr, os.ErrNotExist) {
 		return stateErr
@@ -647,6 +741,50 @@ func RecoverMaintenance(ctx context.Context, runner Runner, statePath string, co
 	return setMaintenance(ctx, runner, false)
 }

+func recoverPromotion(ctx context.Context, runner Runner, statePath, transactionOverride string, state *State) error {
+	currentOverride := currentImageOverridePath(statePath)
+	selected, currentErr := readLifecycleOverride(currentOverride)
+	if currentErr != nil || selected != state.Candidate.Reference {
+		pending, pendingErr := readLifecycleOverride(transactionOverride)
+		if pendingErr != nil || pending != state.Candidate.Reference {
+			if currentErr == nil && selected == state.Previous.Reference {
+				if err := verifyRestoredCurrent(ctx, runner, state.Previous); err != nil {
+					return ErrInterruptedUpdate
+				}
+				state.Phase, state.Error = PhaseRolledBack, ""
+				return writeState(statePath, *state)
+			}
+			return ErrInterruptedUpdate
+		}
+		if err := promoteLifecycleOverride(transactionOverride, currentOverride, state.Candidate.Reference); err != nil {
+			return err
+		}
+	}
+	if err := verifyCandidate(ctx, runner, state.Target.Version, state.Previous); err != nil {
+		return err
+	}
+	state.Phase, state.Error = PhaseVerified, ""
+	return writeState(statePath, *state)
+}
+
+func verifyRestoredCurrent(ctx context.Context, runner Runner, previous Image) error {
+	configured, err := renderedCore(ctx, runner)
+	if err != nil {
+		return err
+	}
+	after, err := runningImage(ctx, runner, configured.Reference)
+	if err != nil {
+		return err
+	}
+	if after.ID != previous.ID || configured.ConfigurationSHA != previous.ConfigurationSHA || !sameMounts(previous.Mounts, after.Mounts) {
+		return errors.New("running core does not match the durable previous-image selector")
+	}
+	if err := Doctor(ctx, runner); err != nil {
+		return err
+	}
+	return Test(ctx, runner)
+}
+
 func sameStrings(left, right []string) bool {
 	left, right = append([]string(nil), left...), append([]string(nil), right...)
 	sort.Strings(left)