feat(thothctl): add safe Pi core restart

2026-08-14 17:52:35 +02:00
parent a94df262f4
commit 422f1d47b4
5 changed files with 570 additions and 29 deletions
@@ -0,0 +1,235 @@
+package pi
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+var (
+	errInterruptedRestart        = errors.New("a previous Pi restart is incomplete; recover lifecycle maintenance before restarting again")
+	errRestartImageDrift         = errors.New("core image changed during Pi restart")
+	errRestartConfigurationDrift = errors.New("external endpoint configuration changed during Pi restart")
+	errRestartMountDrift         = errors.New("core persistence mount contract changed during Pi restart")
+)
+
+type RestartRequest struct {
+	StatePath       string
+	UpdateStatePath string
+	Confirm         bool
+	Drain           bool
+}
+
+type RestartResult struct {
+	StatePath string
+	Version   string
+}
+
+func Restart(ctx context.Context, runner Runner, request RestartRequest) (RestartResult, error) {
+	return restartWithHooks(ctx, runner, request, defaultLifecycleHooks)
+}
+
+func restartWithHooks(
+	ctx context.Context,
+	runner Runner,
+	request RestartRequest,
+	hooks lifecycleHooks,
+) (result RestartResult, retErr error) {
+	if err := validateRestartStatePaths(request.StatePath, request.UpdateStatePath); err != nil {
+		return RestartResult{}, err
+	}
+	lock, err := acquireLock(request.StatePath)
+	if err != nil {
+		return RestartResult{StatePath: request.StatePath}, err
+	}
+	defer lock.Release()
+	if !request.Confirm {
+		return RestartResult{StatePath: request.StatePath}, ErrConfirmationRequired
+	}
+	if state, err := readState(request.UpdateStatePath); err == nil && stateNeedsRecovery(state) {
+		return RestartResult{StatePath: request.StatePath}, ErrInterruptedUpdate
+	} else if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return RestartResult{StatePath: request.StatePath}, err
+	}
+	if state, err := readState(request.StatePath); err == nil && state.MutationStarted {
+		return RestartResult{StatePath: request.StatePath}, errInterruptedRestart
+	} else if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return RestartResult{StatePath: request.StatePath}, err
+	}
+	if err := setMaintenance(ctx, runner, true); err != nil {
+		return RestartResult{StatePath: request.StatePath}, err
+	}
+	clearMaintenance := true
+	mutationStarted := false
+	var state State
+	defer func() {
+		if !clearMaintenance {
+			return
+		}
+		if clearErr := setMaintenance(context.Background(), runner, false); clearErr != nil {
+			result.StatePath = request.StatePath
+			if mutationStarted {
+				clearMaintenance = false
+				cause := clearErr
+				if writeErr := hooks.writeState(request.StatePath, state); writeErr != nil {
+					cause = errors.Join(cause, fmt.Errorf("restart recovery state could not be restored: %w", writeErr))
+				}
+				retErr = errors.Join(retErr, recoveryRequired("Pi restart maintenance cleanup failed", cause))
+				return
+			}
+			retErr = errors.Join(retErr, fmt.Errorf("maintenance admission gate could not be cleared: %w", clearErr))
+		}
+	}()
+
+	if err := waitForInactiveSessions(ctx, runner, request.Drain, hooks.sleep); err != nil {
+		return RestartResult{StatePath: request.StatePath}, err
+	}
+	if err := Doctor(ctx, runner); err != nil {
+		return RestartResult{StatePath: request.StatePath}, err
+	}
+	version, err := Status(ctx, runner)
+	if err != nil {
+		return RestartResult{StatePath: request.StatePath}, err
+	}
+	configured, err := renderedCore(ctx, runner)
+	if err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, err
+	}
+	previous, err := runningImage(ctx, runner, configured.Reference)
+	if err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, err
+	}
+	previous.ConfigurationSHA = configured.ConfigurationSHA
+	state = State{
+		Transaction: lifecycleTransaction(request.StatePath),
+		Phase:       PhasePreflight,
+		Target:      Target{Version: version, Source: "restart"},
+		Previous:    previous,
+	}
+	if err := hooks.writeState(request.StatePath, state); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, err
+	}
+	if running, err := activeSessions(ctx, runner); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, err
+	} else if running {
+		return RestartResult{StatePath: request.StatePath, Version: version}, ErrActiveSessions
+	}
+	state.MutationStarted = true
+	if err := hooks.writeState(request.StatePath, state); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, err
+	}
+	mutationStarted = true
+	clearMaintenance = false
+	if err := recreateCore(ctx, runner); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart core recreation failed", err)
+	}
+	if err := ensureMaintenance(ctx, runner); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart maintenance proof failed", err)
+	}
+	state.Phase = PhaseRecreated
+	if err := hooks.writeState(request.StatePath, state); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart recreation state could not be recorded", err)
+	}
+	if err := verifyRestart(ctx, runner, version, previous); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart verification failed", err)
+	}
+	state.Phase = PhaseVerified
+	if err := hooks.writeState(request.StatePath, state); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart verification state could not be recorded", err)
+	}
+	if err := hooks.removeFile(request.StatePath); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart recovery state could not be removed", err)
+	}
+	clearMaintenance = true
+	return RestartResult{StatePath: request.StatePath, Version: version}, nil
+}
+
+func verifyRestart(ctx context.Context, runner Runner, wanted string, previous Image) error {
+	if err := Doctor(ctx, runner); err != nil {
+		return err
+	}
+	version, err := Status(ctx, runner)
+	if err != nil {
+		return err
+	}
+	if version != wanted {
+		return errors.New("Pi version changed during core restart")
+	}
+	configured, err := renderedCore(ctx, runner)
+	if err != nil {
+		return err
+	}
+	after, err := runningImage(ctx, runner, configured.Reference)
+	if err != nil {
+		return err
+	}
+	if after.ID != previous.ID {
+		return errRestartImageDrift
+	}
+	if configured.ConfigurationSHA != previous.ConfigurationSHA {
+		return errRestartConfigurationDrift
+	}
+	if !sameMounts(previous.Mounts, after.Mounts) {
+		return errRestartMountDrift
+	}
+	return nil
+}
+
+func RecoverLifecycleMaintenance(
+	ctx context.Context,
+	runner Runner,
+	updateStatePath string,
+	restartStatePath string,
+	confirm bool,
+) error {
+	if err := validateRestartStatePaths(restartStatePath, updateStatePath); err != nil {
+		return err
+	}
+	if !confirm {
+		return ErrConfirmationRequired
+	}
+	lock, err := acquireLock(restartStatePath)
+	if err != nil {
+		return err
+	}
+	defer lock.Release()
+
+	restartState, restartErr := readState(restartStatePath)
+	if restartErr == nil {
+		if restartState.MutationStarted {
+			if restartState.Target.Source != "restart" || restartState.Target.Version == "" {
+				return errors.New("restart recovery state is incomplete")
+			}
+			if err := ensureMaintenance(ctx, runner); err != nil {
+				return recoveryRequired("Pi restart maintenance recovery failed", err)
+			}
+			if err := verifyRestart(ctx, runner, restartState.Target.Version, restartState.Previous); err != nil {
+				return recoveryRequired("Pi restart recovery verification failed", err)
+			}
+		}
+		if err := durableRemove(restartStatePath); err != nil {
+			return recoveryRequired("Pi restart recovery state could not be removed", err)
+		}
+	} else if !errors.Is(restartErr, os.ErrNotExist) {
+		return restartErr
+	}
+	return recoverMaintenanceLocked(ctx, runner, updateStatePath)
+}
+
+func validateRestartStatePaths(restartStatePath, updateStatePath string) error {
+	if restartStatePath == "" {
+		return errors.New("restart state path is required")
+	}
+	if updateStatePath == "" {
+		return errors.New("update state path is required")
+	}
+	if filepath.Clean(restartStatePath) == filepath.Clean(updateStatePath) {
+		return fmt.Errorf("%w: restart and update state paths must remain separate", ErrInvalidRequest)
+	}
+	if lifecycleLockPath(restartStatePath) != lifecycleLockPath(updateStatePath) {
+		return fmt.Errorf("%w: restart and update state paths must share one lifecycle control directory", ErrInvalidRequest)
+	}
+	return nil
+}