fix(thothctl): harden Pi restart recovery

tools/thothctl/internal/pi/restart.go

@@ -13,8 +13,25 @@ var (
 	errRestartImageDrift         = errors.New("core image changed during Pi restart")
 	errRestartConfigurationDrift = errors.New("external endpoint configuration changed during Pi restart")
 	errRestartMountDrift         = errors.New("core persistence mount contract changed during Pi restart")
+	errRestartConfirmation       = &restartDiagnosticError{
+		message: "restart requires --yes after reviewing the planned Pi core recreation",
+		cause:   ErrConfirmationRequired,
+	}
+	errRestartActiveSessions = &restartDiagnosticError{
+		message: "active sessions must be drained before restarting Pi; use --drain only after they are complete",
+		cause:   ErrActiveSessions,
+	}
 )
 
+type restartDiagnosticError struct {
+	message string
+	cause   error
+}
+
+func (e *restartDiagnosticError) Error() string { return e.message }
+
+func (e *restartDiagnosticError) Unwrap() error { return e.cause }
+
 type RestartRequest struct {
 	StatePath       string
 	UpdateStatePath string
@@ -46,21 +63,23 @@ func restartWithHooks(
 	}
 	defer lock.Release()
 	if !request.Confirm {
-		return RestartResult{StatePath: request.StatePath}, ErrConfirmationRequired
+		return RestartResult{StatePath: request.StatePath}, errRestartConfirmation
 	}
 	if state, err := readState(request.UpdateStatePath); err == nil && stateNeedsRecovery(state) {
 		return RestartResult{StatePath: request.StatePath}, ErrInterruptedUpdate
 	} else if err != nil && !errors.Is(err, os.ErrNotExist) {
 		return RestartResult{StatePath: request.StatePath}, err
 	}
-	if state, err := readState(request.StatePath); err == nil && state.MutationStarted {
-		return RestartResult{StatePath: request.StatePath}, errInterruptedRestart
+	if state, err := readState(request.StatePath); err == nil {
+		if err := validateRestartRecoveryState(state); err != nil {
+			return RestartResult{StatePath: request.StatePath}, err
+		}
+		if state.MutationStarted {
+			return RestartResult{StatePath: request.StatePath}, errInterruptedRestart
+		}
 	} else if err != nil && !errors.Is(err, os.ErrNotExist) {
 		return RestartResult{StatePath: request.StatePath}, err
 	}
-	if err := setMaintenance(ctx, runner, true); err != nil {
-		return RestartResult{StatePath: request.StatePath}, err
-	}
 	clearMaintenance := true
 	mutationStarted := false
 	var state State
@@ -82,9 +101,12 @@ func restartWithHooks(
 			retErr = errors.Join(retErr, fmt.Errorf("maintenance admission gate could not be cleared: %w", clearErr))
 		}
 	}()
+	if err := setMaintenance(ctx, runner, true); err != nil {
+		return RestartResult{StatePath: request.StatePath}, err
+	}
 
 	if err := waitForInactiveSessions(ctx, runner, request.Drain, hooks.sleep); err != nil {
-		return RestartResult{StatePath: request.StatePath}, err
+		return RestartResult{StatePath: request.StatePath}, restartDiagnostic(err)
 	}
 	if err := Doctor(ctx, runner); err != nil {
 		return RestartResult{StatePath: request.StatePath}, err
@@ -114,7 +136,7 @@ func restartWithHooks(
 	if running, err := activeSessions(ctx, runner); err != nil {
 		return RestartResult{StatePath: request.StatePath, Version: version}, err
 	} else if running {
-		return RestartResult{StatePath: request.StatePath, Version: version}, ErrActiveSessions
+		return RestartResult{StatePath: request.StatePath, Version: version}, errRestartActiveSessions
 	}
 	state.MutationStarted = true
 	if err := hooks.writeState(request.StatePath, state); err != nil {
@@ -188,7 +210,7 @@ func RecoverLifecycleMaintenance(
 		return err
 	}
 	if !confirm {
-		return ErrConfirmationRequired
+		return errRestartConfirmation
 	}
 	lock, err := acquireLock(restartStatePath)
 	if err != nil {
@@ -198,10 +220,10 @@ func RecoverLifecycleMaintenance(
 
 	restartState, restartErr := readState(restartStatePath)
 	if restartErr == nil {
+		if err := validateRestartRecoveryState(restartState); err != nil {
+			return err
+		}
 		if restartState.MutationStarted {
-			if restartState.Target.Source != "restart" || restartState.Target.Version == "" {
-				return errors.New("restart recovery state is incomplete")
-			}
 			if err := ensureMaintenance(ctx, runner); err != nil {
 				return recoveryRequired("Pi restart maintenance recovery failed", err)
 			}
@@ -218,6 +240,43 @@ func RecoverLifecycleMaintenance(
 	return recoverMaintenanceLocked(ctx, runner, updateStatePath)
 }
 
+func restartDiagnostic(err error) error {
+	if errors.Is(err, ErrActiveSessions) {
+		return errRestartActiveSessions
+	}
+	return err
+}
+
+func validateRestartRecoveryState(state State) error {
+	invalid := func(reason string) error {
+		return fmt.Errorf("%w: restart recovery state is invalid: %s", ErrInvalidRequest, reason)
+	}
+	if state.Transaction == "" {
+		return invalid("transaction is missing")
+	}
+	if state.Target.Source != "restart" || state.Target.Version == "" {
+		return invalid("target is not a restart with a recorded version")
+	}
+	if state.Previous.ConfigurationSHA == "" {
+		return invalid("previous external configuration identity is missing")
+	}
+	if state.Candidate.ID != "" || state.Candidate.Reference != "" || len(state.Candidate.Mounts) != 0 ||
+		state.Candidate.MountFingerprint != "" || state.Candidate.ConfigurationSHA != "" {
+		return invalid("restart state contains image candidate metadata")
+	}
+	switch state.Phase {
+	case PhasePreflight:
+		return nil
+	case PhaseRecreated, PhaseVerified:
+		if state.MutationStarted {
+			return nil
+		}
+		return invalid("post-recreation phase has no mutation marker")
+	default:
+		return invalid("phase is not valid for restart")
+	}
+}
+
 func validateRestartStatePaths(restartStatePath, updateStatePath string) error {
 	if restartStatePath == "" {
 		return errors.New("restart state path is required")