fix: close Pi management final review findings

tools/thothctl/internal/pi/restart.go

@@ -9,7 +9,7 @@ import (
 )
 
 var (
-	errInterruptedRestart        = errors.New("a previous Pi restart is incomplete; recover lifecycle maintenance before restarting again")
+	ErrInterruptedRestart        = errors.New("a previous Pi restart is incomplete; recover lifecycle maintenance before another Pi lifecycle operation")
 	errRestartImageDrift         = errors.New("core image changed during Pi restart")
 	errRestartConfigurationDrift = errors.New("external endpoint configuration changed during Pi restart")
 	errRestartMountDrift         = errors.New("core persistence mount contract changed during Pi restart")
@@ -70,14 +70,7 @@ func restartWithHooks(
 	} else if err != nil && !errors.Is(err, os.ErrNotExist) {
 		return RestartResult{StatePath: request.StatePath}, err
 	}
-	if state, err := readState(request.StatePath); err == nil {
-		if err := validateRestartRecoveryState(state); err != nil {
-			return RestartResult{StatePath: request.StatePath}, err
-		}
-		if state.MutationStarted {
-			return RestartResult{StatePath: request.StatePath}, errInterruptedRestart
-		}
-	} else if err != nil && !errors.Is(err, os.ErrNotExist) {
+	if err := prepareLifecycleMutation(request.StatePath, hooks.removeFile); err != nil {
 		return RestartResult{StatePath: request.StatePath}, err
 	}
 	clearMaintenance := true
@@ -124,8 +117,13 @@ func restartWithHooks(
 		return RestartResult{StatePath: request.StatePath, Version: version}, err
 	}
 	previous.ConfigurationSHA = configured.ConfigurationSHA
+	transaction := lifecycleTransaction(request.StatePath)
+	previous.Reference = lifecycleImageTag(transaction, "restart")
+	if err := tagImage(ctx, runner, previous.ID, previous.Reference, "restart image pin"); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, err
+	}
 	state = State{
-		Transaction: lifecycleTransaction(request.StatePath),
+		Transaction: transaction,
 		Phase:       PhasePreflight,
 		Target:      Target{Version: version, Source: "restart"},
 		Previous:    previous,
@@ -133,6 +131,11 @@ func restartWithHooks(
 	if err := hooks.writeState(request.StatePath, state); err != nil {
 		return RestartResult{StatePath: request.StatePath, Version: version}, err
 	}
+	overridePath := lifecycleOverridePath(request.StatePath, transaction)
+	if err := writeLifecycleOverride(overridePath, previous.Reference); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, err
+	}
+	lifecycle := composeOverrideRunner{Runner: runner, path: overridePath}
 	if running, err := activeSessions(ctx, runner); err != nil {
 		return RestartResult{StatePath: request.StatePath, Version: version}, err
 	} else if running {
@@ -144,23 +147,26 @@ func restartWithHooks(
 	}
 	mutationStarted = true
 	clearMaintenance = false
-	if err := recreateCore(ctx, runner); err != nil {
+	if err := recreateCoreWithoutImageChanges(ctx, lifecycle); err != nil {
 		return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart core recreation failed", err)
 	}
-	if err := ensureMaintenance(ctx, runner); err != nil {
+	if err := ensureMaintenance(ctx, lifecycle); err != nil {
 		return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart maintenance proof failed", err)
 	}
 	state.Phase = PhaseRecreated
 	if err := hooks.writeState(request.StatePath, state); err != nil {
 		return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart recreation state could not be recorded", err)
 	}
-	if err := verifyRestart(ctx, runner, version, previous); err != nil {
+	if err := verifyRestart(ctx, lifecycle, version, previous); err != nil {
 		return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart verification failed", err)
 	}
 	state.Phase = PhaseVerified
 	if err := hooks.writeState(request.StatePath, state); err != nil {
 		return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart verification state could not be recorded", err)
 	}
+	if err := hooks.removeFile(overridePath); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart image override could not be removed", err)
+	}
 	if err := hooks.removeFile(request.StatePath); err != nil {
 		return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart recovery state could not be removed", err)
 	}
@@ -168,6 +174,28 @@ func restartWithHooks(
 	return RestartResult{StatePath: request.StatePath, Version: version}, nil
 }
 
+func recreateCoreWithoutImageChanges(ctx context.Context, runner Runner) error {
+	result, err := runCompose(
+		ctx,
+		runner,
+		"up",
+		"--detach",
+		"--wait",
+		"--wait-timeout",
+		"45",
+		"--no-deps",
+		"--force-recreate",
+		"--no-build",
+		"--pull",
+		"never",
+		"core",
+	)
+	if err != nil {
+		return commandError("core recreation", result, err)
+	}
+	return nil
+}
+
 func verifyRestart(ctx context.Context, runner Runner, wanted string, previous Image) error {
 	if err := Doctor(ctx, runner); err != nil {
 		return err
@@ -223,14 +251,25 @@ func RecoverLifecycleMaintenance(
 		if err := validateRestartRecoveryState(restartState); err != nil {
 			return err
 		}
+		restartOverride := lifecycleOverridePath(restartStatePath, restartState.Transaction)
 		if restartState.MutationStarted {
-			if err := ensureMaintenance(ctx, runner); err != nil {
+			if err := tagImage(ctx, runner, restartState.Previous.ID, restartState.Previous.Reference, "restart recovery image pin"); err != nil {
+				return recoveryRequired("Pi restart recovery image pin could not be restored", err)
+			}
+			if err := writeLifecycleOverride(restartOverride, restartState.Previous.Reference); err != nil {
+				return recoveryRequired("Pi restart recovery image override could not be restored", err)
+			}
+			lifecycle := composeOverrideRunner{Runner: runner, path: restartOverride}
+			if err := ensureMaintenance(ctx, lifecycle); err != nil {
 				return recoveryRequired("Pi restart maintenance recovery failed", err)
 			}
-			if err := verifyRestart(ctx, runner, restartState.Target.Version, restartState.Previous); err != nil {
+			if err := verifyRestart(ctx, lifecycle, restartState.Target.Version, restartState.Previous); err != nil {
 				return recoveryRequired("Pi restart recovery verification failed", err)
 			}
 		}
+		if err := durableRemove(restartOverride); err != nil {
+			return recoveryRequired("Pi restart recovery image override could not be removed", err)
+		}
 		if err := durableRemove(restartStatePath); err != nil {
 			return recoveryRequired("Pi restart recovery state could not be removed", err)
 		}
@@ -292,3 +331,30 @@ func validateRestartStatePaths(restartStatePath, updateStatePath string) error {
 	}
 	return nil
 }
+
+func pairedRestartStatePath(updateStatePath string) string {
+	return filepath.Join(filepath.Dir(updateStatePath), "restart-state.json")
+}
+
+func prepareLifecycleMutation(restartStatePath string, removeFile func(string) error) error {
+	state, err := readState(restartStatePath)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("restart recovery state could not be validated: %w", err)
+	}
+	if err := validateRestartRecoveryState(state); err != nil {
+		return err
+	}
+	if state.Phase != PhaseVerified || !state.MutationStarted {
+		return ErrInterruptedRestart
+	}
+	if err := removeFile(lifecycleOverridePath(restartStatePath, state.Transaction)); err != nil {
+		return recoveryRequired("verified restart override could not be cleaned up", err)
+	}
+	if err := removeFile(restartStatePath); err != nil {
+		return recoveryRequired("verified restart state could not be cleaned up", err)
+	}
+	return nil
+}