refactor(cli): rename operator command to tht

2026-08-15 21:56:40 +02:00
parent 460caa550c
commit aa8a2e9278
49 changed files with 303 additions and 261 deletions
@@ -0,0 +1,360 @@
+package pi
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+var (
+	ErrInterruptedRestart        = errors.New("a previous Pi restart is incomplete; recover lifecycle maintenance before another Pi lifecycle operation")
+	errRestartImageDrift         = errors.New("core image changed during Pi restart")
+	errRestartConfigurationDrift = errors.New("external endpoint configuration changed during Pi restart")
+	errRestartMountDrift         = errors.New("core persistence mount contract changed during Pi restart")
+	errRestartConfirmation       = &restartDiagnosticError{
+		message: "restart requires --yes after reviewing the planned Pi core recreation",
+		cause:   ErrConfirmationRequired,
+	}
+	errRestartActiveSessions = &restartDiagnosticError{
+		message: "active sessions must be drained before restarting Pi; use --drain only after they are complete",
+		cause:   ErrActiveSessions,
+	}
+)
+
+type restartDiagnosticError struct {
+	message string
+	cause   error
+}
+
+func (e *restartDiagnosticError) Error() string { return e.message }
+
+func (e *restartDiagnosticError) Unwrap() error { return e.cause }
+
+type RestartRequest struct {
+	StatePath       string
+	UpdateStatePath string
+	Confirm         bool
+	Drain           bool
+}
+
+type RestartResult struct {
+	StatePath string
+	Version   string
+}
+
+func Restart(ctx context.Context, runner Runner, request RestartRequest) (RestartResult, error) {
+	return restartWithHooks(ctx, runner, request, defaultLifecycleHooks)
+}
+
+func restartWithHooks(
+	ctx context.Context,
+	runner Runner,
+	request RestartRequest,
+	hooks lifecycleHooks,
+) (result RestartResult, retErr error) {
+	if err := validateRestartStatePaths(request.StatePath, request.UpdateStatePath); err != nil {
+		return RestartResult{}, err
+	}
+	lock, err := acquireLock(request.StatePath)
+	if err != nil {
+		return RestartResult{StatePath: request.StatePath}, err
+	}
+	defer lock.Release()
+	if !request.Confirm {
+		return RestartResult{StatePath: request.StatePath}, errRestartConfirmation
+	}
+	if state, err := readState(request.UpdateStatePath); err == nil && stateNeedsRecovery(state) {
+		return RestartResult{StatePath: request.StatePath}, ErrInterruptedUpdate
+	} else if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return RestartResult{StatePath: request.StatePath}, err
+	}
+	if err := prepareLifecycleMutation(request.StatePath, hooks.removeFile); err != nil {
+		return RestartResult{StatePath: request.StatePath}, err
+	}
+	clearMaintenance := true
+	mutationStarted := false
+	var state State
+	defer func() {
+		if !clearMaintenance {
+			return
+		}
+		if clearErr := setMaintenance(context.Background(), runner, false); clearErr != nil {
+			result.StatePath = request.StatePath
+			if mutationStarted {
+				clearMaintenance = false
+				cause := clearErr
+				if writeErr := hooks.writeState(request.StatePath, state); writeErr != nil {
+					cause = errors.Join(cause, fmt.Errorf("restart recovery state could not be restored: %w", writeErr))
+				}
+				retErr = errors.Join(retErr, recoveryRequired("Pi restart maintenance cleanup failed", cause))
+				return
+			}
+			retErr = errors.Join(retErr, fmt.Errorf("maintenance admission gate could not be cleared: %w", clearErr))
+		}
+	}()
+	if err := setMaintenance(ctx, runner, true); err != nil {
+		return RestartResult{StatePath: request.StatePath}, err
+	}
+
+	if err := waitForInactiveSessions(ctx, runner, request.Drain, hooks.sleep); err != nil {
+		return RestartResult{StatePath: request.StatePath}, restartDiagnostic(err)
+	}
+	if err := Doctor(ctx, runner); err != nil {
+		return RestartResult{StatePath: request.StatePath}, err
+	}
+	version, err := Status(ctx, runner)
+	if err != nil {
+		return RestartResult{StatePath: request.StatePath}, err
+	}
+	configured, err := renderedCore(ctx, runner)
+	if err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, err
+	}
+	previous, err := runningImage(ctx, runner, configured.Reference)
+	if err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, err
+	}
+	previous.ConfigurationSHA = configured.ConfigurationSHA
+	transaction := lifecycleTransaction(request.StatePath)
+	previous.Reference = lifecycleImageTag(transaction, "restart")
+	if err := tagImage(ctx, runner, previous.ID, previous.Reference, "restart image pin"); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, err
+	}
+	state = State{
+		Transaction: transaction,
+		Phase:       PhasePreflight,
+		Target:      Target{Version: version, Source: "restart"},
+		Previous:    previous,
+	}
+	if err := hooks.writeState(request.StatePath, state); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, err
+	}
+	overridePath := lifecycleOverridePath(request.StatePath, transaction)
+	if err := writeLifecycleOverride(overridePath, previous.Reference); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, err
+	}
+	lifecycle := composeOverrideRunner{Runner: runner, path: overridePath}
+	if running, err := activeSessions(ctx, runner); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, err
+	} else if running {
+		return RestartResult{StatePath: request.StatePath, Version: version}, errRestartActiveSessions
+	}
+	state.MutationStarted = true
+	if err := hooks.writeState(request.StatePath, state); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, err
+	}
+	mutationStarted = true
+	clearMaintenance = false
+	if err := recreateCoreWithoutImageChanges(ctx, lifecycle); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart core recreation failed", err)
+	}
+	if err := ensureMaintenance(ctx, lifecycle); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart maintenance proof failed", err)
+	}
+	state.Phase = PhaseRecreated
+	if err := hooks.writeState(request.StatePath, state); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart recreation state could not be recorded", err)
+	}
+	if err := verifyRestart(ctx, lifecycle, version, previous); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart verification failed", err)
+	}
+	state.Phase = PhaseVerified
+	if err := hooks.writeState(request.StatePath, state); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart verification state could not be recorded", err)
+	}
+	if err := hooks.removeFile(overridePath); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart image override could not be removed", err)
+	}
+	if err := hooks.removeFile(request.StatePath); err != nil {
+		return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart recovery state could not be removed", err)
+	}
+	clearMaintenance = true
+	return RestartResult{StatePath: request.StatePath, Version: version}, nil
+}
+
+func recreateCoreWithoutImageChanges(ctx context.Context, runner Runner) error {
+	result, err := runCompose(
+		ctx,
+		runner,
+		"up",
+		"--detach",
+		"--wait",
+		"--wait-timeout",
+		"45",
+		"--no-deps",
+		"--force-recreate",
+		"--no-build",
+		"--pull",
+		"never",
+		"core",
+	)
+	if err != nil {
+		return commandError("core recreation", result, err)
+	}
+	return nil
+}
+
+func verifyRestart(ctx context.Context, runner Runner, wanted string, previous Image) error {
+	if err := Doctor(ctx, runner); err != nil {
+		return err
+	}
+	version, err := Status(ctx, runner)
+	if err != nil {
+		return err
+	}
+	if version != wanted {
+		return errors.New("Pi version changed during core restart")
+	}
+	configured, err := renderedCore(ctx, runner)
+	if err != nil {
+		return err
+	}
+	after, err := runningImage(ctx, runner, configured.Reference)
+	if err != nil {
+		return err
+	}
+	if after.ID != previous.ID {
+		return errRestartImageDrift
+	}
+	if configured.ConfigurationSHA != previous.ConfigurationSHA {
+		return errRestartConfigurationDrift
+	}
+	if !sameMounts(previous.Mounts, after.Mounts) {
+		return errRestartMountDrift
+	}
+	return nil
+}
+
+func RecoverLifecycleMaintenance(
+	ctx context.Context,
+	runner Runner,
+	updateStatePath string,
+	restartStatePath string,
+	confirm bool,
+) error {
+	if err := validateRestartStatePaths(restartStatePath, updateStatePath); err != nil {
+		return err
+	}
+	if !confirm {
+		return errRestartConfirmation
+	}
+	lock, err := acquireLock(restartStatePath)
+	if err != nil {
+		return err
+	}
+	defer lock.Release()
+
+	restartState, restartErr := readState(restartStatePath)
+	if restartErr == nil {
+		if err := validateRestartRecoveryState(restartState); err != nil {
+			return err
+		}
+		restartOverride := lifecycleOverridePath(restartStatePath, restartState.Transaction)
+		if restartState.MutationStarted {
+			if err := tagImage(ctx, runner, restartState.Previous.ID, restartState.Previous.Reference, "restart recovery image pin"); err != nil {
+				return recoveryRequired("Pi restart recovery image pin could not be restored", err)
+			}
+			if err := writeLifecycleOverride(restartOverride, restartState.Previous.Reference); err != nil {
+				return recoveryRequired("Pi restart recovery image override could not be restored", err)
+			}
+			lifecycle := composeOverrideRunner{Runner: runner, path: restartOverride}
+			if err := ensureMaintenance(ctx, lifecycle); err != nil {
+				return recoveryRequired("Pi restart maintenance recovery failed", err)
+			}
+			if err := verifyRestart(ctx, lifecycle, restartState.Target.Version, restartState.Previous); err != nil {
+				return recoveryRequired("Pi restart recovery verification failed", err)
+			}
+		}
+		if err := durableRemove(restartOverride); err != nil {
+			return recoveryRequired("Pi restart recovery image override could not be removed", err)
+		}
+		if err := durableRemove(restartStatePath); err != nil {
+			return recoveryRequired("Pi restart recovery state could not be removed", err)
+		}
+	} else if !errors.Is(restartErr, os.ErrNotExist) {
+		return restartErr
+	}
+	return recoverMaintenanceLocked(ctx, runner, updateStatePath)
+}
+
+func restartDiagnostic(err error) error {
+	if errors.Is(err, ErrActiveSessions) {
+		return errRestartActiveSessions
+	}
+	return err
+}
+
+func validateRestartRecoveryState(state State) error {
+	invalid := func(reason string) error {
+		return fmt.Errorf("%w: restart recovery state is invalid: %s", ErrInvalidRequest, reason)
+	}
+	if state.Transaction == "" {
+		return invalid("transaction is missing")
+	}
+	if state.Target.Source != "restart" || state.Target.Version == "" {
+		return invalid("target is not a restart with a recorded version")
+	}
+	if state.Previous.ConfigurationSHA == "" {
+		return invalid("previous external configuration identity is missing")
+	}
+	if state.Candidate.ID != "" || state.Candidate.Reference != "" || len(state.Candidate.Mounts) != 0 ||
+		state.Candidate.MountFingerprint != "" || state.Candidate.ConfigurationSHA != "" {
+		return invalid("restart state contains image candidate metadata")
+	}
+	switch state.Phase {
+	case PhasePreflight:
+		return nil
+	case PhaseRecreated, PhaseVerified:
+		if state.MutationStarted {
+			return nil
+		}
+		return invalid("post-recreation phase has no mutation marker")
+	default:
+		return invalid("phase is not valid for restart")
+	}
+}
+
+func validateRestartStatePaths(restartStatePath, updateStatePath string) error {
+	if restartStatePath == "" {
+		return errors.New("restart state path is required")
+	}
+	if updateStatePath == "" {
+		return errors.New("update state path is required")
+	}
+	if filepath.Clean(restartStatePath) == filepath.Clean(updateStatePath) {
+		return fmt.Errorf("%w: restart and update state paths must remain separate", ErrInvalidRequest)
+	}
+	if lifecycleLockPath(restartStatePath) != lifecycleLockPath(updateStatePath) {
+		return fmt.Errorf("%w: restart and update state paths must share one lifecycle control directory", ErrInvalidRequest)
+	}
+	return nil
+}
+
+func pairedRestartStatePath(updateStatePath string) string {
+	return filepath.Join(filepath.Dir(updateStatePath), "restart-state.json")
+}
+
+func prepareLifecycleMutation(restartStatePath string, removeFile func(string) error) error {
+	state, err := readState(restartStatePath)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("restart recovery state could not be validated: %w", err)
+	}
+	if err := validateRestartRecoveryState(state); err != nil {
+		return err
+	}
+	if state.Phase != PhaseVerified || !state.MutationStarted {
+		return ErrInterruptedRestart
+	}
+	if err := removeFile(lifecycleOverridePath(restartStatePath, state.Transaction)); err != nil {
+		return recoveryRequired("verified restart override could not be cleaned up", err)
+	}
+	if err := removeFile(restartStatePath); err != nil {
+		return recoveryRequired("verified restart state could not be cleaned up", err)
+	}
+	return nil
+}