fix: harden Pi lifecycle recovery

2026-08-04 20:28:09 +02:00
parent 5b3ce93e31
commit 5ba2821a1b
24 changed files with 1764 additions and 384 deletions
@@ -2,15 +2,19 @@ package pi

 import (
 	"context"
+	"crypto/sha256"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"os"
+	"path/filepath"
 	"regexp"
 	"sort"
 	"strings"
 	"time"

+	"github.com/aritmolab/thothii/tools/thothctl/internal/compose"
 	"github.com/distribution/reference"
 )

@@ -46,29 +50,49 @@ type Result struct {
 	StatePath string
 }

+type lifecycleHooks struct {
+	writeState func(string, State) error
+	removeFile func(string) error
+	sleep      func(time.Duration)
+}
+
+var defaultLifecycleHooks = lifecycleHooks{
+	writeState: writeState,
+	removeFile: durableRemove,
+	sleep:      time.Sleep,
+}
+
 // Update performs a recoverable core-only Pi update using the default Compose command layout.
 func Update(ctx context.Context, runner Runner, request Request) (result Result, retErr error) {
+	return updateWithHooks(ctx, runner, request, defaultLifecycleHooks)
+}
+
+func updateWithHooks(ctx context.Context, runner Runner, request Request, hooks lifecycleHooks) (result Result, retErr error) {
+	if request.StatePath == "" {
+		return Result{}, errors.New("update state path is required")
+	}
 	lock, err := acquireLock(request.StatePath)
 	if err != nil {
 		return Result{StatePath: request.StatePath}, err
 	}
 	defer lock.Release()
-	if request.StatePath == "" {
-		return Result{}, errors.New("update state path is required")
-	}
 	if !request.Confirm {
 		return Result{StatePath: request.StatePath}, ErrConfirmationRequired
 	}
 	if !versionPattern.MatchString(request.Version) {
 		return Result{StatePath: request.StatePath}, fmt.Errorf("%w: Pi version must be an explicit pinned version", ErrInvalidRequest)
 	}
-	if request.Source == "" { return Result{StatePath: request.StatePath}, fmt.Errorf("%w: Pi update requires an explicit source: build or pull", ErrInvalidRequest) }
+	if request.Source == "" {
+		return Result{StatePath: request.StatePath}, fmt.Errorf("%w: Pi update requires an explicit source: build or pull", ErrInvalidRequest)
+	}
 	if request.Source != BuildSource && request.Source != PullSource {
 		return Result{StatePath: request.StatePath}, fmt.Errorf("%w: Pi update source must be build or pull", ErrInvalidRequest)
 	}
 	if request.Source == PullSource {
 		canonical, err := canonicalDigestReference(request.Image)
-		if err != nil { return Result{StatePath: request.StatePath}, fmt.Errorf("%w: %v", ErrInvalidRequest, err) }
+		if err != nil {
+			return Result{StatePath: request.StatePath}, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
+		}
 		request.Image = canonical
 	}
 	if old, err := readState(request.StatePath); err == nil && old.Phase != PhaseVerified && old.Phase != PhaseRolledBack && old.Phase != PhaseNoop {
@@ -76,12 +100,21 @@ func Update(ctx context.Context, runner Runner, request Request) (result Result,
 	} else if err != nil && !errors.Is(err, os.ErrNotExist) {
 		return Result{StatePath: request.StatePath}, err
 	}
-	if err := setMaintenance(ctx, runner, true); err != nil { return Result{StatePath: request.StatePath}, err }
+	if err := setMaintenance(ctx, runner, true); err != nil {
+		return Result{StatePath: request.StatePath}, err
+	}
+	clearMaintenance := true
 	defer func() {
+		if !clearMaintenance {
+			return
+		}
 		if clearErr := setMaintenance(context.Background(), runner, false); clearErr != nil {
 			result = Result{Phase: PhaseFailed, StatePath: request.StatePath}
-			if retErr == nil { retErr = errors.New("maintenance admission gate could not be cleared: recovery required")
-			} else { retErr = fmt.Errorf("%w; maintenance admission gate could not be cleared: recovery required", retErr) }
+			if retErr == nil {
+				retErr = errors.New("maintenance admission gate could not be cleared: recovery required")
+			} else {
+				retErr = fmt.Errorf("%w; maintenance admission gate could not be cleared: recovery required", retErr)
+			}
 		}
 	}()

@@ -101,7 +134,7 @@ func Update(ctx context.Context, runner Runner, request Request) (result Result,
 			if !running {
 				break
 			}
-			time.Sleep(time.Second)
+			hooks.sleep(time.Second)
 		}
 		if running {
 			return Result{StatePath: request.StatePath}, ErrActiveSessions
@@ -127,46 +160,87 @@ func Update(ctx context.Context, runner Runner, request Request) (result Result,
 		return Result{StatePath: request.StatePath}, err
 	}
 	previous.ConfigurationSHA = configured.ConfigurationSHA
-	state := State{Phase: PhasePreflight, Target: Target{Version: request.Version, Source: sourceValue(request)}, Previous: previous}
-	if err := writeState(request.StatePath, state); err != nil {
+	transaction := lifecycleTransaction(request.StatePath)
+	previous.Reference = lifecycleImageTag(transaction, "previous")
+	candidateReference := lifecycleImageTag(transaction, "candidate")
+	if err := tagImage(ctx, runner, previous.ID, previous.Reference, "previous Pi image pin"); err != nil {
 		return Result{StatePath: request.StatePath}, err
 	}
+	state := State{
+		Transaction: transaction,
+		Phase:       PhasePreflight,
+		Target:      Target{Version: request.Version, Source: sourceValue(request)},
+		Previous:    previous,
+		Candidate:   Image{Reference: candidateReference},
+	}
+	if err := hooks.writeState(request.StatePath, state); err != nil {
+		return Result{StatePath: request.StatePath}, err
+	}
+	overridePath := lifecycleOverridePath(request.StatePath, transaction)
+	if err := writeLifecycleOverride(overridePath, candidateReference); err != nil {
+		return Result{StatePath: request.StatePath}, err
+	}
+	lifecycle := composeOverrideRunner{Runner: runner, path: overridePath}

 	state.Phase = PhaseBuilding
-	if err := writeState(request.StatePath, state); err != nil { return Result{Phase: PhaseFailed, StatePath: request.StatePath}, err }
-	if err := prepareCandidate(ctx, runner, request, previous.Reference); err != nil {
-		return compensate(ctx, runner, request.StatePath, state, err)
+	clearMaintenance = false
+	if err := hooks.writeState(request.StatePath, state); err != nil {
+		result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
+		return result, retErr
 	}
-	if request.Drain {
-		running, err = activeSessions(ctx, runner)
-		if err != nil {
-			return compensate(ctx, runner, request.StatePath, state, err)
-		}
-		if running {
-			return compensate(ctx, runner, request.StatePath, state, ErrActiveSessions)
-		}
+	if err := prepareCandidate(ctx, lifecycle, request, candidateReference); err != nil {
+		result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
+		return result, retErr
 	}
-	if err := recreateCore(ctx, runner); err != nil {
-		return compensate(ctx, runner, request.StatePath, state, err)
+	running, err = activeSessions(ctx, runner)
+	if err != nil {
+		result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
+		return result, retErr
+	}
+	if running {
+		result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, ErrActiveSessions, hooks)
+		return result, retErr
+	}
+	if err := recreateCore(ctx, lifecycle); err != nil {
+		result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
+		return result, retErr
+	}
+	if err := ensureMaintenance(ctx, lifecycle); err != nil {
+		result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
+		return result, retErr
 	}
 	state.Phase = PhaseRecreated
-	state.Candidate, err = runningImage(ctx, runner, previous.Reference)
-	if err != nil { return compensate(ctx, runner, request.StatePath, state, err) }
-	if err := writeState(request.StatePath, state); err != nil {
-		return compensate(ctx, runner, request.StatePath, state, err)
+	state.Candidate, err = runningImage(ctx, lifecycle, candidateReference)
+	if err != nil {
+		result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
+		return result, retErr
 	}
-	if err := verifyCandidate(ctx, runner, request.Version, previous); err != nil {
-		return compensate(ctx, runner, request.StatePath, state, err)
+	if err := hooks.writeState(request.StatePath, state); err != nil {
+		result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
+		return result, retErr
+	}
+	if err := verifyCandidate(ctx, lifecycle, request.Version, previous); err != nil {
+		result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
+		return result, retErr
 	}
 	state.Phase, state.Error = PhaseVerified, ""
-	if err := writeState(request.StatePath, state); err != nil {
-		return compensate(ctx, runner, request.StatePath, state, err)
+	if err := hooks.writeState(request.StatePath, state); err != nil {
+		result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
+		return result, retErr
 	}
+	if err := hooks.removeFile(overridePath); err != nil {
+		return Result{Phase: PhaseFailed, StatePath: request.StatePath}, errors.New("verified update override cleanup failed: maintenance recovery required")
+	}
+	clearMaintenance = true
 	return Result{Phase: PhaseVerified, StatePath: request.StatePath}, nil
 }

 // Rollback restores the image recorded in durable update state. It is safe for interrupted runs.
 func Rollback(ctx context.Context, runner Runner, statePath string, confirm bool) (result Result, retErr error) {
+	return rollbackWithHooks(ctx, runner, statePath, confirm, defaultLifecycleHooks)
+}
+
+func rollbackWithHooks(ctx context.Context, runner Runner, statePath string, confirm bool, hooks lifecycleHooks) (result Result, retErr error) {
 	lock, err := acquireLock(statePath)
 	if err != nil {
 		return Result{StatePath: statePath}, err
@@ -175,51 +249,88 @@ func Rollback(ctx context.Context, runner Runner, statePath string, confirm bool
 	if !confirm {
 		return Result{StatePath: statePath}, ErrConfirmationRequired
 	}
-	if err := setMaintenance(ctx, runner, true); err != nil { return Result{StatePath: statePath}, err }
-	defer func() {
-		if clearErr := setMaintenance(context.Background(), runner, false); clearErr != nil {
-			result = Result{Phase: PhaseFailed, StatePath: statePath}
-			if retErr == nil { retErr = errors.New("maintenance admission gate could not be cleared: recovery required")
-			} else { retErr = fmt.Errorf("%w; maintenance admission gate could not be cleared: recovery required", retErr) }
-		}
-	}()
-	if active, err := activeSessions(ctx, runner); err != nil { return Result{StatePath: statePath}, err
-	} else if active { return Result{StatePath: statePath}, ErrActiveSessions }
-	state, err := readState(statePath)
-	if err != nil {
+	if err := setMaintenance(ctx, runner, true); err != nil {
 		return Result{StatePath: statePath}, err
 	}
-	if err := restore(ctx, runner, state.Previous); err != nil {
+	clearMaintenance := true
+	defer func() {
+		if !clearMaintenance {
+			return
+		}
+		if clearErr := setMaintenance(context.Background(), runner, false); clearErr != nil {
+			result = Result{Phase: PhaseFailed, StatePath: statePath}
+			if retErr == nil {
+				retErr = errors.New("maintenance admission gate could not be cleared: recovery required")
+			} else {
+				retErr = fmt.Errorf("%w; maintenance admission gate could not be cleared: recovery required", retErr)
+			}
+		}
+	}()
+	if active, err := activeSessions(ctx, runner); err != nil {
+		return Result{StatePath: statePath}, err
+	} else if active {
+		return Result{StatePath: statePath}, ErrActiveSessions
+	}
+	state, err := readState(statePath)
+	if err != nil {
+		clearMaintenance = false
+		return Result{StatePath: statePath}, err
+	}
+	overridePath := lifecycleOverridePath(statePath, state.Transaction)
+	if err := writeLifecycleOverride(overridePath, state.Previous.Reference); err != nil {
+		clearMaintenance = false
+		return Result{StatePath: statePath}, err
+	}
+	clearMaintenance = false
+	lifecycle := composeOverrideRunner{Runner: runner, path: overridePath}
+	if err := restore(ctx, lifecycle, state.Previous); err != nil {
 		state.Phase, state.Error = PhaseFailed, "rollback failed"
-		if writeErr := writeState(statePath, state); writeErr != nil { return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("rollback failed and recovery state could not be persisted") }
+		if writeErr := hooks.writeState(statePath, state); writeErr != nil {
+			return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("rollback failed and recovery state could not be persisted")
+		}
 		return Result{Phase: PhaseFailed, StatePath: statePath}, err
 	}
 	state.Phase, state.Error = PhaseRolledBack, ""
-	if err := writeState(statePath, state); err != nil {
+	if err := hooks.writeState(statePath, state); err != nil {
 		return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("rollback restored the core but recovery state could not be persisted")
 	}
+	if err := hooks.removeFile(overridePath); err != nil {
+		return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("rollback override cleanup failed: maintenance recovery required")
+	}
+	clearMaintenance = true
 	return Result{Phase: PhaseRolledBack, StatePath: statePath}, nil
 }

-func compensate(ctx context.Context, runner Runner, statePath string, state State, cause error) (Result, error) {
-	if restoreErr := restore(ctx, runner, state.Previous); restoreErr != nil {
+func compensate(ctx context.Context, runner Runner, statePath, overridePath string, state State, cause error, hooks lifecycleHooks) (Result, error, bool) {
+	if err := ensureMaintenance(context.Background(), runner); err != nil {
+		state.Phase, state.Error = PhaseFailed, "maintenance recovery failed"
+		_ = hooks.writeState(statePath, state)
+		return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("update failed and maintenance could not be reactivated: recovery required"), false
+	}
+	if active, err := activeSessions(context.Background(), runner); err != nil || active {
+		state.Phase, state.Error = PhaseFailed, "rollback inventory failed"
+		_ = hooks.writeState(statePath, state)
+		return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("update failed and rollback inventory is not quiescent: recovery required"), false
+	}
+	if err := writeLifecycleOverride(overridePath, state.Previous.Reference); err != nil {
+		return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("update failed and rollback override could not be prepared: recovery required"), false
+	}
+	lifecycle := composeOverrideRunner{Runner: runner, path: overridePath}
+	if restoreErr := restore(ctx, lifecycle, state.Previous); restoreErr != nil {
 		state.Phase, state.Error = PhaseFailed, "candidate verification and automatic rollback failed"
-		if writeErr := writeState(statePath, state); writeErr != nil {
-			return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("update failed and rollback proof failed; recovery state could not be persisted")
+		if writeErr := hooks.writeState(statePath, state); writeErr != nil {
+			return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("update failed and rollback proof failed; recovery state could not be persisted"), false
 		}
-		return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("update failed; automatic rollback also failed: recovery required")
+		return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("update failed; automatic rollback also failed: recovery required"), false
 	}
 	state.Phase, state.Error = PhaseRolledBack, ""
-	if writeErr := writeState(statePath, state); writeErr != nil {
-		return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("previous core image was restored but recovery state write failed: recovery required")
+	if writeErr := hooks.writeState(statePath, state); writeErr != nil {
+		return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("previous core image was restored but recovery state write failed: recovery required"), false
 	}
-	return Result{Phase: PhaseRolledBack, StatePath: statePath}, fmt.Errorf("update failed; previous core image was restored")
-}
-
-func recordFailure(path string, state State, label string, cause error) error {
-	state.Phase, state.Error = PhaseFailed, label
-	if err := writeState(path, state); err != nil { return fmt.Errorf("%w; recovery state write failed", cause) }
-	return cause
+	if err := hooks.removeFile(overridePath); err != nil {
+		return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("previous core image was restored but override cleanup failed: recovery required"), false
+	}
+	return Result{Phase: PhaseRolledBack, StatePath: statePath}, fmt.Errorf("update failed; previous core image was restored"), true
 }

 func sourceValue(request Request) string {
@@ -234,7 +345,9 @@ func canonicalDigestReference(value string) (string, error) {
 		return "", errors.New("pulled Pi image must be a credential-free canonical sha256 digest reference")
 	}
 	parsed, err := reference.ParseAnyReference(value)
-	if err != nil { return "", errors.New("pulled Pi image must be a valid canonical sha256 digest reference") }
+	if err != nil {
+		return "", errors.New("pulled Pi image must be a valid canonical sha256 digest reference")
+	}
 	canonical, ok := parsed.(reference.Canonical)
 	if !ok || canonical.Digest().Algorithm().String() != "sha256" || len(canonical.Digest().Encoded()) != 64 {
 		return "", errors.New("pulled Pi image must use an immutable sha256 digest")
@@ -244,14 +357,56 @@ func canonicalDigestReference(value string) (string, error) {

 func setMaintenance(ctx context.Context, runner Runner, enabled bool) error {
 	path := "deactivate"
-	if enabled { path = "activate" }
-	args := append([]string{"exec", "-T", "core", "curl", "-fsS", "-X", "POST"}, internalIdentityHeaders...)
-	args = append(args, "http://127.0.0.1:8787/internal/maintenance/"+path)
+	if enabled {
+		path = "activate"
+	}
+	args := []string{"exec", "-T", "core", "curl", "-fsS", "-X", "POST", "http://127.0.0.1:8787/internal/maintenance/" + path}
 	result, err := runCompose(ctx, runner, args...)
-	if err != nil { return commandError("maintenance admission gate", result, err) }
-	var status struct { Active bool `json:"active"`; Admissions int `json:"admissions"` }
-	if json.Unmarshal([]byte(result.Stdout), &status) != nil || status.Active != enabled || status.Admissions != 0 { return errors.New("maintenance admission gate did not acknowledge a quiescent state") }
-	return nil
+	status, valid := parseMaintenanceStatus(result.Stdout)
+	if err == nil && valid && status.Active == enabled && status.Admissions == 0 {
+		return nil
+	}
+	// A core recreate or transport interruption may lose only the response. Resolve ambiguity by
+	// reading the durable gate state before deciding that operator recovery is required.
+	observed, statusErr := MaintenanceStatus(ctx, runner)
+	if statusErr == nil && observed.Active == enabled && observed.Admissions == 0 {
+		return nil
+	}
+	if err != nil {
+		return commandError("maintenance admission gate", result, err)
+	}
+	return errors.New("maintenance admission gate did not acknowledge a quiescent state")
+}
+
+type MaintenanceState struct {
+	Active     bool `json:"active"`
+	Admissions int  `json:"admissions"`
+}
+
+func parseMaintenanceStatus(value string) (MaintenanceState, bool) {
+	var status MaintenanceState
+	err := json.Unmarshal([]byte(value), &status)
+	return status, err == nil && status.Admissions >= 0
+}
+
+func MaintenanceStatus(ctx context.Context, runner Runner) (MaintenanceState, error) {
+	result, err := runCompose(ctx, runner, "exec", "-T", "core", "curl", "-fsS", "http://127.0.0.1:8787/internal/maintenance/status")
+	if err != nil {
+		return MaintenanceState{}, commandError("maintenance status check", result, err)
+	}
+	status, valid := parseMaintenanceStatus(result.Stdout)
+	if !valid {
+		return MaintenanceState{}, errors.New("maintenance status check returned invalid data")
+	}
+	return status, nil
+}
+
+func ensureMaintenance(ctx context.Context, runner Runner) error {
+	status, err := MaintenanceStatus(ctx, runner)
+	if err == nil && status.Active && status.Admissions == 0 {
+		return nil
+	}
+	return setMaintenance(ctx, runner, true)
 }

 func activeSessions(ctx context.Context, runner Runner) (bool, error) {
@@ -261,14 +416,14 @@ func activeSessions(ctx context.Context, runner Runner) (bool, error) {
 	if err != nil {
 		return false, commandError("active-session check", result, err)
 	}
-	var payload struct { Sessions []struct {
+	var payload []struct {
 		Status   string `json:"status"`
 		Archived bool   `json:"archived"`
-	} `json:"sessions"` }
+	}
 	if err := json.Unmarshal([]byte(result.Stdout), &payload); err != nil {
 		return false, errors.New("active-session check returned invalid session data")
 	}
-	for _, session := range payload.Sessions {
+	for _, session := range payload {
 		if !session.Archived && session.Status != "finalized" && session.Status != "closed" {
 			return true, nil
 		}
@@ -291,9 +446,14 @@ func runningImage(ctx context.Context, runner Runner, reference string) (Image,
 		return Image{}, commandError("core volume check", mounts, err)
 	}
 	var raw []struct {
-		Type string `json:"Type"`; Name string `json:"Name"`; Source string `json:"Source"`
-		Destination string `json:"Destination"`; RW bool `json:"RW"`; Mode string `json:"Mode"`
-		Propagation string `json:"Propagation"`; Driver string `json:"Driver"`
+		Type        string `json:"Type"`
+		Name        string `json:"Name"`
+		Source      string `json:"Source"`
+		Destination string `json:"Destination"`
+		RW          bool   `json:"RW"`
+		Mode        string `json:"Mode"`
+		Propagation string `json:"Propagation"`
+		Driver      string `json:"Driver"`
 	}
 	if err := json.Unmarshal([]byte(mounts.Stdout), &raw); err != nil {
 		return Image{}, errors.New("core returned invalid persistence mount data")
@@ -302,18 +462,16 @@ func runningImage(ctx context.Context, runner Runner, reference string) (Image,
 		return Image{}, errors.New("core has no persistence mounts to preserve")
 	}
 	contract := make([]Mount, 0, len(raw))
-	volumes := make([]string, 0, len(raw))
 	for _, mount := range raw {
-		if mount.Type == "" || mount.Source == "" || mount.Destination == "" { return Image{}, errors.New("core returned incomplete persistence mount data") }
-		contract = append(contract, Mount{Type: mount.Type, Name: mount.Name, SourceSHA256: mountSourceHash(mount.Source), Destination: mount.Destination, RW: mount.RW, Options: strings.Join([]string{mount.Mode, mount.Propagation, mount.Driver}, "\x00")})
-		if mount.Type == "volume" && mount.Name != "" {
-			volumes = append(volumes, mount.Name)
+		if mount.Type == "" || mount.Source == "" || mount.Destination == "" {
+			return Image{}, errors.New("core returned incomplete persistence mount data")
 		}
+		contract = append(contract, Mount{Type: mount.Type, Name: mount.Name, SourceSHA256: mountSourceHash(mount.Source), Destination: mount.Destination, RW: mount.RW, Options: strings.Join([]string{mount.Mode, mount.Propagation, mount.Driver}, "\x00")})
 	}
-	return Image{ID: strings.TrimSpace(image.Stdout), Reference: reference, Volumes: volumes, Mounts: contract, MountFingerprint: mountFingerprint(contract)}, nil
+	return Image{ID: strings.TrimSpace(image.Stdout), Reference: reference, Mounts: contract, MountFingerprint: mountFingerprint(contract)}, nil
 }

-func prepareCandidate(ctx context.Context, runner Runner, request Request, reference string) error {
+func prepareCandidate(ctx context.Context, runner Runner, request Request, candidateReference string) error {
 	if request.Source == BuildSource {
 		result, err := runCompose(ctx, runner, "build", "--pull", "--build-arg", "PI_VERSION="+request.Version, "core")
 		if err != nil {
@@ -325,11 +483,7 @@ func prepareCandidate(ctx context.Context, runner Runner, request Request, refer
 	if err != nil {
 		return commandError("Pi image pull", pull, err)
 	}
-	tag, err := runner.Run(ctx, []string{"image", "tag", request.Image, reference}, nil)
-	if err != nil {
-		return commandError("Pi image tag", tag, err)
-	}
-	return nil
+	return tagImage(ctx, runner, request.Image, candidateReference, "Pi image tag")
 }

 func recreateCore(ctx context.Context, runner Runner) error {
@@ -373,13 +527,15 @@ func verifyCandidate(ctx context.Context, runner Runner, wanted string, previous
 }

 func restore(ctx context.Context, runner Runner, previous Image) error {
-	tag, err := runner.Run(ctx, []string{"image", "tag", previous.ID, previous.Reference}, nil)
-	if err != nil {
-		return commandError("rollback image restore", tag, err)
+	if err := tagImage(ctx, runner, previous.ID, previous.Reference, "rollback image restore"); err != nil {
+		return err
 	}
 	if err := recreateCore(ctx, runner); err != nil {
 		return err
 	}
+	if err := ensureMaintenance(ctx, runner); err != nil {
+		return err
+	}
 	configured, err := renderedCore(ctx, runner)
 	if err != nil {
 		return err
@@ -406,16 +562,91 @@ func restore(ctx context.Context, runner Runner, previous Image) error {
 	return nil
 }

-func nonEmptyLines(text string) []string {
-	var values []string
-	for _, value := range strings.Split(text, "\n") {
-		if value = strings.TrimSpace(value); value != "" {
-			values = append(values, value)
-		}
+func tagImage(ctx context.Context, runner Runner, source, target, label string) error {
+	result, err := runner.Run(ctx, []string{"image", "tag", source, target}, nil)
+	if err != nil {
+		return commandError(label, result, err)
 	}
-	sort.Strings(values)
-	return values
+	return nil
 }
+
+type composeOverrideRunner struct {
+	Runner
+	path string
+}
+
+func (r composeOverrideRunner) Run(ctx context.Context, args []string, stdin io.Reader) (compose.Result, error) {
+	if len(args) > 0 && args[0] == "compose" {
+		withOverride := append([]string{"compose", "-f", r.path}, args[1:]...)
+		return r.Runner.Run(ctx, withOverride, stdin)
+	}
+	return r.Runner.Run(ctx, args, stdin)
+}
+
+func lifecycleTransaction(statePath string) string {
+	value := fmt.Sprintf("%s\x00%d\x00%d", filepath.Clean(statePath), os.Getpid(), time.Now().UnixNano())
+	sum := sha256.Sum256([]byte(value))
+	return fmt.Sprintf("%x", sum[:8])
+}
+
+func lifecycleImageTag(transaction, role string) string {
+	return "thothii-core:thothctl-" + transaction + "-" + role
+}
+
+func lifecycleOverridePath(statePath, transaction string) string {
+	if transaction == "" {
+		transaction = "recovery"
+	}
+	return filepath.Join(filepath.Dir(statePath), "pi-lifecycle-"+transaction+".yaml")
+}
+
+func writeLifecycleOverride(path, image string) error {
+	quoted, err := json.Marshal(image)
+	if err != nil {
+		return errors.New("lifecycle image override could not be encoded")
+	}
+	contents := []byte("services:\n  core:\n    image: " + string(quoted) + "\n")
+	if err := writeFileDurably(path, ".pi-lifecycle-", contents); err != nil {
+		return errors.New("lifecycle image override could not be written durably")
+	}
+	return nil
+}
+
+// RecoverMaintenance clears a stale durable gate only after the running core and terminal
+// recovery metadata prove that no rollback is still required.
+func RecoverMaintenance(ctx context.Context, runner Runner, statePath string, confirm bool) error {
+	if !confirm {
+		return ErrConfirmationRequired
+	}
+	lock, err := acquireLock(statePath)
+	if err != nil {
+		return err
+	}
+	defer lock.Release()
+	state, stateErr := readState(statePath)
+	if stateErr == nil {
+		if state.Phase != PhaseVerified && state.Phase != PhaseRolledBack && state.Phase != PhaseNoop {
+			return ErrInterruptedUpdate
+		}
+		if err := durableRemove(lifecycleOverridePath(statePath, state.Transaction)); err != nil {
+			return errors.New("maintenance recovery could not remove the lifecycle override")
+		}
+	} else if !errors.Is(stateErr, os.ErrNotExist) {
+		return stateErr
+	}
+	status, err := MaintenanceStatus(ctx, runner)
+	if err != nil {
+		return err
+	}
+	if !status.Active {
+		return nil
+	}
+	if err := Doctor(ctx, runner); err != nil {
+		return err
+	}
+	return setMaintenance(ctx, runner, false)
+}
+
 func sameStrings(left, right []string) bool {
 	left, right = append([]string(nil), left...), append([]string(nil), right...)
 	sort.Strings(left)