fix(auth): harden backup restore lifecycle cleanup

tools/tht/internal/backup/restore.go

@@ -47,6 +47,26 @@ type restoreDependencies struct {
 	verify                   map[string]restoreVerify
 }
 
+// restoreTransactionState is the restore admission state machine. The durable maintenance
+// barrier is released only after the target archive has been verified, or after a separately
+// verified checkpoint recovery. Every mutable Docker command is tracked before it is invoked.
+type restoreTransactionState struct {
+	wasRunning           bool
+	maintenanceAttempted bool
+	stopAttempted        bool
+	mutated              bool
+	verified             bool
+	recovered            bool
+}
+
+func (state restoreTransactionState) recoveryRequired(resultErr error) bool {
+	return resultErr != nil && state.mutated && !state.verified
+}
+
+func (state restoreTransactionState) mayDeactivateMaintenance() bool {
+	return !state.mutated || state.verified || state.recovered
+}
+
 // Restore runs the host transaction through the same concrete Docker/filesystem boundaries used
 // by backup creation. The injectable core below exists only to make every failure boundary
 // deterministic in tests.
@@ -94,8 +114,9 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
 		return result, err
 	}
 	defer func() {
-		if releaseErr := lock.Release(); releaseErr != nil && resultErr == nil {
-			resultErr = releaseErr
+		if releaseErr := lock.Release(); releaseErr != nil {
+			result = RestoreResult{}
+			resultErr = errors.Join(resultErr, fmt.Errorf("release restore lifecycle lock: %w", releaseErr))
 		}
 	}()
 
@@ -103,35 +124,63 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
 	if err != nil {
 		return result, err
 	}
-	mutated := false
-	maintenanceAttempted := false
-	stopAttempted := false
+	state := restoreTransactionState{wasRunning: wasRunning}
 	defer func() {
-		if resultErr != nil && mutated {
-			if recoveryErr := deps.recover(context.Background(), installation, recovery, wasRunning); recoveryErr != nil {
+		if state.recoveryRequired(resultErr) {
+			recoveryContext, cancel := boundedCleanupContext()
+			recoveryErr := deps.recover(recoveryContext, installation, recovery, state.wasRunning)
+			cancel()
+			if recoveryErr != nil {
 				resultErr = errors.Join(resultErr, fmt.Errorf("restore recovery checkpoint: %w", recoveryErr))
+				if recoveryReachedVerifiedState(recoveryErr) {
+					state.recovered = true
+					state.stopAttempted = false
+				}
 			} else {
-				stopAttempted = false
+				state.recovered = true
+				state.stopAttempted = false
 			}
 		}
-		if maintenanceAttempted {
-			cleanupContext := context.Background()
-			if wasRunning && stopAttempted {
-				if startErr := composeStartAndVerify(cleanupContext, installation, deps.runner); startErr != nil {
-					resultErr = errors.Join(resultErr, fmt.Errorf("restore maintenance cleanup restart: %w", startErr))
-				} else {
-					stopAttempted = false
-				}
+		if !state.maintenanceAttempted {
+			return
+		}
+		var cleanupErr error
+		restartCompleted := true
+		if state.wasRunning && state.stopAttempted && state.mayDeactivateMaintenance() {
+			startErr, started := retryBoundedCleanup(func(cleanupContext context.Context) error {
+				return composeStartAndVerify(cleanupContext, installation, deps.runner)
+			})
+			if startErr != nil {
+				cleanupErr = errors.Join(cleanupErr, fmt.Errorf("restore maintenance cleanup restart: %w", startErr))
 			}
-			if deactivateErr := maintenance(cleanupContext, installation, deps.runner, false); deactivateErr != nil {
-				resultErr = errors.Join(resultErr, fmt.Errorf("restore maintenance cleanup: %w", deactivateErr))
+			if started {
+				state.stopAttempted = false
+			} else {
+				restartCompleted = false
 			}
 		}
+		// A failed checkpoint recovery deliberately leaves admissions blocked. Starting or
+		// deactivating at that point would expose an unverified, possibly partial restore.
+		if state.mayDeactivateMaintenance() && restartCompleted {
+			deactivateErr, deactivated := retryBoundedCleanup(func(cleanupContext context.Context) error {
+				return maintenance(cleanupContext, installation, deps.runner, false)
+			})
+			if deactivateErr != nil {
+				cleanupErr = errors.Join(cleanupErr, fmt.Errorf("restore maintenance cleanup: %w", deactivateErr))
+			}
+			if deactivated {
+				state.maintenanceAttempted = false
+			}
+		}
+		if cleanupErr != nil {
+			result = RestoreResult{}
+			resultErr = errors.Join(resultErr, cleanupErr)
+		}
 	}()
-	if wasRunning {
+	if state.wasRunning {
 		// The activation command may take effect even when its response is lost. Track the attempt,
-		// not merely a successful return, so every subsequent path removes the admissions barrier.
-		maintenanceAttempted = true
+		// not merely a successful return, so every subsequent path compensates from durable state.
+		state.maintenanceAttempted = true
 		if err := maintenance(ctx, installation, deps.runner, true); err != nil {
 			return result, err
 		}
@@ -140,40 +189,61 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
 		}
 		// Compose may stop the core and then lose its response. Cleanup must therefore restart after
 		// any stop attempt, including a command that returns an error.
-		stopAttempted = true
+		state.stopAttempted = true
 		if err := runCompose(ctx, installation, deps.runner, "stop"); err != nil {
 			return result, err
 		}
 	}
-	mutated = true
+	state.mutated = true
 	if err := restoreVerifiedEntries(ctx, installation, preflight, archive, deps.restoreFile, deps.restoreVolume); err != nil {
 		return result, err
 	}
 	if err := deps.resetAuthenticationState(ctx, installation, deps.runner); err != nil {
 		return result, fmt.Errorf("reset authentication state: %w", err)
 	}
-	if wasRunning {
+	if state.wasRunning {
 		if err := composeStartAndVerify(ctx, installation, deps.runner); err != nil {
 			return result, err
 		}
-		stopAttempted = false
+		state.stopAttempted = false
 		result.Restarted = true
+	}
+	if err := verifyRestoreTransaction(ctx, installation, deps); err != nil {
+		return result, err
+	}
+	state.verified = true
+	result.Verified = true
+	if state.wasRunning {
 		if err := maintenance(ctx, installation, deps.runner, false); err != nil {
 			return result, err
 		}
-		maintenanceAttempted = false
+		state.maintenanceAttempted = false
 	}
+	return result, nil
+}
+
+func verifyRestoreTransaction(ctx context.Context, installation config.Installation, deps restoreDependencies) error {
 	for _, name := range []string{"health", "doctor", "pi", "workspace"} {
 		check := deps.verify[name]
 		if check == nil {
-			return result, fmt.Errorf("restore verification %q is unavailable", name)
+			return fmt.Errorf("restore verification %q is unavailable", name)
 		}
 		if err := check(ctx, installation, deps.runner); err != nil {
-			return result, fmt.Errorf("restore verification %s: %w", name, err)
+			return fmt.Errorf("restore verification %s: %w", name, err)
 		}
 	}
-	result.Verified = true
-	return result, nil
+	return nil
+}
+
+type verifiedRecoveryError struct{ err error }
+
+func (err *verifiedRecoveryError) Error() string { return err.err.Error() }
+
+func (err *verifiedRecoveryError) Unwrap() error { return err.err }
+
+func recoveryReachedVerifiedState(err error) bool {
+	var verifiedErr *verifiedRecoveryError
+	return errors.As(err, &verifiedErr)
 }
 
 func restoreVerifiedEntries(