fix(auth): harden backup restore lifecycle cleanup

tools/tht/internal/backup/create.go

@@ -56,6 +56,10 @@ const (
 	helperImage       = "busybox:1.36.1"
 	drainPollInterval = time.Second
 	maxDrainPolls     = 300
+	// Cleanup must survive a caller timeout or a lost Docker response, but it must not run
+	// indefinitely after the command has returned. Archive recovery can require volume work, so
+	// keep this deliberately longer than an individual health check.
+	cleanupOperationTimeout = 5 * time.Minute
 )
 
 // CreateRequest controls one explicit backup request.
@@ -124,9 +128,9 @@ func createWithDependencies(ctx context.Context, installation config.Installatio
 		return Result{}, err
 	}
 	defer func() {
-		if releaseErr := lock.Release(); releaseErr != nil && resultErr == nil {
+		if releaseErr := lock.Release(); releaseErr != nil {
 			result = Result{}
-			resultErr = releaseErr
+			resultErr = errors.Join(resultErr, fmt.Errorf("release backup lifecycle lock: %w", releaseErr))
 		}
 	}()
 
@@ -187,35 +191,57 @@ func createWithDependencies(ctx context.Context, installation config.Installatio
 	if err != nil {
 		return Result{}, err
 	}
-	maintenanceActive := false
-	stopped := false
+	// Record attempted mutations before invoking Docker. Docker can apply a mutation and then
+	// lose its response, so a successful return is not evidence that compensation is unnecessary.
+	maintenanceAttempted := false
+	stopAttempted := false
 	defer func() {
-		if stopped {
-			if startErr := composeStartAndVerify(ctx, installation, dependencies.runner); startErr != nil && resultErr == nil {
-				result = Result{}
-				resultErr = startErr
+		var cleanupErr error
+		restartCompleted := true
+		if wasRunning && stopAttempted {
+			startErr, started := retryBoundedCleanup(func(cleanupContext context.Context) error {
+				return composeStartAndVerify(cleanupContext, installation, dependencies.runner)
+			})
+			if startErr != nil {
+				cleanupErr = errors.Join(cleanupErr, fmt.Errorf("backup maintenance cleanup restart: %w", startErr))
+			}
+			if started {
+				stopAttempted = false
+			} else {
+				restartCompleted = false
 			}
 		}
-		if maintenanceActive {
-			if deactivateErr := maintenance(ctx, installation, dependencies.runner, false); deactivateErr != nil && resultErr == nil {
-				result = Result{}
-				resultErr = deactivateErr
+		// Do not reopen admissions while the installation is known to be stopped. If a retry could
+		// not establish a running core, retain the durable barrier and report every cleanup error.
+		if maintenanceAttempted && restartCompleted {
+			deactivateErr, deactivated := retryBoundedCleanup(func(cleanupContext context.Context) error {
+				return maintenance(cleanupContext, installation, dependencies.runner, false)
+			})
+			if deactivateErr != nil {
+				cleanupErr = errors.Join(cleanupErr, fmt.Errorf("backup maintenance cleanup: %w", deactivateErr))
 			}
+			if deactivated {
+				maintenanceAttempted = false
+			}
+		}
+		if cleanupErr != nil {
+			result = Result{}
+			resultErr = errors.Join(resultErr, cleanupErr)
 		}
 	}()
 
 	if wasRunning {
+		maintenanceAttempted = true
 		if err := maintenance(ctx, installation, dependencies.runner, true); err != nil {
 			return Result{}, err
 		}
-		maintenanceActive = true
 		if err := waitForNoActiveSessions(ctx, installation, dependencies.runner, request.Drain, dependencies.sleep); err != nil {
 			return Result{}, err
 		}
+		stopAttempted = true
 		if err := runCompose(ctx, installation, dependencies.runner, "stop"); err != nil {
 			return Result{}, err
 		}
-		stopped = true
 	}
 
 	manifest := Manifest{
@@ -236,11 +262,11 @@ func createWithDependencies(ctx context.Context, installation config.Installatio
 		if err := composeStartAndVerify(ctx, installation, dependencies.runner); err != nil {
 			return Result{}, err
 		}
-		stopped = false
+		stopAttempted = false
 		if err := maintenance(ctx, installation, dependencies.runner, false); err != nil {
 			return Result{}, err
 		}
-		maintenanceActive = false
+		maintenanceAttempted = false
 	}
 	result = Result{Path: output}
 	if manifest.IncludesSecrets {
@@ -689,6 +715,27 @@ func composeStartAndVerify(ctx context.Context, installation config.Installation
 	return service.WaitForHealthy(ctx, installation, runner)
 }
 
+func boundedCleanupContext() (context.Context, context.CancelFunc) {
+	return context.WithTimeout(context.Background(), cleanupOperationTimeout)
+}
+
+// retryBoundedCleanup retries an idempotent compensating mutation once. It preserves a lost
+// response as part of the returned error while reporting whether the retry established the final
+// state needed by the next cleanup action.
+func retryBoundedCleanup(operation func(context.Context) error) (error, bool) {
+	run := func() error {
+		cleanupContext, cancel := boundedCleanupContext()
+		defer cancel()
+		return operation(cleanupContext)
+	}
+	firstErr := run()
+	if firstErr == nil {
+		return nil, true
+	}
+	retryErr := run()
+	return errors.Join(firstErr, retryErr), retryErr == nil
+}
+
 func runCompose(ctx context.Context, installation config.Installation, runner archiveRunner, command ...string) error {
 	result, err := runner.Run(ctx, installation.ComposeArgs(command...), nil)
 	if err != nil {