fix(cli): harden backup publication and quiescing

2026-08-16 01:11:59 +02:00
parent 11fbf0a138
commit e4d0097639
4 changed files with 256 additions and 43 deletions
@@ -70,12 +70,13 @@ type archiveRunner interface {
 }

 type dependencies struct {
-	runner   archiveRunner
-	now      func() time.Time
-	homeDir  func() (string, error)
-	revision func(context.Context, string) (string, error)
-	sleep    func(time.Duration)
-	rename   func(string, string) error
+	runner          archiveRunner
+	now             func() time.Time
+	homeDir         func() (string, error)
+	revision        func(context.Context, string) (string, error)
+	sleep           func(time.Duration)
+	reserveOutput   func(string) (*archiveReservation, error)
+	publishReserved func(*archiveReservation, string) error
 }

 // Create creates an archive with the real Docker command boundary. It performs no shell
@@ -94,13 +95,14 @@ func Create(ctx context.Context, installation config.Installation, request Creat
 			}
 			return strings.TrimSpace(string(value)), nil
 		},
-		sleep:  time.Sleep,
-		rename: os.Rename,
+		sleep:           time.Sleep,
+		reserveOutput:   reserveArchiveOutput,
+		publishReserved: publishReservedArchive,
 	})
 }

 func createWithDependencies(ctx context.Context, installation config.Installation, request CreateRequest, dependencies dependencies) (result Result, resultErr error) {
-	if dependencies.runner == nil || dependencies.now == nil || dependencies.homeDir == nil || dependencies.revision == nil || dependencies.sleep == nil || dependencies.rename == nil {
+	if dependencies.runner == nil || dependencies.now == nil || dependencies.homeDir == nil || dependencies.revision == nil || dependencies.sleep == nil || dependencies.reserveOutput == nil || dependencies.publishReserved == nil {
 		return Result{}, errors.New("backup dependencies are incomplete")
 	}
 	if request.IncludeSecrets && !request.Confirm {
@@ -142,6 +144,16 @@ func createWithDependencies(ctx context.Context, installation config.Installatio
 	if err := ensureNewArchivePath(output); err != nil {
 		return Result{}, err
 	}
+	reservation, err := dependencies.reserveOutput(output)
+	if err != nil {
+		return Result{}, err
+	}
+	published := false
+	defer func() {
+		if !published {
+			_ = reservation.RemoveIfOwned()
+		}
+	}()

 	rendered, err := renderedConfiguration(ctx, installation, dependencies.runner)
 	if err != nil {
@@ -201,9 +213,10 @@ func createWithDependencies(ctx context.Context, installation config.Installatio
 		Images:          images,
 		Volumes:         volumes,
 	}
-	if err := writeArchive(ctx, output, installation, request, secretPaths, manifest, volumes, dependencies); err != nil {
+	if err := writeArchive(ctx, output, reservation, installation, request, secretPaths, manifest, volumes, dependencies); err != nil {
 		return Result{}, err
 	}
+	published = true
 	if wasRunning {
 		if err := composeStartAndVerify(ctx, installation, dependencies.runner); err != nil {
 			return Result{}, err
@@ -286,15 +299,90 @@ func ensureNewArchivePath(output string) error {
 	if err := os.MkdirAll(filepath.Dir(output), 0o700); err != nil {
 		return fmt.Errorf("create backup directory: %w", err)
 	}
-	if info, err := os.Lstat(output); err == nil || !errors.Is(err, os.ErrNotExist) {
-		if err == nil && info.Mode().IsRegular() {
-			return errors.New("backup output already exists")
-		}
-		return errors.New("backup output path is unavailable")
+	return nil
+}
+
+// archiveReservation claims an output path without replacing an existing archive. Its file
+// identity is retained so cleanup never removes a path another process took over.
+type archiveReservation struct {
+	path string
+	file *os.File
+	info os.FileInfo
+}
+
+func reserveArchiveOutput(output string) (*archiveReservation, error) {
+	file, err := os.OpenFile(output, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
+	if errors.Is(err, os.ErrExist) {
+		return nil, errors.New("backup output already exists")
+	}
+	if err != nil {
+		return nil, fmt.Errorf("reserve backup output: %w", err)
+	}
+	info, statErr := file.Stat()
+	if statErr != nil {
+		_ = file.Close()
+		return nil, fmt.Errorf("inspect reserved backup output: %w", statErr)
+	}
+	return &archiveReservation{path: output, file: file, info: info}, nil
+}
+
+func publishReservedArchive(reservation *archiveReservation, temporaryPath string) error {
+	if reservation == nil || reservation.file == nil {
+		return errors.New("backup output reservation is unavailable")
+	}
+	temporary, err := os.Open(temporaryPath)
+	if err != nil {
+		return fmt.Errorf("open temporary backup archive: %w", err)
+	}
+	defer temporary.Close()
+	if err := reservation.file.Truncate(0); err != nil {
+		return fmt.Errorf("prepare reserved backup output: %w", err)
+	}
+	if _, err := reservation.file.Seek(0, io.SeekStart); err != nil {
+		return fmt.Errorf("seek reserved backup output: %w", err)
+	}
+	if _, err := io.Copy(reservation.file, temporary); err != nil {
+		return fmt.Errorf("write reserved backup output: %w", err)
+	}
+	if err := reservation.file.Sync(); err != nil {
+		return fmt.Errorf("fsync reserved backup output: %w", err)
+	}
+	if err := reservation.file.Close(); err != nil {
+		return fmt.Errorf("close reserved backup output: %w", err)
+	}
+	reservation.file = nil
+	current, err := os.Stat(reservation.path)
+	if err != nil || !os.SameFile(reservation.info, current) {
+		return errors.New("backup output ownership changed before publication")
 	}
 	return nil
 }

+// RemoveIfOwned removes the reservation only when the output path still names the exact file
+// created by this invocation. It is safe to call after another process has claimed the path.
+func (reservation *archiveReservation) RemoveIfOwned() error {
+	if reservation == nil {
+		return nil
+	}
+	if reservation.file != nil {
+		if err := reservation.file.Close(); err != nil {
+			return err
+		}
+		reservation.file = nil
+	}
+	current, err := os.Stat(reservation.path)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	if !os.SameFile(reservation.info, current) {
+		return nil
+	}
+	return os.Remove(reservation.path)
+}
+
 type renderedCompose struct {
 	Volumes map[string]struct {
 		Name string `json:"name"`
@@ -398,11 +486,74 @@ func imageIdentities(ctx context.Context, installation config.Installation, runn
 }

 func installationRunning(ctx context.Context, installation config.Installation, runner archiveRunner) (bool, error) {
-	result, err := runner.Run(ctx, installation.ComposeArgs("ps", "--status", "running", "-q"), nil)
+	result, err := runner.Run(ctx, installation.ComposeArgs("ps", "--all", "--format", "json"), nil)
 	if err != nil {
-		return false, dockerError("inspect running services", result, err)
+		return false, dockerError("inspect service states", result, err)
 	}
-	return strings.TrimSpace(result.Stdout) != "", nil
+	states, err := backupServiceStates(result.Stdout)
+	if err != nil {
+		return false, err
+	}
+	running := false
+	coreRunning := false
+	for _, service := range states {
+		switch service.State {
+		case "exited", "dead":
+			continue
+		case "running":
+			running = true
+			if service.Service == "core" {
+				coreRunning = true
+			}
+		default:
+			return false, fmt.Errorf("service %q is %q and is not safely quiesced; stop the installation before backup", service.Service, service.State)
+		}
+	}
+	if running && !coreRunning {
+		return false, errors.New("core is not running while other services are active; stop the installation before backup")
+	}
+	return running, nil
+}
+
+type backupServiceState struct {
+	Service string `json:"Service"`
+	State   string `json:"State"`
+}
+
+func backupServiceStates(value string) ([]backupServiceState, error) {
+	trimmed := strings.TrimSpace(value)
+	if trimmed == "" || trimmed == "[]" {
+		return nil, nil
+	}
+	var array []backupServiceState
+	if err := json.Unmarshal([]byte(trimmed), &array); err == nil {
+		return normalizeBackupServiceStates(array)
+	}
+	decoder := json.NewDecoder(strings.NewReader(trimmed))
+	var states []backupServiceState
+	for {
+		var state backupServiceState
+		err := decoder.Decode(&state)
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return nil, errors.New("Docker Compose returned invalid service states")
+		}
+		states = append(states, state)
+	}
+	return normalizeBackupServiceStates(states)
+}
+
+func normalizeBackupServiceStates(states []backupServiceState) ([]backupServiceState, error) {
+	for index := range states {
+		states[index].Service = strings.TrimSpace(states[index].Service)
+		states[index].State = strings.ToLower(strings.TrimSpace(states[index].State))
+		if states[index].Service == "" || states[index].State == "" {
+			return nil, errors.New("Docker Compose returned invalid service states")
+		}
+	}
+	return states, nil
 }

 func maintenance(ctx context.Context, installation config.Installation, runner archiveRunner, activate bool) error {
@@ -480,14 +631,13 @@ func runCompose(ctx context.Context, installation config.Installation, runner ar
 	return nil
 }

-func writeArchive(ctx context.Context, output string, installation config.Installation, request CreateRequest, secretPaths []string, manifest Manifest, volumes []VolumeMetadata, dependencies dependencies) (resultErr error) {
+func writeArchive(ctx context.Context, output string, reservation *archiveReservation, installation config.Installation, request CreateRequest, secretPaths []string, manifest Manifest, volumes []VolumeMetadata, dependencies dependencies) (resultErr error) {
 	directory := filepath.Dir(output)
 	temporary, err := os.CreateTemp(directory, ".tht-backup-*.tmp")
 	if err != nil {
 		return fmt.Errorf("create temporary backup archive: %w", err)
 	}
 	temporaryPath := temporary.Name()
-	published := false
 	closedFile := false
 	defer func() {
 		var closeErr error
@@ -497,10 +647,7 @@ func writeArchive(ctx context.Context, output string, installation config.Instal
 		if closeErr != nil && resultErr == nil {
 			resultErr = closeErr
 		}
-		if !published {
-			_ = os.Remove(temporaryPath)
-			_ = os.Remove(output)
-		}
+		_ = os.Remove(temporaryPath)
 	}()
 	if err := temporary.Chmod(0o600); err != nil {
 		return fmt.Errorf("protect temporary backup archive: %w", err)
@@ -632,10 +779,9 @@ func writeArchive(ctx context.Context, output string, installation config.Instal
 		return fmt.Errorf("close backup archive: %w", err)
 	}
 	closedFile = true
-	if err := dependencies.rename(temporaryPath, output); err != nil {
+	if err := dependencies.publishReserved(reservation, temporaryPath); err != nil {
 		return fmt.Errorf("publish backup archive: %w", err)
 	}
-	published = true
 	return nil
 }