fix: harden pi maintenance lifecycle

tools/thothctl/internal/pi/state.go

@@ -5,13 +5,15 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"crypto/sha256"
 	"os"
 	"path/filepath"
-	"runtime"
+	"sort"
+	"strings"
 	"time"
 )
 
-const stateFileVersion = 1
+const stateFileVersion = 2
 
 // Phase describes the durable point reached by a Pi update.
 type Phase string
@@ -32,6 +34,7 @@ type Image struct {
 	Reference        string   `json:"reference"`
 	Volumes          []string `json:"volumes"`
 	Mounts           []Mount  `json:"mounts"`
+	MountFingerprint string   `json:"mount_fingerprint"`
 	ConfigurationSHA string   `json:"configuration_sha256,omitempty"`
 }
 
@@ -39,9 +42,10 @@ type Image struct {
 type Mount struct {
 	Type        string `json:"type"`
 	Name        string `json:"name,omitempty"`
-	Source      string `json:"source"`
+	SourceSHA256 string `json:"source_sha256"`
 	Destination string `json:"destination"`
 	RW          bool   `json:"rw"`
+	Options     string `json:"options,omitempty"`
 }
 
 // Target records the immutable input selected by the operator. Source is either build or a
@@ -72,14 +76,14 @@ func readState(path string) (State, error) {
 	if err := json.Unmarshal(contents, &state); err != nil {
 		return State{}, errors.New("update recovery state is invalid")
 	}
-	if state.Version != stateFileVersion || state.Previous.ID == "" || state.Previous.Reference == "" {
+	if state.Version != stateFileVersion || state.Previous.ID == "" || state.Previous.Reference == "" || state.Previous.MountFingerprint == "" {
 		return State{}, errors.New("update recovery state is incomplete")
 	}
 	return state, nil
 }
 
 func writeState(path string, state State) error {
-	if state.Previous.ID == "" || state.Previous.Reference == "" {
+	if state.Previous.ID == "" || state.Previous.Reference == "" || state.Previous.MountFingerprint == "" {
 		return errors.New("refusing to write incomplete update recovery state")
 	}
 	state.Version = stateFileVersion
@@ -89,45 +93,52 @@ func writeState(path string, state State) error {
 		return fmt.Errorf("encode update recovery state: %w", err)
 	}
 	contents = append(contents, '\n')
-	directory := filepath.Dir(path)
-	if err := os.MkdirAll(directory, 0o700); err != nil {
-		return errors.New("could not create update recovery directory")
-	}
-	temporary, err := os.CreateTemp(directory, ".update-state-*.tmp")
-	if err != nil {
-		return errors.New("could not write update recovery state")
-	}
-	temporaryName := temporary.Name()
-	defer os.Remove(temporaryName)
-	if err := temporary.Chmod(0o600); err != nil {
-		temporary.Close()
-		return errors.New("could not protect update recovery state")
-	}
-	if _, err := temporary.Write(contents); err != nil {
-		temporary.Close()
-		return errors.New("could not write update recovery state")
-	}
-	if err := temporary.Sync(); err != nil {
-		temporary.Close()
-		return errors.New("could not durably write update recovery state")
-	}
-	if err := temporary.Close(); err != nil {
-		return errors.New("could not write update recovery state")
-	}
-	if err := os.Rename(temporaryName, path); err != nil {
-		return errors.New("could not finalize update recovery state")
-	}
-	if runtime.GOOS != "windows" {
-		if directoryHandle, err := os.Open(directory); err == nil {
-			_ = directoryHandle.Sync()
-			_ = directoryHandle.Close()
-		}
+	if err := writeFileDurably(path, ".update-state-", contents); err != nil {
+		return fmt.Errorf("could not durably write update recovery state: %w", err)
 	}
 	return nil
 }
 
+func writeFileDurably(path, prefix string, contents []byte) error {
+	directory := filepath.Dir(path)
+	if err := os.MkdirAll(directory, 0o700); err != nil { return err }
+	temporary, err := os.CreateTemp(directory, prefix+"*.tmp")
+	if err != nil { return err }
+	temporaryName := temporary.Name()
+	defer os.Remove(temporaryName)
+	if err := temporary.Chmod(0o600); err != nil { temporary.Close(); return err }
+	if _, err := temporary.Write(contents); err != nil { temporary.Close(); return err }
+	if err := temporary.Sync(); err != nil { temporary.Close(); return err }
+	if err := temporary.Close(); err != nil { return err }
+	return durableReplace(temporaryName, path, directory)
+}
+
+func mountSourceHash(source string) string {
+	sum := sha256.Sum256([]byte(source))
+	return fmt.Sprintf("%x", sum[:])
+}
+
+func mountFingerprint(mounts []Mount) string {
+	values := make([]string, len(mounts))
+	for i, mount := range mounts {
+		values[i] = strings.Join([]string{mount.Type, mount.Name, mount.SourceSHA256, mount.Destination, fmt.Sprint(mount.RW), mount.Options}, "\x00")
+	}
+	sort.Strings(values)
+	sum := sha256.Sum256([]byte(strings.Join(values, "\n")))
+	return fmt.Sprintf("%x", sum[:])
+}
+
+type lockOwner struct {
+	PID         int       `json:"pid"`
+	Host        string    `json:"host"`
+	StartedAt   time.Time `json:"started_at"`
+	Transaction string    `json:"transaction"`
+}
+
 type updateLock struct{ path string }
 
+var ErrLockHeld = errors.New("another Pi update or rollback is already in progress")
+
 func acquireLock(statePath string) (*updateLock, error) {
 	if err := os.MkdirAll(filepath.Dir(statePath), 0o700); err != nil {
 		return nil, errors.New("could not create Pi update recovery directory")
@@ -135,10 +146,36 @@ func acquireLock(statePath string) (*updateLock, error) {
 	path := statePath + ".lock"
 	if err := os.Mkdir(path, 0o700); err != nil {
 		if errors.Is(err, os.ErrExist) {
-			return nil, errors.New("another Pi update or rollback is already in progress; recovery lock retained")
+			if reclaimDeadLocalLock(path) {
+				return acquireLock(statePath)
+			}
+			return nil, ErrLockHeld
 		}
 		return nil, errors.New("could not acquire Pi update lock")
 	}
+	host, err := os.Hostname()
+	if err != nil { _ = os.Remove(path); return nil, errors.New("could not identify Pi update lock owner") }
+	owner := lockOwner{PID: os.Getpid(), Host: host, StartedAt: time.Now().UTC(), Transaction: fmt.Sprintf("%d-%d", os.Getpid(), time.Now().UnixNano())}
+	contents, err := json.Marshal(owner)
+	if err != nil { _ = os.Remove(path); return nil, errors.New("could not record Pi update lock owner") }
+	if err := writeFileDurably(filepath.Join(path, "owner.json"), ".owner-", append(contents, '\n')); err != nil {
+		_ = os.Remove(path)
+		return nil, errors.New("could not record Pi update lock owner")
+	}
 	return &updateLock{path: path}, nil
 }
-func (l *updateLock) Release() { _ = os.Remove(l.path) }
+func (l *updateLock) Release() { _ = os.Remove(filepath.Join(l.path, "owner.json")); _ = os.Remove(l.path) }
+
+// reclaimDeadLocalLock is deliberately conservative: a malformed, remote, or merely old lock
+// is recovery-required. Only a process we can prove is gone on this machine is reclaimed.
+func reclaimDeadLocalLock(path string) bool {
+	contents, err := os.ReadFile(filepath.Join(path, "owner.json"))
+	if err != nil { return false }
+	var owner lockOwner
+	if json.Unmarshal(contents, &owner) != nil || owner.PID <= 0 || owner.Host == "" { return false }
+	host, err := os.Hostname()
+	if err != nil || owner.Host != host { return false }
+	if processAlive(owner.PID) { return false }
+	if err := os.Remove(filepath.Join(path, "owner.json")); err != nil { return false }
+	return os.Remove(path) == nil
+}