mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix: harden Pi lifecycle recovery

Browse Source
This commit is contained in:
marcopan
2026-08-04 20:28:09 +02:00
parent 5b3ce93e31
commit 5ba2821a1b
24 changed files with 1764 additions and 384 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/thothctl/internal/pi/state.go
+67 -61
View File
@@ -2,18 +2,20 @@
package pi
import (
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"crypto/sha256"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/gofrs/flock"
)
const stateFileVersion = 2
const stateFileVersion = 3
// Phase describes the durable point reached by a Pi update.
type Phase string
@@ -30,22 +32,21 @@ const (
// Image is the non-secret recovery identity of a core image and its mounted volume names.
type Image struct {
ID string `json:"id"`
Reference string `json:"reference"`
Volumes []string `json:"volumes"`
Mounts []Mount `json:"mounts"`
MountFingerprint string `json:"mount_fingerprint"`
ConfigurationSHA string `json:"configuration_sha256,omitempty"`
ID string `json:"id"`
Reference string `json:"reference"`
Mounts []Mount `json:"mounts"`
MountFingerprint string `json:"mount_fingerprint"`
ConfigurationSHA string `json:"configuration_sha256,omitempty"`
}
// Mount is the complete persistence identity relevant to safe core recreation.
type Mount struct {
Type string `json:"type"`
Name string `json:"name,omitempty"`
Type string `json:"type"`
Name string `json:"name,omitempty"`
SourceSHA256 string `json:"source_sha256"`
Destination string `json:"destination"`
RW bool `json:"rw"`
Options string `json:"options,omitempty"`
Destination string `json:"destination"`
RW bool `json:"rw"`
Options string `json:"options,omitempty"`
}
// Target records the immutable input selected by the operator. Source is either build or a
@@ -58,13 +59,14 @@ type Target struct {
// State is recovery metadata stored below the installation project. It never stores environment
// values, secret paths, credentials, or command output.
type State struct {
Version int `json:"version"`
Phase Phase `json:"phase"`
UpdatedAt time.Time `json:"updated_at"`
Target Target `json:"target,omitempty"`
Previous Image `json:"previous"`
Candidate Image `json:"candidate,omitempty"`
Error string `json:"error,omitempty"`
Version int `json:"version"`
Transaction string `json:"transaction"`
Phase Phase `json:"phase"`
UpdatedAt time.Time `json:"updated_at"`
Target Target `json:"target,omitempty"`
Previous Image `json:"previous"`
Candidate Image `json:"candidate,omitempty"`
Error string `json:"error,omitempty"`
}
func readState(path string) (State, error) {
@@ -104,15 +106,30 @@ func writeState(path string, state State) error {
func writeFileDurably(path, prefix string, contents []byte) error {
directory := filepath.Dir(path)
if err := os.MkdirAll(directory, 0o700); err != nil { return err }
if err := os.MkdirAll(directory, 0o700); err != nil {
return err
}
temporary, err := os.CreateTemp(directory, prefix+"*.tmp")
if err != nil { return err }
if err != nil {
return err
}
temporaryName := temporary.Name()
defer os.Remove(temporaryName)
if err := temporary.Chmod(0o600); err != nil { temporary.Close(); return err }
if _, err := temporary.Write(contents); err != nil { temporary.Close(); return err }
if err := temporary.Sync(); err != nil { temporary.Close(); return err }
if err := temporary.Close(); err != nil { return err }
if err := temporary.Chmod(0o600); err != nil {
temporary.Close()
return err
}
if _, err := temporary.Write(contents); err != nil {
temporary.Close()
return err
}
if err := temporary.Sync(); err != nil {
temporary.Close()
return err
}
if err := temporary.Close(); err != nil {
return err
}
return durableReplace(temporaryName, path, directory)
}
@@ -138,7 +155,10 @@ type lockOwner struct {
Transaction string `json:"transaction"`
}
type updateLock struct{ path string }
type updateLock struct {
file *flock.Flock
metadata string
}
var ErrLockHeld = errors.New("another Pi update or rollback is already in progress")
@@ -147,47 +167,33 @@ func acquireLock(statePath string) (*updateLock, error) {
return nil, errors.New("could not create Pi update recovery directory")
}
path := statePath + ".lock"
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
file := flock.New(path, flock.SetPermissions(0o600))
locked, err := file.TryLock()
if err != nil {
if errors.Is(err, os.ErrExist) {
if reclaimDeadLocalLock(path, statePath) {
return acquireLock(statePath)
}
return nil, ErrLockHeld
}
return nil, errors.New("could not acquire Pi update lock")
}
if !locked {
return nil, ErrLockHeld
}
host, err := os.Hostname()
if err != nil { _ = file.Close(); _ = os.Remove(path); return nil, errors.New("could not identify Pi update lock owner") }
if err != nil {
_ = file.Unlock()
return nil, errors.New("could not identify Pi update lock owner")
}
owner := lockOwner{PID: os.Getpid(), Host: host, StartedAt: time.Now().UTC(), Transaction: fmt.Sprintf("%d-%d", os.Getpid(), time.Now().UnixNano())}
contents, err := json.Marshal(owner)
if err != nil { _ = file.Close(); _ = os.Remove(path); return nil, errors.New("could not record Pi update lock owner") }
if _, err := file.Write(append(contents, '\n')); err != nil || file.Sync() != nil || file.Close() != nil {
_ = file.Close(); _ = os.Remove(path)
if err != nil {
_ = file.Unlock()
return nil, errors.New("could not record Pi update lock owner")
}
return &updateLock{path: path}, nil
metadata := path + ".owner.json"
if err := writeFileDurably(metadata, ".lock-owner-", append(contents, '\n')); err != nil {
_ = file.Unlock()
return nil, errors.New("could not record Pi update lock owner")
}
return &updateLock{file: file, metadata: metadata}, nil
}
func (l *updateLock) Release() { _ = os.Remove(l.path) }
// reclaimDeadLocalLock is deliberately conservative: a malformed, remote, or merely old lock
// is recovery-required. Only a process we can prove is gone on this machine is reclaimed.
func reclaimDeadLocalLock(path, statePath string) bool {
info, err := os.Stat(path)
if err != nil || time.Since(info.ModTime()) < 5*time.Minute || !hasPendingRecoveryState(statePath) { return false }
contents, err := os.ReadFile(path)
if err != nil { return os.Remove(path) == nil }
var owner lockOwner
if json.Unmarshal(contents, &owner) != nil || owner.PID <= 0 || owner.Host == "" { return false }
host, err := os.Hostname()
if err != nil || owner.Host != host { return false }
if processAlive(owner.PID) { return false }
return os.Remove(path) == nil
}
func hasPendingRecoveryState(path string) bool {
contents, err := os.ReadFile(path); if err != nil { return false }
var state State
if json.Unmarshal(contents, &state) != nil { return false }
return state.Phase != PhaseVerified && state.Phase != PhaseRolledBack && state.Phase != PhaseNoop
func (l *updateLock) Release() {
_ = durableRemove(l.metadata)
_ = l.file.Unlock()
}