mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix: harden pi maintenance lifecycle

Browse Source
This commit is contained in:
marcopan
2026-08-04 19:09:04 +02:00
parent 8fde1f81c7
commit 0b9ad7f53f
21 changed files with 432 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/thothctl/internal/pi/update.go
+84 -52
View File
@@ -10,14 +10,16 @@ import (
"sort"
"strings"
"time"
"github.com/distribution/reference"
)
var (
ErrConfirmationRequired = errors.New("update requires --yes after reviewing the planned Pi version")
ErrActiveSessions = errors.New("active sessions must be drained before updating Pi; use --drain only after they are complete")
ErrInterruptedUpdate = errors.New("a previous Pi update is incomplete; run pi rollback --yes before starting another update")
ErrInvalidRequest = errors.New("invalid Pi lifecycle request")
versionPattern = regexp.MustCompile(`^[0-9]+(?:\.[0-9]+){1,3}(?:[-+][0-9A-Za-z.-]+)?$`)
digestPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/:@-]*@sha256:[a-f0-9]{64}$`)
)
// Source chooses whether the candidate is built from this checkout or pulled from an immutable image.
@@ -45,7 +47,7 @@ type Result struct {
}
// Update performs a recoverable core-only Pi update using the default Compose command layout.
func Update(ctx context.Context, runner Runner, request Request) (Result, error) {
func Update(ctx context.Context, runner Runner, request Request) (result Result, retErr error) {
lock, err := acquireLock(request.StatePath)
if err != nil {
return Result{StatePath: request.StatePath}, err
@@ -58,42 +60,39 @@ func Update(ctx context.Context, runner Runner, request Request) (Result, error)
return Result{StatePath: request.StatePath}, ErrConfirmationRequired
}
if !versionPattern.MatchString(request.Version) {
return Result{StatePath: request.StatePath}, errors.New("Pi version must be an explicit pinned version")
}
if request.Source == "" {
request.Source = BuildSource
return Result{StatePath: request.StatePath}, fmt.Errorf("%w: Pi version must be an explicit pinned version", ErrInvalidRequest)
}
if request.Source == "" { return Result{StatePath: request.StatePath}, fmt.Errorf("%w: Pi update requires an explicit source: build or pull", ErrInvalidRequest) }
if request.Source != BuildSource && request.Source != PullSource {
return Result{StatePath: request.StatePath}, errors.New("Pi update source must be build or pull")
return Result{StatePath: request.StatePath}, fmt.Errorf("%w: Pi update source must be build or pull", ErrInvalidRequest)
}
if request.Source == PullSource && !digestPattern.MatchString(request.Image) {
return Result{StatePath: request.StatePath}, errors.New("pulled Pi image must use an immutable sha256 digest")
if request.Source == PullSource {
canonical, err := canonicalDigestReference(request.Image)
if err != nil { return Result{StatePath: request.StatePath}, fmt.Errorf("%w: %v", ErrInvalidRequest, err) }
request.Image = canonical
}
if old, err := readState(request.StatePath); err == nil && old.Phase != PhaseVerified && old.Phase != PhaseRolledBack && old.Phase != PhaseNoop {
return Result{StatePath: request.StatePath}, ErrInterruptedUpdate
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
return Result{StatePath: request.StatePath}, err
}
if err := setMaintenance(ctx, runner, true); err != nil { return Result{StatePath: request.StatePath}, err }
defer func() {
if clearErr := setMaintenance(context.Background(), runner, false); clearErr != nil {
result = Result{Phase: PhaseFailed, StatePath: request.StatePath}
if retErr == nil { retErr = errors.New("maintenance admission gate could not be cleared: recovery required")
} else { retErr = fmt.Errorf("%w; maintenance admission gate could not be cleared: recovery required", retErr) }
}
}()
running, err := activeSessions(ctx, runner)
if err != nil {
return Result{StatePath: request.StatePath}, err
}
frontendStopped := false
if running {
if !request.Drain {
return Result{StatePath: request.StatePath}, ErrActiveSessions
}
stopped, stopErr := runCompose(ctx, runner, "stop", "frontend")
if stopErr != nil {
return Result{StatePath: request.StatePath}, commandError("frontend admission gate", stopped, stopErr)
}
frontendStopped = true
defer func() {
if frontendStopped {
_, _ = runCompose(context.Background(), runner, "up", "--detach", "frontend")
}
}()
for attempts := 0; attempts < 30; attempts++ {
running, err = activeSessions(ctx, runner)
if err != nil {
@@ -134,41 +133,34 @@ func Update(ctx context.Context, runner Runner, request Request) (Result, error)
}
state.Phase = PhaseBuilding
if err := writeState(request.StatePath, state); err != nil {
return Result{StatePath: request.StatePath}, err
}
if err := writeState(request.StatePath, state); err != nil { return Result{Phase: PhaseFailed, StatePath: request.StatePath}, err }
if err := prepareCandidate(ctx, runner, request, previous.Reference); err != nil {
state.Phase, state.Error = PhaseFailed, "candidate image preparation failed"
_ = writeState(request.StatePath, state)
return Result{Phase: PhaseFailed, StatePath: request.StatePath}, err
return compensate(ctx, runner, request.StatePath, state, err)
}
if frontendStopped {
if request.Drain {
running, err = activeSessions(ctx, runner)
if err != nil {
return Result{Phase: PhaseFailed, StatePath: request.StatePath}, err
return compensate(ctx, runner, request.StatePath, state, err)
}
if running {
state.Phase, state.Error = PhaseFailed, "new session admitted while draining"
_ = writeState(request.StatePath, state)
return Result{Phase: PhaseFailed, StatePath: request.StatePath}, ErrActiveSessions
return compensate(ctx, runner, request.StatePath, state, ErrActiveSessions)
}
}
if err := recreateCore(ctx, runner); err != nil {
state.Phase, state.Error = PhaseFailed, "core recreation failed"
_ = writeState(request.StatePath, state)
return Result{Phase: PhaseFailed, StatePath: request.StatePath}, err
return compensate(ctx, runner, request.StatePath, state, err)
}
state.Phase = PhaseRecreated
state.Candidate, _ = runningImage(ctx, runner, previous.Reference)
state.Candidate, err = runningImage(ctx, runner, previous.Reference)
if err != nil { return compensate(ctx, runner, request.StatePath, state, err) }
if err := writeState(request.StatePath, state); err != nil {
return Result{Phase: PhaseRecreated, StatePath: request.StatePath}, err
return compensate(ctx, runner, request.StatePath, state, err)
}
if err := verifyCandidate(ctx, runner, request.Version, previous); err != nil {
return rollbackAfterFailure(ctx, runner, request.StatePath, state, err)
return compensate(ctx, runner, request.StatePath, state, err)
}
state.Phase, state.Error = PhaseVerified, ""
if err := writeState(request.StatePath, state); err != nil {
return Result{Phase: PhaseVerified, StatePath: request.StatePath}, err
return compensate(ctx, runner, request.StatePath, state, err)
}
return Result{Phase: PhaseVerified, StatePath: request.StatePath}, nil
}
@@ -189,25 +181,35 @@ func Rollback(ctx context.Context, runner Runner, statePath string, confirm bool
}
if err := restore(ctx, runner, state.Previous); err != nil {
state.Phase, state.Error = PhaseFailed, "rollback failed"
_ = writeState(statePath, state)
if writeErr := writeState(statePath, state); writeErr != nil { return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("rollback failed and recovery state could not be persisted") }
return Result{Phase: PhaseFailed, StatePath: statePath}, err
}
state.Phase, state.Error = PhaseRolledBack, ""
if err := writeState(statePath, state); err != nil {
return Result{Phase: PhaseRolledBack, StatePath: statePath}, err
return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("rollback restored the core but recovery state could not be persisted")
}
return Result{Phase: PhaseRolledBack, StatePath: statePath}, nil
}
func rollbackAfterFailure(ctx context.Context, runner Runner, statePath string, state State, cause error) (Result, error) {
func compensate(ctx context.Context, runner Runner, statePath string, state State, cause error) (Result, error) {
if restoreErr := restore(ctx, runner, state.Previous); restoreErr != nil {
state.Phase, state.Error = PhaseFailed, "candidate verification and automatic rollback failed"
_ = writeState(statePath, state)
return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("candidate verification failed; automatic rollback also failed")
if writeErr := writeState(statePath, state); writeErr != nil {
return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("update failed and rollback proof failed; recovery state could not be persisted")
}
return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("update failed; automatic rollback also failed: recovery required")
}
state.Phase, state.Error = PhaseRolledBack, ""
_ = writeState(statePath, state)
return Result{Phase: PhaseRolledBack, StatePath: statePath}, fmt.Errorf("candidate verification failed; previous core image was restored")
if writeErr := writeState(statePath, state); writeErr != nil {
return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("previous core image was restored but recovery state write failed: recovery required")
}
return Result{Phase: PhaseRolledBack, StatePath: statePath}, fmt.Errorf("update failed; previous core image was restored")
}
func recordFailure(path string, state State, label string, cause error) error {
state.Phase, state.Error = PhaseFailed, label
if err := writeState(path, state); err != nil { return fmt.Errorf("%w; recovery state write failed", cause) }
return cause
}
func sourceValue(request Request) string {
@@ -217,6 +219,29 @@ func sourceValue(request Request) string {
return string(BuildSource)
}
func canonicalDigestReference(value string) (string, error) {
if strings.Contains(value, "://") || strings.ContainsAny(value, "?#") || strings.Contains(value, "@") && strings.Contains(strings.Split(value, "@")[0], ":") && strings.Contains(strings.Split(value, "@")[0], "//") {
return "", errors.New("pulled Pi image must be a credential-free canonical sha256 digest reference")
}
parsed, err := reference.ParseAnyReference(value)
if err != nil { return "", errors.New("pulled Pi image must be a valid canonical sha256 digest reference") }
canonical, ok := parsed.(reference.Canonical)
if !ok || canonical.Digest().Algorithm().String() != "sha256" || len(canonical.Digest().Encoded()) != 64 {
return "", errors.New("pulled Pi image must use an immutable sha256 digest")
}
return reference.FamiliarString(canonical), nil
}
// The command text is fixed; no operator input or host path is interpolated into the core shell.
// The marker lives alongside SETTINGS_FILE's named/bind-mounted directory and is read by backend.
func setMaintenance(ctx context.Context, runner Runner, enabled bool) error {
command := "mkdir -p /data/settings && : > /data/settings/maintenance.json && chmod 600 /data/settings/maintenance.json"
if !enabled { command = "rm -f /data/settings/maintenance.json" }
result, err := runCompose(ctx, runner, "exec", "-T", "core", "sh", "-ceu", command)
if err != nil { return commandError("maintenance admission gate", result, err) }
return nil
}
func activeSessions(ctx context.Context, runner Runner) (bool, error) {
result, err := runCompose(ctx, runner, "exec", "-T", "core", "tht", "session", "list", "--json")
if err != nil {
@@ -251,20 +276,27 @@ func runningImage(ctx context.Context, runner Runner, reference string) (Image,
if err != nil {
return Image{}, commandError("core volume check", mounts, err)
}
var contract []Mount
if err := json.Unmarshal([]byte(mounts.Stdout), &contract); err != nil {
var raw []struct {
Type string `json:"Type"`; Name string `json:"Name"`; Source string `json:"Source"`
Destination string `json:"Destination"`; RW bool `json:"RW"`; Mode string `json:"Mode"`
Propagation string `json:"Propagation"`; Driver string `json:"Driver"`
}
if err := json.Unmarshal([]byte(mounts.Stdout), &raw); err != nil {
return Image{}, errors.New("core returned invalid persistence mount data")
}
if len(contract) == 0 {
if len(raw) == 0 {
return Image{}, errors.New("core has no persistence mounts to preserve")
}
volumes := make([]string, 0, len(contract))
for _, mount := range contract {
contract := make([]Mount, 0, len(raw))
volumes := make([]string, 0, len(raw))
for _, mount := range raw {
if mount.Type == "" || mount.Source == "" || mount.Destination == "" { return Image{}, errors.New("core returned incomplete persistence mount data") }
contract = append(contract, Mount{Type: mount.Type, Name: mount.Name, SourceSHA256: mountSourceHash(mount.Source), Destination: mount.Destination, RW: mount.RW, Options: strings.Join([]string{mount.Mode, mount.Propagation, mount.Driver}, "\x00")})
if mount.Type == "volume" && mount.Name != "" {
volumes = append(volumes, mount.Name)
}
}
return Image{ID: strings.TrimSpace(image.Stdout), Reference: reference, Volumes: volumes, Mounts: contract}, nil
return Image{ID: strings.TrimSpace(image.Stdout), Reference: reference, Volumes: volumes, Mounts: contract, MountFingerprint: mountFingerprint(contract)}, nil
}
func prepareCandidate(ctx context.Context, runner Runner, request Request, reference string) error {
@@ -382,7 +414,7 @@ func sameMounts(left, right []Mount) bool {
return false
}
key := func(m Mount) string {
return m.Type + "\x00" + m.Name + "\x00" + m.Source + "\x00" + m.Destination + "\x00" + fmt.Sprint(m.RW)
return m.Type + "\x00" + m.Name + "\x00" + m.SourceSHA256 + "\x00" + m.Destination + "\x00" + fmt.Sprint(m.RW) + "\x00" + m.Options
}
a, b := make([]string, len(left)), make([]string, len(right))
for i := range left {