396 lines
13 KiB
Go
396 lines
13 KiB
Go
package pi
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
var (
|
|
ErrConfirmationRequired = errors.New("update requires --yes after reviewing the planned Pi version")
|
|
ErrActiveSessions = errors.New("active sessions must be drained before updating Pi; use --drain only after they are complete")
|
|
ErrInterruptedUpdate = errors.New("a previous Pi update is incomplete; run pi rollback --yes before starting another update")
|
|
versionPattern = regexp.MustCompile(`^[0-9]+(?:\.[0-9]+){1,3}(?:[-+][0-9A-Za-z.-]+)?$`)
|
|
digestPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/:@-]*@sha256:[a-f0-9]{64}$`)
|
|
)
|
|
|
|
// Source chooses whether the candidate is built from this checkout or pulled from an immutable image.
|
|
type Source string
|
|
|
|
const (
|
|
BuildSource Source = "build"
|
|
PullSource Source = "pull"
|
|
)
|
|
|
|
// Request contains only non-secret operator inputs.
|
|
type Request struct {
|
|
StatePath string
|
|
Version string
|
|
Source Source
|
|
Image string
|
|
Confirm bool
|
|
Drain bool
|
|
}
|
|
|
|
// Result summarizes the completed, failed, or recovered transaction without command output.
|
|
type Result struct {
|
|
Phase Phase
|
|
StatePath string
|
|
}
|
|
|
|
// Update performs a recoverable core-only Pi update using the default Compose command layout.
|
|
func Update(ctx context.Context, runner Runner, request Request) (Result, error) {
|
|
lock, err := acquireLock(request.StatePath)
|
|
if err != nil {
|
|
return Result{StatePath: request.StatePath}, err
|
|
}
|
|
defer lock.Release()
|
|
if request.StatePath == "" {
|
|
return Result{}, errors.New("update state path is required")
|
|
}
|
|
if !request.Confirm {
|
|
return Result{StatePath: request.StatePath}, ErrConfirmationRequired
|
|
}
|
|
if !versionPattern.MatchString(request.Version) {
|
|
return Result{StatePath: request.StatePath}, errors.New("Pi version must be an explicit pinned version")
|
|
}
|
|
if request.Source == "" {
|
|
request.Source = BuildSource
|
|
}
|
|
if request.Source != BuildSource && request.Source != PullSource {
|
|
return Result{StatePath: request.StatePath}, errors.New("Pi update source must be build or pull")
|
|
}
|
|
if request.Source == PullSource && !digestPattern.MatchString(request.Image) {
|
|
return Result{StatePath: request.StatePath}, errors.New("pulled Pi image must use an immutable sha256 digest")
|
|
}
|
|
if old, err := readState(request.StatePath); err == nil && old.Phase != PhaseVerified && old.Phase != PhaseRolledBack && old.Phase != PhaseNoop {
|
|
return Result{StatePath: request.StatePath}, ErrInterruptedUpdate
|
|
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
return Result{StatePath: request.StatePath}, err
|
|
}
|
|
|
|
running, err := activeSessions(ctx, runner)
|
|
if err != nil {
|
|
return Result{StatePath: request.StatePath}, err
|
|
}
|
|
frontendStopped := false
|
|
if running {
|
|
if !request.Drain {
|
|
return Result{StatePath: request.StatePath}, ErrActiveSessions
|
|
}
|
|
stopped, stopErr := runCompose(ctx, runner, "stop", "frontend")
|
|
if stopErr != nil {
|
|
return Result{StatePath: request.StatePath}, commandError("frontend admission gate", stopped, stopErr)
|
|
}
|
|
frontendStopped = true
|
|
defer func() {
|
|
if frontendStopped {
|
|
_, _ = runCompose(context.Background(), runner, "up", "--detach", "frontend")
|
|
}
|
|
}()
|
|
for attempts := 0; attempts < 30; attempts++ {
|
|
running, err = activeSessions(ctx, runner)
|
|
if err != nil {
|
|
return Result{StatePath: request.StatePath}, err
|
|
}
|
|
if !running {
|
|
break
|
|
}
|
|
time.Sleep(time.Second)
|
|
}
|
|
if running {
|
|
return Result{StatePath: request.StatePath}, ErrActiveSessions
|
|
}
|
|
}
|
|
if err := Doctor(ctx, runner); err != nil {
|
|
return Result{StatePath: request.StatePath}, err
|
|
}
|
|
|
|
currentVersion, err := Status(ctx, runner)
|
|
if err != nil {
|
|
return Result{StatePath: request.StatePath}, err
|
|
}
|
|
if currentVersion == request.Version {
|
|
return Result{Phase: PhaseNoop, StatePath: request.StatePath}, nil
|
|
}
|
|
configured, err := renderedCore(ctx, runner)
|
|
if err != nil {
|
|
return Result{StatePath: request.StatePath}, err
|
|
}
|
|
previous, err := runningImage(ctx, runner, configured.Reference)
|
|
if err != nil {
|
|
return Result{StatePath: request.StatePath}, err
|
|
}
|
|
previous.ConfigurationSHA = configured.ConfigurationSHA
|
|
state := State{Phase: PhasePreflight, Target: Target{Version: request.Version, Source: sourceValue(request)}, Previous: previous}
|
|
if err := writeState(request.StatePath, state); err != nil {
|
|
return Result{StatePath: request.StatePath}, err
|
|
}
|
|
|
|
state.Phase = PhaseBuilding
|
|
if err := writeState(request.StatePath, state); err != nil {
|
|
return Result{StatePath: request.StatePath}, err
|
|
}
|
|
if err := prepareCandidate(ctx, runner, request, previous.Reference); err != nil {
|
|
state.Phase, state.Error = PhaseFailed, "candidate image preparation failed"
|
|
_ = writeState(request.StatePath, state)
|
|
return Result{Phase: PhaseFailed, StatePath: request.StatePath}, err
|
|
}
|
|
if frontendStopped {
|
|
running, err = activeSessions(ctx, runner)
|
|
if err != nil {
|
|
return Result{Phase: PhaseFailed, StatePath: request.StatePath}, err
|
|
}
|
|
if running {
|
|
state.Phase, state.Error = PhaseFailed, "new session admitted while draining"
|
|
_ = writeState(request.StatePath, state)
|
|
return Result{Phase: PhaseFailed, StatePath: request.StatePath}, ErrActiveSessions
|
|
}
|
|
}
|
|
if err := recreateCore(ctx, runner); err != nil {
|
|
state.Phase, state.Error = PhaseFailed, "core recreation failed"
|
|
_ = writeState(request.StatePath, state)
|
|
return Result{Phase: PhaseFailed, StatePath: request.StatePath}, err
|
|
}
|
|
state.Phase = PhaseRecreated
|
|
state.Candidate, _ = runningImage(ctx, runner, previous.Reference)
|
|
if err := writeState(request.StatePath, state); err != nil {
|
|
return Result{Phase: PhaseRecreated, StatePath: request.StatePath}, err
|
|
}
|
|
if err := verifyCandidate(ctx, runner, request.Version, previous); err != nil {
|
|
return rollbackAfterFailure(ctx, runner, request.StatePath, state, err)
|
|
}
|
|
state.Phase, state.Error = PhaseVerified, ""
|
|
if err := writeState(request.StatePath, state); err != nil {
|
|
return Result{Phase: PhaseVerified, StatePath: request.StatePath}, err
|
|
}
|
|
return Result{Phase: PhaseVerified, StatePath: request.StatePath}, nil
|
|
}
|
|
|
|
// Rollback restores the image recorded in durable update state. It is safe for interrupted runs.
|
|
func Rollback(ctx context.Context, runner Runner, statePath string, confirm bool) (Result, error) {
|
|
lock, err := acquireLock(statePath)
|
|
if err != nil {
|
|
return Result{StatePath: statePath}, err
|
|
}
|
|
defer lock.Release()
|
|
if !confirm {
|
|
return Result{StatePath: statePath}, ErrConfirmationRequired
|
|
}
|
|
state, err := readState(statePath)
|
|
if err != nil {
|
|
return Result{StatePath: statePath}, err
|
|
}
|
|
if err := restore(ctx, runner, state.Previous); err != nil {
|
|
state.Phase, state.Error = PhaseFailed, "rollback failed"
|
|
_ = writeState(statePath, state)
|
|
return Result{Phase: PhaseFailed, StatePath: statePath}, err
|
|
}
|
|
state.Phase, state.Error = PhaseRolledBack, ""
|
|
if err := writeState(statePath, state); err != nil {
|
|
return Result{Phase: PhaseRolledBack, StatePath: statePath}, err
|
|
}
|
|
return Result{Phase: PhaseRolledBack, StatePath: statePath}, nil
|
|
}
|
|
|
|
func rollbackAfterFailure(ctx context.Context, runner Runner, statePath string, state State, cause error) (Result, error) {
|
|
if restoreErr := restore(ctx, runner, state.Previous); restoreErr != nil {
|
|
state.Phase, state.Error = PhaseFailed, "candidate verification and automatic rollback failed"
|
|
_ = writeState(statePath, state)
|
|
return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("candidate verification failed; automatic rollback also failed")
|
|
}
|
|
state.Phase, state.Error = PhaseRolledBack, ""
|
|
_ = writeState(statePath, state)
|
|
return Result{Phase: PhaseRolledBack, StatePath: statePath}, fmt.Errorf("candidate verification failed; previous core image was restored")
|
|
}
|
|
|
|
func sourceValue(request Request) string {
|
|
if request.Source == PullSource {
|
|
return request.Image
|
|
}
|
|
return string(BuildSource)
|
|
}
|
|
|
|
func activeSessions(ctx context.Context, runner Runner) (bool, error) {
|
|
result, err := runCompose(ctx, runner, "exec", "-T", "core", "tht", "session", "list", "--json")
|
|
if err != nil {
|
|
return false, commandError("active-session check", result, err)
|
|
}
|
|
var sessions []struct {
|
|
Status string `json:"status"`
|
|
Archived bool `json:"archived"`
|
|
}
|
|
if err := json.Unmarshal([]byte(result.Stdout), &sessions); err != nil {
|
|
return false, errors.New("active-session check returned invalid session data")
|
|
}
|
|
for _, session := range sessions {
|
|
if !session.Archived && session.Status == "open" {
|
|
return true, nil
|
|
}
|
|
}
|
|
return false, nil
|
|
}
|
|
|
|
func runningImage(ctx context.Context, runner Runner, reference string) (Image, error) {
|
|
container, err := runCompose(ctx, runner, "ps", "-q", "core")
|
|
if err != nil || strings.TrimSpace(container.Stdout) == "" {
|
|
return Image{}, commandError("running core image check", container, err)
|
|
}
|
|
id := strings.TrimSpace(container.Stdout)
|
|
image, err := runner.Run(ctx, []string{"inspect", "--format", "{{.Image}}", id}, nil)
|
|
if err != nil || strings.TrimSpace(image.Stdout) == "" {
|
|
return Image{}, commandError("running core image check", image, err)
|
|
}
|
|
mounts, err := runner.Run(ctx, []string{"inspect", "--format", "{{json .Mounts}}", id}, nil)
|
|
if err != nil {
|
|
return Image{}, commandError("core volume check", mounts, err)
|
|
}
|
|
var contract []Mount
|
|
if err := json.Unmarshal([]byte(mounts.Stdout), &contract); err != nil {
|
|
return Image{}, errors.New("core returned invalid persistence mount data")
|
|
}
|
|
if len(contract) == 0 {
|
|
return Image{}, errors.New("core has no persistence mounts to preserve")
|
|
}
|
|
volumes := make([]string, 0, len(contract))
|
|
for _, mount := range contract {
|
|
if mount.Type == "volume" && mount.Name != "" {
|
|
volumes = append(volumes, mount.Name)
|
|
}
|
|
}
|
|
return Image{ID: strings.TrimSpace(image.Stdout), Reference: reference, Volumes: volumes, Mounts: contract}, nil
|
|
}
|
|
|
|
func prepareCandidate(ctx context.Context, runner Runner, request Request, reference string) error {
|
|
if request.Source == BuildSource {
|
|
result, err := runCompose(ctx, runner, "build", "--pull", "--build-arg", "PI_VERSION="+request.Version, "core")
|
|
if err != nil {
|
|
return commandError("Pi image build", result, err)
|
|
}
|
|
return nil
|
|
}
|
|
pull, err := runner.Run(ctx, []string{"pull", request.Image}, nil)
|
|
if err != nil {
|
|
return commandError("Pi image pull", pull, err)
|
|
}
|
|
tag, err := runner.Run(ctx, []string{"image", "tag", request.Image, reference}, nil)
|
|
if err != nil {
|
|
return commandError("Pi image tag", tag, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func recreateCore(ctx context.Context, runner Runner) error {
|
|
result, err := runCompose(ctx, runner, "up", "--detach", "--wait", "--wait-timeout", "45", "--no-deps", "--force-recreate", "core")
|
|
if err != nil {
|
|
return commandError("core recreation", result, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func verifyCandidate(ctx context.Context, runner Runner, wanted string, previous Image) error {
|
|
health, err := runCompose(ctx, runner, "exec", "-T", "core", "curl", "-fsS", "http://127.0.0.1:8787/health")
|
|
if err != nil {
|
|
return commandError("core health check", health, err)
|
|
}
|
|
version, err := Status(ctx, runner)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if version != wanted {
|
|
return errors.New("candidate Pi version does not match requested pinned version")
|
|
}
|
|
if err := Test(ctx, runner); err != nil {
|
|
return err
|
|
}
|
|
configured, err := renderedCore(ctx, runner)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if configured.ConfigurationSHA != previous.ConfigurationSHA {
|
|
return errors.New("external endpoint configuration changed during Pi update")
|
|
}
|
|
after, err := runningImage(ctx, runner, configured.Reference)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !sameMounts(previous.Mounts, after.Mounts) {
|
|
return errors.New("core persistence mount contract changed during Pi update")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func restore(ctx context.Context, runner Runner, previous Image) error {
|
|
tag, err := runner.Run(ctx, []string{"image", "tag", previous.ID, previous.Reference}, nil)
|
|
if err != nil {
|
|
return commandError("rollback image restore", tag, err)
|
|
}
|
|
if err := recreateCore(ctx, runner); err != nil {
|
|
return err
|
|
}
|
|
configured, err := renderedCore(ctx, runner)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
after, err := runningImage(ctx, runner, configured.Reference)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if after.ID != previous.ID {
|
|
return errors.New("rollback core image does not match recorded previous image")
|
|
}
|
|
if configured.ConfigurationSHA != previous.ConfigurationSHA {
|
|
return errors.New("external endpoint configuration drift prevents rollback proof")
|
|
}
|
|
if !sameMounts(previous.Mounts, after.Mounts) {
|
|
return errors.New("core persistence mount contract changed during rollback")
|
|
}
|
|
if err := Doctor(ctx, runner); err != nil {
|
|
return err
|
|
}
|
|
if err := Test(ctx, runner); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func nonEmptyLines(text string) []string {
|
|
var values []string
|
|
for _, value := range strings.Split(text, "\n") {
|
|
if value = strings.TrimSpace(value); value != "" {
|
|
values = append(values, value)
|
|
}
|
|
}
|
|
sort.Strings(values)
|
|
return values
|
|
}
|
|
func sameStrings(left, right []string) bool {
|
|
left, right = append([]string(nil), left...), append([]string(nil), right...)
|
|
sort.Strings(left)
|
|
sort.Strings(right)
|
|
return strings.Join(left, "\x00") == strings.Join(right, "\x00")
|
|
}
|
|
|
|
func sameMounts(left, right []Mount) bool {
|
|
if len(left) != len(right) {
|
|
return false
|
|
}
|
|
key := func(m Mount) string {
|
|
return m.Type + "\x00" + m.Name + "\x00" + m.Source + "\x00" + m.Destination + "\x00" + fmt.Sprint(m.RW)
|
|
}
|
|
a, b := make([]string, len(left)), make([]string, len(right))
|
|
for i := range left {
|
|
a[i] = key(left[i])
|
|
}
|
|
for i := range right {
|
|
b[i] = key(right[i])
|
|
}
|
|
return sameStrings(a, b)
|
|
}
|