feat: manage embedded pi with thothctl
This commit is contained in:
@@ -0,0 +1,318 @@
|
||||
package pi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrConfirmationRequired = errors.New("update requires --yes after reviewing the planned Pi version")
|
||||
ErrActiveSessions = errors.New("active sessions must be drained before updating Pi; use --drain only after they are complete")
|
||||
ErrInterruptedUpdate = errors.New("a previous Pi update is incomplete; run pi rollback --yes before starting another update")
|
||||
versionPattern = regexp.MustCompile(`^[0-9]+(?:\.[0-9]+){1,3}(?:[-+][0-9A-Za-z.-]+)?$`)
|
||||
digestPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/:@-]*@sha256:[a-f0-9]{64}$`)
|
||||
)
|
||||
|
||||
// Source chooses whether the candidate is built from this checkout or pulled from an immutable image.
|
||||
type Source string
|
||||
|
||||
const (
|
||||
BuildSource Source = "build"
|
||||
PullSource Source = "pull"
|
||||
)
|
||||
|
||||
// Request contains only non-secret operator inputs.
|
||||
type Request struct {
|
||||
StatePath string
|
||||
Version string
|
||||
Source Source
|
||||
Image string
|
||||
Confirm bool
|
||||
Drain bool
|
||||
}
|
||||
|
||||
// Result summarizes the completed, failed, or recovered transaction without command output.
|
||||
type Result struct {
|
||||
Phase Phase
|
||||
StatePath string
|
||||
}
|
||||
|
||||
// Update performs a recoverable core-only Pi update using the default Compose command layout.
|
||||
func Update(ctx context.Context, runner Runner, request Request) (Result, error) {
|
||||
if request.StatePath == "" {
|
||||
return Result{}, errors.New("update state path is required")
|
||||
}
|
||||
if !request.Confirm {
|
||||
return Result{StatePath: request.StatePath}, ErrConfirmationRequired
|
||||
}
|
||||
if !versionPattern.MatchString(request.Version) {
|
||||
return Result{StatePath: request.StatePath}, errors.New("Pi version must be an explicit pinned version")
|
||||
}
|
||||
if request.Source == "" {
|
||||
request.Source = BuildSource
|
||||
}
|
||||
if request.Source != BuildSource && request.Source != PullSource {
|
||||
return Result{StatePath: request.StatePath}, errors.New("Pi update source must be build or pull")
|
||||
}
|
||||
if request.Source == PullSource && !digestPattern.MatchString(request.Image) {
|
||||
return Result{StatePath: request.StatePath}, errors.New("pulled Pi image must use an immutable sha256 digest")
|
||||
}
|
||||
if old, err := readState(request.StatePath); err == nil && old.Phase != PhaseVerified && old.Phase != PhaseRolledBack && old.Phase != PhaseNoop {
|
||||
return Result{StatePath: request.StatePath}, ErrInterruptedUpdate
|
||||
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
}
|
||||
|
||||
running, err := activeSessions(ctx, runner)
|
||||
if err != nil {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
}
|
||||
if running {
|
||||
if !request.Drain {
|
||||
return Result{StatePath: request.StatePath}, ErrActiveSessions
|
||||
}
|
||||
running, err = activeSessions(ctx, runner)
|
||||
if err != nil {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
}
|
||||
if running {
|
||||
return Result{StatePath: request.StatePath}, ErrActiveSessions
|
||||
}
|
||||
}
|
||||
if err := Doctor(ctx, runner); err != nil {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
}
|
||||
|
||||
currentVersion, err := Status(ctx, runner)
|
||||
if err != nil {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
}
|
||||
if currentVersion == request.Version {
|
||||
return Result{Phase: PhaseNoop, StatePath: request.StatePath}, nil
|
||||
}
|
||||
configured, err := renderedCore(ctx, runner)
|
||||
if err != nil {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
}
|
||||
previous, err := runningImage(ctx, runner, configured.Reference)
|
||||
if err != nil {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
}
|
||||
previous.ConfigurationSHA = configured.ConfigurationSHA
|
||||
state := State{Phase: PhasePreflight, Target: Target{Version: request.Version, Source: sourceValue(request)}, Previous: previous}
|
||||
if err := writeState(request.StatePath, state); err != nil {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
}
|
||||
|
||||
state.Phase = PhaseBuilding
|
||||
if err := writeState(request.StatePath, state); err != nil {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
}
|
||||
if err := prepareCandidate(ctx, runner, request, previous.Reference); err != nil {
|
||||
state.Phase, state.Error = PhaseFailed, "candidate image preparation failed"
|
||||
_ = writeState(request.StatePath, state)
|
||||
return Result{Phase: PhaseFailed, StatePath: request.StatePath}, err
|
||||
}
|
||||
if err := recreateCore(ctx, runner); err != nil {
|
||||
state.Phase, state.Error = PhaseFailed, "core recreation failed"
|
||||
_ = writeState(request.StatePath, state)
|
||||
return Result{Phase: PhaseFailed, StatePath: request.StatePath}, err
|
||||
}
|
||||
state.Phase = PhaseRecreated
|
||||
state.Candidate, _ = runningImage(ctx, runner, previous.Reference)
|
||||
if err := writeState(request.StatePath, state); err != nil {
|
||||
return Result{Phase: PhaseRecreated, StatePath: request.StatePath}, err
|
||||
}
|
||||
if err := verifyCandidate(ctx, runner, request.Version, previous.Volumes, previous.ConfigurationSHA); err != nil {
|
||||
return rollbackAfterFailure(ctx, runner, request.StatePath, state, err)
|
||||
}
|
||||
state.Phase, state.Error = PhaseVerified, ""
|
||||
if err := writeState(request.StatePath, state); err != nil {
|
||||
return Result{Phase: PhaseVerified, StatePath: request.StatePath}, err
|
||||
}
|
||||
return Result{Phase: PhaseVerified, StatePath: request.StatePath}, nil
|
||||
}
|
||||
|
||||
// Rollback restores the image recorded in durable update state. It is safe for interrupted runs.
|
||||
func Rollback(ctx context.Context, runner Runner, statePath string, confirm bool) (Result, error) {
|
||||
if !confirm {
|
||||
return Result{StatePath: statePath}, ErrConfirmationRequired
|
||||
}
|
||||
state, err := readState(statePath)
|
||||
if err != nil {
|
||||
return Result{StatePath: statePath}, err
|
||||
}
|
||||
if err := restore(ctx, runner, state.Previous); err != nil {
|
||||
state.Phase, state.Error = PhaseFailed, "rollback failed"
|
||||
_ = writeState(statePath, state)
|
||||
return Result{Phase: PhaseFailed, StatePath: statePath}, err
|
||||
}
|
||||
state.Phase, state.Error = PhaseRolledBack, ""
|
||||
if err := writeState(statePath, state); err != nil {
|
||||
return Result{Phase: PhaseRolledBack, StatePath: statePath}, err
|
||||
}
|
||||
return Result{Phase: PhaseRolledBack, StatePath: statePath}, nil
|
||||
}
|
||||
|
||||
func rollbackAfterFailure(ctx context.Context, runner Runner, statePath string, state State, cause error) (Result, error) {
|
||||
if restoreErr := restore(ctx, runner, state.Previous); restoreErr != nil {
|
||||
state.Phase, state.Error = PhaseFailed, "candidate verification and automatic rollback failed"
|
||||
_ = writeState(statePath, state)
|
||||
return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("candidate verification failed; automatic rollback also failed")
|
||||
}
|
||||
state.Phase, state.Error = PhaseRolledBack, ""
|
||||
_ = writeState(statePath, state)
|
||||
return Result{Phase: PhaseRolledBack, StatePath: statePath}, fmt.Errorf("candidate verification failed; previous core image was restored")
|
||||
}
|
||||
|
||||
func sourceValue(request Request) string {
|
||||
if request.Source == PullSource {
|
||||
return request.Image
|
||||
}
|
||||
return string(BuildSource)
|
||||
}
|
||||
|
||||
func activeSessions(ctx context.Context, runner Runner) (bool, error) {
|
||||
result, err := runCompose(ctx, runner, "exec", "-T", "core", "tht", "session", "list", "--json")
|
||||
if err != nil {
|
||||
return false, commandError("active-session check", result, err)
|
||||
}
|
||||
var sessions []struct {
|
||||
Status string `json:"status"`
|
||||
Archived bool `json:"archived"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(result.Stdout), &sessions); err != nil {
|
||||
return false, errors.New("active-session check returned invalid session data")
|
||||
}
|
||||
for _, session := range sessions {
|
||||
if !session.Archived && session.Status == "open" {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func runningImage(ctx context.Context, runner Runner, reference string) (Image, error) {
|
||||
container, err := runCompose(ctx, runner, "ps", "-q", "core")
|
||||
if err != nil || strings.TrimSpace(container.Stdout) == "" {
|
||||
return Image{}, commandError("running core image check", container, err)
|
||||
}
|
||||
id := strings.TrimSpace(container.Stdout)
|
||||
image, err := runner.Run(ctx, []string{"inspect", "--format", "{{.Image}}", id}, nil)
|
||||
if err != nil || strings.TrimSpace(image.Stdout) == "" {
|
||||
return Image{}, commandError("running core image check", image, err)
|
||||
}
|
||||
mounts, err := runner.Run(ctx, []string{"inspect", "--format", "{{range .Mounts}}{{if eq .Type \"volume\"}}{{.Name}}{{\"\\n\"}}{{end}}{{end}}", id}, nil)
|
||||
if err != nil {
|
||||
return Image{}, commandError("core volume check", mounts, err)
|
||||
}
|
||||
volumes := nonEmptyLines(mounts.Stdout)
|
||||
if len(volumes) == 0 {
|
||||
return Image{}, errors.New("core has no named volumes to preserve")
|
||||
}
|
||||
return Image{ID: strings.TrimSpace(image.Stdout), Reference: reference, Volumes: volumes}, nil
|
||||
}
|
||||
|
||||
func prepareCandidate(ctx context.Context, runner Runner, request Request, reference string) error {
|
||||
if request.Source == BuildSource {
|
||||
result, err := runCompose(ctx, runner, "build", "--pull", "--build-arg", "PI_VERSION="+request.Version, "core")
|
||||
if err != nil {
|
||||
return commandError("Pi image build", result, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
pull, err := runner.Run(ctx, []string{"pull", request.Image}, nil)
|
||||
if err != nil {
|
||||
return commandError("Pi image pull", pull, err)
|
||||
}
|
||||
tag, err := runner.Run(ctx, []string{"image", "tag", request.Image, reference}, nil)
|
||||
if err != nil {
|
||||
return commandError("Pi image tag", tag, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func recreateCore(ctx context.Context, runner Runner) error {
|
||||
result, err := runCompose(ctx, runner, "up", "--detach", "--no-deps", "--force-recreate", "core")
|
||||
if err != nil {
|
||||
return commandError("core recreation", result, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyCandidate(ctx context.Context, runner Runner, wanted string, previousVolumes []string, configurationSHA string) error {
|
||||
health, err := runCompose(ctx, runner, "exec", "-T", "core", "curl", "-fsS", "http://127.0.0.1:8787/health")
|
||||
if err != nil {
|
||||
return commandError("core health check", health, err)
|
||||
}
|
||||
version, err := Status(ctx, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if version != wanted {
|
||||
return errors.New("candidate Pi version does not match requested pinned version")
|
||||
}
|
||||
if err := Test(ctx, runner); err != nil {
|
||||
return err
|
||||
}
|
||||
configured, err := renderedCore(ctx, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if configured.ConfigurationSHA != configurationSHA {
|
||||
return errors.New("external endpoint configuration changed during Pi update")
|
||||
}
|
||||
after, err := runningImage(ctx, runner, configured.Reference)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !sameStrings(previousVolumes, after.Volumes) {
|
||||
return errors.New("core volume set changed during Pi update")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func restore(ctx context.Context, runner Runner, previous Image) error {
|
||||
tag, err := runner.Run(ctx, []string{"image", "tag", previous.ID, previous.Reference}, nil)
|
||||
if err != nil {
|
||||
return commandError("rollback image restore", tag, err)
|
||||
}
|
||||
if err := recreateCore(ctx, runner); err != nil {
|
||||
return err
|
||||
}
|
||||
configured, err := renderedCore(ctx, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
after, err := runningImage(ctx, runner, configured.Reference)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !sameStrings(previous.Volumes, after.Volumes) {
|
||||
return errors.New("core volume set changed during rollback")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func nonEmptyLines(text string) []string {
|
||||
var values []string
|
||||
for _, value := range strings.Split(text, "\n") {
|
||||
if value = strings.TrimSpace(value); value != "" {
|
||||
values = append(values, value)
|
||||
}
|
||||
}
|
||||
sort.Strings(values)
|
||||
return values
|
||||
}
|
||||
func sameStrings(left, right []string) bool {
|
||||
left, right = append([]string(nil), left...), append([]string(nil), right...)
|
||||
sort.Strings(left)
|
||||
sort.Strings(right)
|
||||
return strings.Join(left, "\x00") == strings.Join(right, "\x00")
|
||||
}
|
||||
Reference in New Issue
Block a user