refactor(cli): rename operator command to tht
This commit is contained in:
@@ -0,0 +1,365 @@
|
||||
package pi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/compose"
|
||||
)
|
||||
|
||||
var choicePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$`)
|
||||
|
||||
type Defaults struct {
|
||||
Provider string `json:"provider"`
|
||||
Model string `json:"model"`
|
||||
Thinking string `json:"thinking"`
|
||||
}
|
||||
|
||||
type ModelOption struct {
|
||||
Provider string `json:"provider"`
|
||||
ID string `json:"id"`
|
||||
}
|
||||
|
||||
type piOptions struct {
|
||||
Providers []string `json:"providers"`
|
||||
Models []ModelOption `json:"models"`
|
||||
Reasoning []string `json:"reasoning"`
|
||||
}
|
||||
|
||||
type settingsFileSnapshot struct {
|
||||
Exists bool `json:"exists"`
|
||||
RawBase64 string `json:"rawBase64"`
|
||||
}
|
||||
|
||||
var internalIdentityHeaders = []string{
|
||||
"-H", "x-thoth-principal-issuer: tht",
|
||||
"-H", "x-thoth-principal-subject: tht-maintenance",
|
||||
"-H", "x-thoth-principal-display-name: Tht maintenance",
|
||||
"-H", "x-thoth-is-admin: 1",
|
||||
}
|
||||
|
||||
// Configure changes the backend's real installation settings through a core-side helper. It
|
||||
// deliberately has no secret or endpoint input: external endpoints remain Compose-owned.
|
||||
func Configure(ctx context.Context, runner Runner, value Defaults) error {
|
||||
if !choicePattern.MatchString(value.Provider) || !choicePattern.MatchString(value.Model) {
|
||||
return errors.New("provider and model must be supported identifiers")
|
||||
}
|
||||
before, err := renderedCore(ctx, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
options, err := configurationOptions(ctx, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
found := false
|
||||
for _, model := range options.Models {
|
||||
if model.Provider == value.Provider && model.ID == value.Model {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return errors.New("provider/model is not in Pi options")
|
||||
}
|
||||
thinkingFound := false
|
||||
for _, reasoning := range options.Reasoning {
|
||||
if reasoning == value.Thinking {
|
||||
thinkingFound = true
|
||||
}
|
||||
}
|
||||
if !thinkingFound {
|
||||
return errors.New("thinking is not in Pi options")
|
||||
}
|
||||
old, err := captureSettingsFile(ctx, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
oldEffective, err := readEffectiveSettings(ctx, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
restore := func(cause error) error {
|
||||
if restoreErr := restoreSettingsFile(context.Background(), runner, old); restoreErr != nil {
|
||||
return fmt.Errorf("%w; previous Pi settings restoration could not be verified: %w", cause, restoreErr)
|
||||
}
|
||||
restoredEffective, restoreErr := readEffectiveSettings(context.Background(), runner)
|
||||
if restoreErr != nil || !bytes.Equal(restoredEffective, oldEffective) {
|
||||
return fmt.Errorf("%w; previous effective Pi settings could not be verified: recovery required", cause)
|
||||
}
|
||||
return cause
|
||||
}
|
||||
result, err := writeDefaults(ctx, runner, value)
|
||||
if err != nil {
|
||||
return restore(commandError("Pi installation settings write", result, err))
|
||||
}
|
||||
settings, err := readEffectiveSettings(ctx, runner)
|
||||
if err != nil {
|
||||
return restore(err)
|
||||
}
|
||||
var saved Defaults
|
||||
if json.Unmarshal(settings, &saved) != nil || saved.Provider != value.Provider || saved.Model != value.Model || saved.Thinking != value.Thinking {
|
||||
return restore(errors.New("Pi installation settings read-back did not match requested provider, model, and thinking"))
|
||||
}
|
||||
after, err := renderedCore(ctx, runner)
|
||||
if err != nil {
|
||||
return restore(err)
|
||||
}
|
||||
if before.ConfigurationSHA != after.ConfigurationSHA {
|
||||
return restore(errors.New("external endpoint configuration changed while configuring Pi"))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ConfigurationOptions(ctx context.Context, runner Runner) ([]ModelOption, error) {
|
||||
options, err := configurationOptions(ctx, runner)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return options.Models, nil
|
||||
}
|
||||
|
||||
func configurationOptions(ctx context.Context, runner Runner) (piOptions, error) {
|
||||
args := append([]string{"exec", "-T", "core", "curl", "-fsS"}, internalIdentityHeaders...)
|
||||
args = append(args, "http://127.0.0.1:8787/pi-management/options")
|
||||
result, err := runCompose(ctx, runner, args...)
|
||||
if err != nil {
|
||||
return piOptions{}, commandError("Pi options check", result, err)
|
||||
}
|
||||
var payload piOptions
|
||||
if json.Unmarshal([]byte(result.Stdout), &payload) != nil || len(payload.Providers) == 0 || len(payload.Models) == 0 || len(payload.Reasoning) == 0 {
|
||||
return piOptions{}, errors.New("Pi options response is invalid or empty")
|
||||
}
|
||||
providers := make(map[string]bool, len(payload.Providers))
|
||||
for _, provider := range payload.Providers {
|
||||
if !choicePattern.MatchString(provider) || providers[provider] {
|
||||
return piOptions{}, errors.New("Pi options response contains an invalid provider")
|
||||
}
|
||||
providers[provider] = true
|
||||
}
|
||||
models := make(map[string]bool, len(payload.Models))
|
||||
for _, option := range payload.Models {
|
||||
key := option.Provider + "\x00" + option.ID
|
||||
if !providers[option.Provider] || !choicePattern.MatchString(option.ID) || models[key] {
|
||||
return piOptions{}, errors.New("Pi options response contains an invalid provider/model")
|
||||
}
|
||||
models[key] = true
|
||||
}
|
||||
reasoning := make(map[string]bool, len(payload.Reasoning))
|
||||
for _, value := range payload.Reasoning {
|
||||
if (value != "low" && value != "medium" && value != "high") || reasoning[value] {
|
||||
return piOptions{}, errors.New("Pi options response contains an invalid reasoning choice")
|
||||
}
|
||||
reasoning[value] = true
|
||||
}
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
func writeDefaults(ctx context.Context, runner Runner, value Defaults) (compose.Result, error) {
|
||||
return runCompose(ctx, runner, "exec", "-T", "core", "node", "/app/backend/dist/settings/settings-cli.js", "--provider", value.Provider, "--model", value.Model, "--thinking", value.Thinking)
|
||||
}
|
||||
|
||||
func captureSettingsFile(ctx context.Context, runner Runner) (settingsFileSnapshot, error) {
|
||||
result, err := runCompose(ctx, runner, "exec", "-T", "core", "node", "/app/backend/dist/settings/settings-cli.js", "--snapshot")
|
||||
if err != nil {
|
||||
return settingsFileSnapshot{}, commandError("Pi installation settings snapshot", result, err)
|
||||
}
|
||||
var snapshot settingsFileSnapshot
|
||||
if json.Unmarshal([]byte(result.Stdout), &snapshot) != nil {
|
||||
return settingsFileSnapshot{}, errors.New("Pi installation settings snapshot is invalid")
|
||||
}
|
||||
raw, decodeErr := base64.StdEncoding.DecodeString(snapshot.RawBase64)
|
||||
if decodeErr != nil || base64.StdEncoding.EncodeToString(raw) != snapshot.RawBase64 || (!snapshot.Exists && len(raw) != 0) {
|
||||
return settingsFileSnapshot{}, errors.New("Pi installation settings snapshot is invalid")
|
||||
}
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
func restoreSettingsFile(ctx context.Context, runner Runner, snapshot settingsFileSnapshot) error {
|
||||
payload, err := json.Marshal(snapshot)
|
||||
if err != nil {
|
||||
return errors.New("Pi installation settings snapshot could not be encoded")
|
||||
}
|
||||
args := []string{"compose", "exec", "-T", "core", "node", "/app/backend/dist/settings/settings-cli.js", "--restore"}
|
||||
result, restoreErr := runner.Run(ctx, args, bytes.NewReader(payload))
|
||||
verified, verifyErr := captureSettingsFile(ctx, runner)
|
||||
if restoreErr != nil {
|
||||
cause := commandError("Pi installation settings restore", result, restoreErr)
|
||||
if verifyErr == nil && verified == snapshot {
|
||||
return recoveryRequired("previous Pi settings bytes were restored but durability was not acknowledged", cause)
|
||||
}
|
||||
return cause
|
||||
}
|
||||
if verifyErr == nil && verified == snapshot {
|
||||
return nil
|
||||
}
|
||||
return errors.New("Pi installation settings restore did not reproduce the exact prior file state")
|
||||
}
|
||||
|
||||
func readEffectiveSettings(ctx context.Context, runner Runner) ([]byte, error) {
|
||||
args := append([]string{"exec", "-T", "core", "curl", "-fsS"}, internalIdentityHeaders...)
|
||||
args = append(args, "http://127.0.0.1:8787/settings")
|
||||
result, err := runCompose(ctx, runner, args...)
|
||||
if err != nil {
|
||||
return nil, commandError("Pi installation settings read-back", result, err)
|
||||
}
|
||||
var settings map[string]json.RawMessage
|
||||
if json.Unmarshal([]byte(result.Stdout), &settings) != nil || settings == nil {
|
||||
return nil, errors.New("Pi installation settings read-back is invalid")
|
||||
}
|
||||
canonical, err := json.Marshal(settings)
|
||||
if err != nil {
|
||||
return nil, errors.New("Pi installation settings read-back could not be normalized")
|
||||
}
|
||||
return canonical, nil
|
||||
}
|
||||
|
||||
// Runner is the narrow, shell-free command boundary shared with tht.
|
||||
type Runner interface {
|
||||
Run(context.Context, []string, io.Reader) (compose.Result, error)
|
||||
}
|
||||
|
||||
// Status reports the image-bundled Pi version without using a host Pi executable.
|
||||
func Status(ctx context.Context, runner Runner) (string, error) {
|
||||
result, err := runCompose(ctx, runner, "exec", "-T", "core", "pi", "--version")
|
||||
if err != nil {
|
||||
return "", commandError("Pi version check", result, err)
|
||||
}
|
||||
version := strings.TrimSpace(result.Stdout)
|
||||
if version == "" {
|
||||
return "", errors.New("Pi version check returned no version")
|
||||
}
|
||||
return version, nil
|
||||
}
|
||||
|
||||
// Doctor verifies the installation-side invariants Pi needs before an update.
|
||||
func Doctor(ctx context.Context, runner Runner) error {
|
||||
if _, err := renderedCore(ctx, runner); err != nil {
|
||||
return err
|
||||
}
|
||||
actual, err := Status(ctx, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
expected, label, err := expectedVersions(ctx, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if actual != expected || actual != label {
|
||||
return errors.New("Pi version does not match the image PI_VERSION and io.thothii.pi.version contract")
|
||||
}
|
||||
for _, check := range [][]string{
|
||||
{"exec", "-T", "core", "sh", "-ceu", "test -w /home/thoth/.pi"},
|
||||
{"exec", "-T", "core", "sh", "-ceu", "test -r /home/thoth/.pi/agent/auth.json"},
|
||||
{"exec", "-T", "core", "curl", "-fsS", "http://127.0.0.1:8787/health"},
|
||||
} {
|
||||
result, err := runCompose(ctx, runner, check...)
|
||||
if err != nil {
|
||||
return commandError("Pi preflight check", result, err)
|
||||
}
|
||||
}
|
||||
return Test(ctx, runner)
|
||||
}
|
||||
|
||||
func expectedVersions(ctx context.Context, runner Runner) (string, string, error) {
|
||||
environment, err := runCompose(ctx, runner, "exec", "-T", "core", "sh", "-ceu", `printf '%s\n' "${PI_VERSION:-}"`)
|
||||
if err != nil {
|
||||
return "", "", commandError("Pi expected-version check", environment, err)
|
||||
}
|
||||
container, err := runCompose(ctx, runner, "ps", "-q", "core")
|
||||
if err != nil || strings.TrimSpace(container.Stdout) == "" {
|
||||
return "", "", commandError("Pi image-label check", container, err)
|
||||
}
|
||||
label, err := runner.Run(ctx, []string{"inspect", "--format", `{{ index .Config.Labels "io.thothii.pi.version" }}`, strings.TrimSpace(container.Stdout)}, nil)
|
||||
if err != nil {
|
||||
return "", "", commandError("Pi image-label check", label, err)
|
||||
}
|
||||
expectedValue, labelValue := strings.TrimSpace(environment.Stdout), strings.TrimSpace(label.Stdout)
|
||||
if expectedValue == "" || labelValue == "" {
|
||||
return "", "", errors.New("Pi image expected-version contract is empty")
|
||||
}
|
||||
return expectedValue, labelValue, nil
|
||||
}
|
||||
|
||||
// Test retains the direct image-version signal, then delegates all Pi configuration/provider smoke
|
||||
// validation to core's dedicated, admin-only Pi Management endpoint.
|
||||
func Test(ctx context.Context, runner Runner) error {
|
||||
if _, err := Status(ctx, runner); err != nil {
|
||||
return err
|
||||
}
|
||||
args := append([]string{"exec", "-T", "core", "curl", "-fsS", "-X", "POST"}, internalIdentityHeaders...)
|
||||
args = append(args, "http://127.0.0.1:8787/pi-management/test")
|
||||
smoke, err := runCompose(ctx, runner, args...)
|
||||
if err != nil {
|
||||
return commandError("Pi smoke check", smoke, err)
|
||||
}
|
||||
var smokePayload struct {
|
||||
Ready bool `json:"ready"`
|
||||
}
|
||||
if json.Unmarshal([]byte(smoke.Stdout), &smokePayload) != nil || !smokePayload.Ready {
|
||||
return errors.New("Pi smoke response is not ready")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func renderedCore(ctx context.Context, runner Runner) (Image, error) {
|
||||
result, err := runCompose(ctx, runner, "config", "--format", "json")
|
||||
if err != nil {
|
||||
return Image{}, commandError("Compose configuration check", result, err)
|
||||
}
|
||||
var document map[string]any
|
||||
if err := json.Unmarshal([]byte(result.Stdout), &document); err != nil {
|
||||
return Image{}, errors.New("Compose returned invalid rendered configuration")
|
||||
}
|
||||
services, ok := document["services"].(map[string]any)
|
||||
if !ok {
|
||||
return Image{}, errors.New("rendered Compose configuration has no services")
|
||||
}
|
||||
core, ok := services["core"].(map[string]any)
|
||||
reference, _ := core["image"].(string)
|
||||
if !ok || reference == "" {
|
||||
return Image{}, errors.New("rendered Compose configuration has no core image")
|
||||
}
|
||||
environment, _ := core["environment"].(map[string]any)
|
||||
endpoint, exists := environment["THT_LLM_URL"].(string)
|
||||
if !exists || strings.TrimSpace(endpoint) == "" {
|
||||
return Image{}, errors.New("THT_LLM_URL must be configured before Pi lifecycle operations")
|
||||
}
|
||||
// Lifecycle overrides intentionally replace only core.image. Normalize that field so the
|
||||
// non-secret configuration digest continues to detect endpoint/mount/configuration drift.
|
||||
core["image"] = "<lifecycle-image>"
|
||||
normalized, err := json.Marshal(document)
|
||||
if err != nil {
|
||||
return Image{}, errors.New("Compose configuration could not be normalized")
|
||||
}
|
||||
digest := sha256.Sum256(normalized)
|
||||
return Image{Reference: reference, ConfigurationSHA: fmt.Sprintf("%x", digest[:])}, nil
|
||||
}
|
||||
|
||||
func runCompose(ctx context.Context, runner Runner, args ...string) (compose.Result, error) {
|
||||
return runner.Run(ctx, append([]string{"compose"}, args...), nil)
|
||||
}
|
||||
|
||||
func commandError(label string, result compose.Result, err error) error {
|
||||
if result.ExitCode != 0 {
|
||||
return commandFailure{message: fmt.Sprintf("%s failed (exit %d)", label, result.ExitCode), exitCode: result.ExitCode}
|
||||
}
|
||||
return commandFailure{message: fmt.Sprintf("%s failed", label)}
|
||||
}
|
||||
|
||||
type commandFailure struct {
|
||||
message string
|
||||
exitCode int
|
||||
}
|
||||
|
||||
func (e commandFailure) Error() string { return e.message }
|
||||
|
||||
// ExitCode exposes a Docker child exit code without exposing its output.
|
||||
func (e commandFailure) ExitCode() int { return e.exitCode }
|
||||
@@ -0,0 +1,282 @@
|
||||
package pi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/compose"
|
||||
)
|
||||
|
||||
func TestDoctorRequiresExternalEndpointAuthPiStateAndHealth(t *testing.T) {
|
||||
fake := newFakeRunner()
|
||||
if err := Doctor(context.Background(), fake); err != nil {
|
||||
t.Fatalf("Doctor() error = %v", err)
|
||||
}
|
||||
for _, command := range []string{"pi --version", "PI_VERSION", "io.thothii.pi.version", "test -w /home/thoth/.pi", "test -r /home/thoth/.pi/agent/auth.json", "/health"} {
|
||||
assertCalled(t, fake.calls, command)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoctorRejectsActualEnvironmentAndImageLabelVersionMismatches(t *testing.T) {
|
||||
for _, mismatch := range []string{"actual", "environment", "label"} {
|
||||
t.Run(mismatch, func(t *testing.T) {
|
||||
fake := newFakeRunner()
|
||||
switch mismatch {
|
||||
case "actual":
|
||||
fake.version = "0.80.2"
|
||||
case "environment":
|
||||
fake.expectedVersion = "0.80.2"
|
||||
case "label":
|
||||
fake.labelVersion = "0.80.2"
|
||||
}
|
||||
if err := Doctor(context.Background(), fake); err == nil || !strings.Contains(err.Error(), "version") {
|
||||
t.Fatalf("Doctor() error = %v, want expected-version mismatch", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigureRestoresAndVerifiesOldSettingsAfterEveryPostSnapshotFailure(t *testing.T) {
|
||||
for _, failure := range []string{"helper", "readback", "digest"} {
|
||||
t.Run(failure, func(t *testing.T) {
|
||||
old := Defaults{Provider: "old", Model: "old-model", Thinking: "low"}
|
||||
raw, _ := json.Marshal(old)
|
||||
fake := &configureRunner{failure: failure, settings: old, settingsExist: true, settingsRaw: raw}
|
||||
err := Configure(context.Background(), fake, Defaults{Provider: "new", Model: "new-model", Thinking: "high"})
|
||||
if err == nil {
|
||||
t.Fatal("Configure() error = nil, want injected failure")
|
||||
}
|
||||
if fake.settings != (Defaults{Provider: "old", Model: "old-model", Thinking: "low"}) {
|
||||
t.Fatalf("settings after failure = %#v, want old snapshot", fake.settings)
|
||||
}
|
||||
if !fake.settingsExist || string(fake.settingsRaw) != string(raw) {
|
||||
t.Fatalf("settings raw snapshot after failure = exists:%t raw:%q, want %q", fake.settingsExist, fake.settingsRaw, raw)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSettingsRestoreDoesNotMaskExplicitDurabilityFailureWithMatchingReadback(t *testing.T) {
|
||||
old := Defaults{Provider: "old", Model: "old-model", Thinking: "low"}
|
||||
raw, _ := json.Marshal(old)
|
||||
fake := &configureRunner{
|
||||
failure: "restore-durability",
|
||||
settings: Defaults{Provider: "new", Model: "new-model", Thinking: "high"},
|
||||
settingsExist: true,
|
||||
settingsRaw: []byte(`{"provider":"new","model":"new-model","thinking":"high"}`),
|
||||
}
|
||||
snapshot := settingsFileSnapshot{Exists: true, RawBase64: base64.StdEncoding.EncodeToString(raw)}
|
||||
|
||||
err := restoreSettingsFile(context.Background(), fake, snapshot)
|
||||
|
||||
var recovery interface{ RecoveryRequired() bool }
|
||||
if err == nil || !errors.As(err, &recovery) || !recovery.RecoveryRequired() {
|
||||
t.Fatalf("restore error = %v; want typed recovery-required result", err)
|
||||
}
|
||||
if !fake.settingsExist || string(fake.settingsRaw) != string(raw) || fake.settings != old {
|
||||
t.Fatalf("restored state = exists:%t raw:%q value:%#v; want exact old bytes", fake.settingsExist, fake.settingsRaw, fake.settings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurePreservesTypedRecoveryRequiredErrorFromSettingsRestore(t *testing.T) {
|
||||
old := Defaults{Provider: "old", Model: "old-model", Thinking: "low"}
|
||||
raw, _ := json.Marshal(old)
|
||||
fake := &configureRunner{
|
||||
failure: "helper",
|
||||
restoreDurabilityFailure: true,
|
||||
settings: old,
|
||||
settingsExist: true,
|
||||
settingsRaw: raw,
|
||||
}
|
||||
|
||||
err := Configure(context.Background(), fake, Defaults{Provider: "new", Model: "new-model", Thinking: "high"})
|
||||
|
||||
var recovery interface{ RecoveryRequired() bool }
|
||||
if err == nil || !errors.As(err, &recovery) || !recovery.RecoveryRequired() {
|
||||
t.Fatalf("Configure() error = %v; want typed recovery-required result", err)
|
||||
}
|
||||
}
|
||||
|
||||
type configureRunner struct {
|
||||
failure string
|
||||
restoreDurabilityFailure bool
|
||||
settings Defaults
|
||||
settingsExist bool
|
||||
settingsRaw []byte
|
||||
settingsReads int
|
||||
configReads int
|
||||
writes int
|
||||
}
|
||||
|
||||
func (f *configureRunner) Run(_ context.Context, args []string, stdin io.Reader) (compose.Result, error) {
|
||||
call := strings.Join(args, " ")
|
||||
switch {
|
||||
case strings.Contains(call, "config --format json"):
|
||||
f.configReads++
|
||||
endpoint := "https://llm.example.invalid"
|
||||
if f.failure == "digest" && f.configReads > 1 {
|
||||
endpoint = "https://drift.example.invalid"
|
||||
}
|
||||
return compose.Result{Stdout: `{"services":{"core":{"image":"thothii-core:local","environment":{"THT_LLM_URL":"` + endpoint + `"}}}}`}, nil
|
||||
case strings.Contains(call, "/pi-management/options"):
|
||||
return compose.Result{Stdout: `{"providers":["old","new"],"models":[{"provider":"old","id":"old-model"},{"provider":"new","id":"new-model"}],"reasoning":["low","medium","high"]}`}, nil
|
||||
case strings.Contains(call, "settings-cli.js --snapshot"):
|
||||
raw := f.settingsRaw
|
||||
payload := map[string]any{"exists": f.settingsExist, "rawBase64": base64.StdEncoding.EncodeToString(raw)}
|
||||
contents, _ := json.Marshal(payload)
|
||||
return compose.Result{Stdout: string(contents)}, nil
|
||||
case strings.Contains(call, "settings-cli.js --restore"):
|
||||
var payload struct {
|
||||
Exists bool `json:"exists"`
|
||||
RawBase64 string `json:"rawBase64"`
|
||||
}
|
||||
contents, _ := io.ReadAll(stdin)
|
||||
if json.Unmarshal(contents, &payload) != nil {
|
||||
return compose.Result{ExitCode: 2}, errors.New("invalid restore payload")
|
||||
}
|
||||
f.settingsExist = payload.Exists
|
||||
f.settingsRaw, _ = base64.StdEncoding.DecodeString(payload.RawBase64)
|
||||
f.settings = Defaults{Provider: "old", Model: "old-model", Thinking: "low"}
|
||||
if payload.Exists {
|
||||
_ = json.Unmarshal(f.settingsRaw, &f.settings)
|
||||
}
|
||||
if f.failure == "restore-durability" || f.restoreDurabilityFailure {
|
||||
return compose.Result{ExitCode: 2}, errors.New("injected post-rename directory fsync failure")
|
||||
}
|
||||
return compose.Result{}, nil
|
||||
case strings.Contains(call, "settings-cli.js"):
|
||||
if strings.Contains(call, "--provider new") {
|
||||
f.settings = Defaults{Provider: "new", Model: "new-model", Thinking: "high"}
|
||||
f.settingsExist = true
|
||||
f.settingsRaw, _ = json.MarshalIndent(f.settings, "", " ")
|
||||
f.writes++
|
||||
if f.failure == "helper" {
|
||||
return compose.Result{ExitCode: 17}, errors.New("injected helper failure")
|
||||
}
|
||||
} else {
|
||||
f.settings = Defaults{Provider: "old", Model: "old-model", Thinking: "low"}
|
||||
f.settingsExist = true
|
||||
f.settingsRaw, _ = json.Marshal(f.settings)
|
||||
}
|
||||
return compose.Result{}, nil
|
||||
case strings.Contains(call, "/settings"):
|
||||
f.settingsReads++
|
||||
if f.failure == "readback" && f.settings.Provider == "new" {
|
||||
return compose.Result{Stdout: `{}`}, nil
|
||||
}
|
||||
if !f.settingsExist {
|
||||
return compose.Result{Stdout: `{"provider":"old","model":"old-model","thinking":"low"}`}, nil
|
||||
}
|
||||
contents, _ := json.Marshal(f.settings)
|
||||
return compose.Result{Stdout: string(contents)}, nil
|
||||
default:
|
||||
return compose.Result{}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigureAllowsAFirstRunWithoutAnExistingSettingsFile(t *testing.T) {
|
||||
fake := &configureRunner{}
|
||||
if err := Configure(context.Background(), fake, Defaults{Provider: "new", Model: "new-model", Thinking: "high"}); err != nil {
|
||||
t.Fatalf("Configure() clean install error = %v", err)
|
||||
}
|
||||
if !fake.settingsExist || fake.writes != 1 || fake.settings.Provider != "new" {
|
||||
t.Fatalf("clean settings = exists:%t writes:%d value:%#v", fake.settingsExist, fake.writes, fake.settings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigureCompensationRestoresAbsentAndExactEmptyPriorFiles(t *testing.T) {
|
||||
for _, prior := range []struct {
|
||||
name string
|
||||
exists bool
|
||||
raw []byte
|
||||
}{
|
||||
{name: "absent"},
|
||||
{name: "empty", exists: true, raw: []byte{}},
|
||||
{name: "exact raw", exists: true, raw: []byte("{\n \"workspace\": \"kept\",\n \"provider\": \"old\",\n \"model\": \"old-model\",\n \"thinking\": \"low\"\n}\n")},
|
||||
} {
|
||||
t.Run(prior.name, func(t *testing.T) {
|
||||
fake := &configureRunner{failure: "digest", settingsExist: prior.exists, settingsRaw: append([]byte{}, prior.raw...), settings: Defaults{Provider: "old", Model: "old-model", Thinking: "low"}}
|
||||
err := Configure(context.Background(), fake, Defaults{Provider: "new", Model: "new-model", Thinking: "high"})
|
||||
if err == nil {
|
||||
t.Fatal("Configure() error = nil, want compensated digest failure")
|
||||
}
|
||||
if fake.writes != 1 {
|
||||
t.Fatalf("settings writes = %d, want selected values written before compensation", fake.writes)
|
||||
}
|
||||
if fake.settingsExist != prior.exists || string(fake.settingsRaw) != string(prior.raw) {
|
||||
t.Fatalf("restored exists/raw = %t/%q, want %t/%q", fake.settingsExist, fake.settingsRaw, prior.exists, prior.raw)
|
||||
}
|
||||
if fake.settingsReads < 3 {
|
||||
t.Fatalf("settings reads = %d, want prior effective state, requested readback, and restored default verification", fake.settingsReads)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Catches tht reading the legacy public model route instead of the admin-only closed Pi
|
||||
// Management choices before it writes shared installation defaults.
|
||||
func TestConfigureLoadsDedicatedClosedOptionsWritesRealCoreSettingsAndUsesUpstreamIdentity(t *testing.T) {
|
||||
fake := newFakeRunner()
|
||||
if err := Configure(context.Background(), fake, Defaults{Provider: "provider", Model: "model", Thinking: "medium"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertCalled(t, fake.calls, "/pi-management/options")
|
||||
assertCalled(t, fake.calls, "node /app/backend/dist/settings/settings-cli.js --provider provider --model model --thinking medium")
|
||||
assertCalled(t, fake.calls, "x-thoth-principal-subject: tht-maintenance")
|
||||
if got := strings.Join(fake.calls, "\n"); strings.Contains(got, "pi-defaults.json") || strings.Contains(got, "secret") {
|
||||
t.Fatalf("commands=%q", got)
|
||||
}
|
||||
if err := Configure(context.Background(), fake, Defaults{Provider: "provider", Model: "unknown", Thinking: "medium"}); err == nil {
|
||||
t.Fatal("expected unknown model rejection")
|
||||
}
|
||||
}
|
||||
|
||||
// Catches a smoke check that composes health/models/settings itself and drifts from the dedicated
|
||||
// backend contract, rather than retaining only the independent in-container version signal.
|
||||
func TestTestUsesDedicatedSmokeEndpointAndIndependentImageVersionProbe(t *testing.T) {
|
||||
fake := newFakeRunner()
|
||||
if err := Test(context.Background(), fake); err != nil {
|
||||
t.Fatalf("Test() error = %v", err)
|
||||
}
|
||||
for _, command := range []string{"pi --version", "/pi-management/test", "x-thoth-principal-subject: tht-maintenance"} {
|
||||
assertCalled(t, fake.calls, command)
|
||||
}
|
||||
for _, legacy := range []string{"/health", "/models", "/settings"} {
|
||||
if strings.Contains(strings.Join(fake.calls, "\n"), legacy) {
|
||||
t.Fatalf("Pi smoke invoked legacy endpoint %q: %s", legacy, strings.Join(fake.calls, "\n"))
|
||||
}
|
||||
}
|
||||
if got := strings.Join(fake.calls, "\n"); strings.Contains(got, "secret") {
|
||||
t.Fatalf("probe commands expose secret: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Catches an ignored negative ready result from the backend smoke endpoint, which would report a
|
||||
// successfully verified candidate image while its configured Pi runtime is unusable.
|
||||
func TestTestRequiresDedicatedSmokeEndpointToReportReady(t *testing.T) {
|
||||
fake := newFakeRunner()
|
||||
fake.piManagementTestWire = `{"ready":false,"message":"provider unavailable"}`
|
||||
if err := Test(context.Background(), fake); err == nil || !strings.Contains(err.Error(), "Pi smoke response is not ready") {
|
||||
t.Fatalf("Test() error = %v, want negative dedicated smoke result", err)
|
||||
}
|
||||
fake.piManagementTestWire = `{"ready":true}`
|
||||
if err := Test(context.Background(), fake); err != nil {
|
||||
t.Fatalf("Test() exact match error = %v", err)
|
||||
}
|
||||
assertCalled(t, fake.calls, "pi --version")
|
||||
}
|
||||
|
||||
// Catches tht accepting a reasoning level that the backend did not publish as a closed
|
||||
// installation option, which would bypass the Pi Management validation surface.
|
||||
func TestConfigureRejectsReasoningOutsideDedicatedClosedOptions(t *testing.T) {
|
||||
fake := newFakeRunner()
|
||||
fake.piManagementOptionsWire = `{"providers":["provider"],"models":[{"provider":"provider","id":"model"}],"reasoning":["low"]}`
|
||||
if err := Configure(context.Background(), fake, Defaults{Provider: "provider", Model: "model", Thinking: "high"}); err == nil || !strings.Contains(err.Error(), "Pi options") {
|
||||
t.Fatalf("Configure() error = %v, want closed reasoning rejection", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
//go:build !windows
|
||||
|
||||
package pi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// durableReplace acknowledges both the data file and its directory entry. A successful return
|
||||
// is the strongest atomic replacement guarantee supported by Unix filesystems.
|
||||
func durableReplace(temporary, target, directory string) error {
|
||||
if err := os.Rename(temporary, target); err != nil {
|
||||
return err
|
||||
}
|
||||
dir, err := os.Open(directory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dir.Close()
|
||||
return dir.Sync()
|
||||
}
|
||||
|
||||
func durableRemove(path string) error {
|
||||
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
dir, err := os.Open(filepath.Dir(path))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dir.Close()
|
||||
return dir.Sync()
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
//go:build windows
|
||||
|
||||
package pi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
// MoveFileEx requests replacement and write-through on Windows. Directory fsync is not exposed
|
||||
// by the Windows API in the same form as Unix, so callers must not claim a stronger guarantee.
|
||||
func durableReplace(temporary, target, _ string) error {
|
||||
from, err := windows.UTF16PtrFromString(temporary)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
to, err := windows.UTF16PtrFromString(target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return windows.MoveFileEx(from, to, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH)
|
||||
}
|
||||
|
||||
func durableRemove(path string) error {
|
||||
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package pi
|
||||
|
||||
// RecoveryRequiredError marks a result whose immediate state may be safe but whose durability
|
||||
// was explicitly not acknowledged. Callers must not report success or clear maintenance.
|
||||
type RecoveryRequiredError struct {
|
||||
Operation string
|
||||
Cause error
|
||||
}
|
||||
|
||||
func (e *RecoveryRequiredError) Error() string {
|
||||
return e.Operation + ": recovery required"
|
||||
}
|
||||
|
||||
func (e *RecoveryRequiredError) Unwrap() error {
|
||||
return e.Cause
|
||||
}
|
||||
|
||||
func (e *RecoveryRequiredError) RecoveryRequired() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func recoveryRequired(operation string, cause error) error {
|
||||
return &RecoveryRequiredError{Operation: operation, Cause: cause}
|
||||
}
|
||||
@@ -0,0 +1,360 @@
|
||||
package pi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInterruptedRestart = errors.New("a previous Pi restart is incomplete; recover lifecycle maintenance before another Pi lifecycle operation")
|
||||
errRestartImageDrift = errors.New("core image changed during Pi restart")
|
||||
errRestartConfigurationDrift = errors.New("external endpoint configuration changed during Pi restart")
|
||||
errRestartMountDrift = errors.New("core persistence mount contract changed during Pi restart")
|
||||
errRestartConfirmation = &restartDiagnosticError{
|
||||
message: "restart requires --yes after reviewing the planned Pi core recreation",
|
||||
cause: ErrConfirmationRequired,
|
||||
}
|
||||
errRestartActiveSessions = &restartDiagnosticError{
|
||||
message: "active sessions must be drained before restarting Pi; use --drain only after they are complete",
|
||||
cause: ErrActiveSessions,
|
||||
}
|
||||
)
|
||||
|
||||
type restartDiagnosticError struct {
|
||||
message string
|
||||
cause error
|
||||
}
|
||||
|
||||
func (e *restartDiagnosticError) Error() string { return e.message }
|
||||
|
||||
func (e *restartDiagnosticError) Unwrap() error { return e.cause }
|
||||
|
||||
type RestartRequest struct {
|
||||
StatePath string
|
||||
UpdateStatePath string
|
||||
Confirm bool
|
||||
Drain bool
|
||||
}
|
||||
|
||||
type RestartResult struct {
|
||||
StatePath string
|
||||
Version string
|
||||
}
|
||||
|
||||
func Restart(ctx context.Context, runner Runner, request RestartRequest) (RestartResult, error) {
|
||||
return restartWithHooks(ctx, runner, request, defaultLifecycleHooks)
|
||||
}
|
||||
|
||||
func restartWithHooks(
|
||||
ctx context.Context,
|
||||
runner Runner,
|
||||
request RestartRequest,
|
||||
hooks lifecycleHooks,
|
||||
) (result RestartResult, retErr error) {
|
||||
if err := validateRestartStatePaths(request.StatePath, request.UpdateStatePath); err != nil {
|
||||
return RestartResult{}, err
|
||||
}
|
||||
lock, err := acquireLock(request.StatePath)
|
||||
if err != nil {
|
||||
return RestartResult{StatePath: request.StatePath}, err
|
||||
}
|
||||
defer lock.Release()
|
||||
if !request.Confirm {
|
||||
return RestartResult{StatePath: request.StatePath}, errRestartConfirmation
|
||||
}
|
||||
if state, err := readState(request.UpdateStatePath); err == nil && stateNeedsRecovery(state) {
|
||||
return RestartResult{StatePath: request.StatePath}, ErrInterruptedUpdate
|
||||
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return RestartResult{StatePath: request.StatePath}, err
|
||||
}
|
||||
if err := prepareLifecycleMutation(request.StatePath, hooks.removeFile); err != nil {
|
||||
return RestartResult{StatePath: request.StatePath}, err
|
||||
}
|
||||
clearMaintenance := true
|
||||
mutationStarted := false
|
||||
var state State
|
||||
defer func() {
|
||||
if !clearMaintenance {
|
||||
return
|
||||
}
|
||||
if clearErr := setMaintenance(context.Background(), runner, false); clearErr != nil {
|
||||
result.StatePath = request.StatePath
|
||||
if mutationStarted {
|
||||
clearMaintenance = false
|
||||
cause := clearErr
|
||||
if writeErr := hooks.writeState(request.StatePath, state); writeErr != nil {
|
||||
cause = errors.Join(cause, fmt.Errorf("restart recovery state could not be restored: %w", writeErr))
|
||||
}
|
||||
retErr = errors.Join(retErr, recoveryRequired("Pi restart maintenance cleanup failed", cause))
|
||||
return
|
||||
}
|
||||
retErr = errors.Join(retErr, fmt.Errorf("maintenance admission gate could not be cleared: %w", clearErr))
|
||||
}
|
||||
}()
|
||||
if err := setMaintenance(ctx, runner, true); err != nil {
|
||||
return RestartResult{StatePath: request.StatePath}, err
|
||||
}
|
||||
|
||||
if err := waitForInactiveSessions(ctx, runner, request.Drain, hooks.sleep); err != nil {
|
||||
return RestartResult{StatePath: request.StatePath}, restartDiagnostic(err)
|
||||
}
|
||||
if err := Doctor(ctx, runner); err != nil {
|
||||
return RestartResult{StatePath: request.StatePath}, err
|
||||
}
|
||||
version, err := Status(ctx, runner)
|
||||
if err != nil {
|
||||
return RestartResult{StatePath: request.StatePath}, err
|
||||
}
|
||||
configured, err := renderedCore(ctx, runner)
|
||||
if err != nil {
|
||||
return RestartResult{StatePath: request.StatePath, Version: version}, err
|
||||
}
|
||||
previous, err := runningImage(ctx, runner, configured.Reference)
|
||||
if err != nil {
|
||||
return RestartResult{StatePath: request.StatePath, Version: version}, err
|
||||
}
|
||||
previous.ConfigurationSHA = configured.ConfigurationSHA
|
||||
transaction := lifecycleTransaction(request.StatePath)
|
||||
previous.Reference = lifecycleImageTag(transaction, "restart")
|
||||
if err := tagImage(ctx, runner, previous.ID, previous.Reference, "restart image pin"); err != nil {
|
||||
return RestartResult{StatePath: request.StatePath, Version: version}, err
|
||||
}
|
||||
state = State{
|
||||
Transaction: transaction,
|
||||
Phase: PhasePreflight,
|
||||
Target: Target{Version: version, Source: "restart"},
|
||||
Previous: previous,
|
||||
}
|
||||
if err := hooks.writeState(request.StatePath, state); err != nil {
|
||||
return RestartResult{StatePath: request.StatePath, Version: version}, err
|
||||
}
|
||||
overridePath := lifecycleOverridePath(request.StatePath, transaction)
|
||||
if err := writeLifecycleOverride(overridePath, previous.Reference); err != nil {
|
||||
return RestartResult{StatePath: request.StatePath, Version: version}, err
|
||||
}
|
||||
lifecycle := composeOverrideRunner{Runner: runner, path: overridePath}
|
||||
if running, err := activeSessions(ctx, runner); err != nil {
|
||||
return RestartResult{StatePath: request.StatePath, Version: version}, err
|
||||
} else if running {
|
||||
return RestartResult{StatePath: request.StatePath, Version: version}, errRestartActiveSessions
|
||||
}
|
||||
state.MutationStarted = true
|
||||
if err := hooks.writeState(request.StatePath, state); err != nil {
|
||||
return RestartResult{StatePath: request.StatePath, Version: version}, err
|
||||
}
|
||||
mutationStarted = true
|
||||
clearMaintenance = false
|
||||
if err := recreateCoreWithoutImageChanges(ctx, lifecycle); err != nil {
|
||||
return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart core recreation failed", err)
|
||||
}
|
||||
if err := ensureMaintenance(ctx, lifecycle); err != nil {
|
||||
return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart maintenance proof failed", err)
|
||||
}
|
||||
state.Phase = PhaseRecreated
|
||||
if err := hooks.writeState(request.StatePath, state); err != nil {
|
||||
return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart recreation state could not be recorded", err)
|
||||
}
|
||||
if err := verifyRestart(ctx, lifecycle, version, previous); err != nil {
|
||||
return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart verification failed", err)
|
||||
}
|
||||
state.Phase = PhaseVerified
|
||||
if err := hooks.writeState(request.StatePath, state); err != nil {
|
||||
return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart verification state could not be recorded", err)
|
||||
}
|
||||
if err := hooks.removeFile(overridePath); err != nil {
|
||||
return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart image override could not be removed", err)
|
||||
}
|
||||
if err := hooks.removeFile(request.StatePath); err != nil {
|
||||
return RestartResult{StatePath: request.StatePath, Version: version}, recoveryRequired("Pi restart recovery state could not be removed", err)
|
||||
}
|
||||
clearMaintenance = true
|
||||
return RestartResult{StatePath: request.StatePath, Version: version}, nil
|
||||
}
|
||||
|
||||
func recreateCoreWithoutImageChanges(ctx context.Context, runner Runner) error {
|
||||
result, err := runCompose(
|
||||
ctx,
|
||||
runner,
|
||||
"up",
|
||||
"--detach",
|
||||
"--wait",
|
||||
"--wait-timeout",
|
||||
"45",
|
||||
"--no-deps",
|
||||
"--force-recreate",
|
||||
"--no-build",
|
||||
"--pull",
|
||||
"never",
|
||||
"core",
|
||||
)
|
||||
if err != nil {
|
||||
return commandError("core recreation", result, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyRestart(ctx context.Context, runner Runner, wanted string, previous Image) error {
|
||||
if err := Doctor(ctx, runner); err != nil {
|
||||
return err
|
||||
}
|
||||
version, err := Status(ctx, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if version != wanted {
|
||||
return errors.New("Pi version changed during core restart")
|
||||
}
|
||||
configured, err := renderedCore(ctx, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
after, err := runningImage(ctx, runner, configured.Reference)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if after.ID != previous.ID {
|
||||
return errRestartImageDrift
|
||||
}
|
||||
if configured.ConfigurationSHA != previous.ConfigurationSHA {
|
||||
return errRestartConfigurationDrift
|
||||
}
|
||||
if !sameMounts(previous.Mounts, after.Mounts) {
|
||||
return errRestartMountDrift
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func RecoverLifecycleMaintenance(
|
||||
ctx context.Context,
|
||||
runner Runner,
|
||||
updateStatePath string,
|
||||
restartStatePath string,
|
||||
confirm bool,
|
||||
) error {
|
||||
if err := validateRestartStatePaths(restartStatePath, updateStatePath); err != nil {
|
||||
return err
|
||||
}
|
||||
if !confirm {
|
||||
return errRestartConfirmation
|
||||
}
|
||||
lock, err := acquireLock(restartStatePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer lock.Release()
|
||||
|
||||
restartState, restartErr := readState(restartStatePath)
|
||||
if restartErr == nil {
|
||||
if err := validateRestartRecoveryState(restartState); err != nil {
|
||||
return err
|
||||
}
|
||||
restartOverride := lifecycleOverridePath(restartStatePath, restartState.Transaction)
|
||||
if restartState.MutationStarted {
|
||||
if err := tagImage(ctx, runner, restartState.Previous.ID, restartState.Previous.Reference, "restart recovery image pin"); err != nil {
|
||||
return recoveryRequired("Pi restart recovery image pin could not be restored", err)
|
||||
}
|
||||
if err := writeLifecycleOverride(restartOverride, restartState.Previous.Reference); err != nil {
|
||||
return recoveryRequired("Pi restart recovery image override could not be restored", err)
|
||||
}
|
||||
lifecycle := composeOverrideRunner{Runner: runner, path: restartOverride}
|
||||
if err := ensureMaintenance(ctx, lifecycle); err != nil {
|
||||
return recoveryRequired("Pi restart maintenance recovery failed", err)
|
||||
}
|
||||
if err := verifyRestart(ctx, lifecycle, restartState.Target.Version, restartState.Previous); err != nil {
|
||||
return recoveryRequired("Pi restart recovery verification failed", err)
|
||||
}
|
||||
}
|
||||
if err := durableRemove(restartOverride); err != nil {
|
||||
return recoveryRequired("Pi restart recovery image override could not be removed", err)
|
||||
}
|
||||
if err := durableRemove(restartStatePath); err != nil {
|
||||
return recoveryRequired("Pi restart recovery state could not be removed", err)
|
||||
}
|
||||
} else if !errors.Is(restartErr, os.ErrNotExist) {
|
||||
return restartErr
|
||||
}
|
||||
return recoverMaintenanceLocked(ctx, runner, updateStatePath)
|
||||
}
|
||||
|
||||
func restartDiagnostic(err error) error {
|
||||
if errors.Is(err, ErrActiveSessions) {
|
||||
return errRestartActiveSessions
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func validateRestartRecoveryState(state State) error {
|
||||
invalid := func(reason string) error {
|
||||
return fmt.Errorf("%w: restart recovery state is invalid: %s", ErrInvalidRequest, reason)
|
||||
}
|
||||
if state.Transaction == "" {
|
||||
return invalid("transaction is missing")
|
||||
}
|
||||
if state.Target.Source != "restart" || state.Target.Version == "" {
|
||||
return invalid("target is not a restart with a recorded version")
|
||||
}
|
||||
if state.Previous.ConfigurationSHA == "" {
|
||||
return invalid("previous external configuration identity is missing")
|
||||
}
|
||||
if state.Candidate.ID != "" || state.Candidate.Reference != "" || len(state.Candidate.Mounts) != 0 ||
|
||||
state.Candidate.MountFingerprint != "" || state.Candidate.ConfigurationSHA != "" {
|
||||
return invalid("restart state contains image candidate metadata")
|
||||
}
|
||||
switch state.Phase {
|
||||
case PhasePreflight:
|
||||
return nil
|
||||
case PhaseRecreated, PhaseVerified:
|
||||
if state.MutationStarted {
|
||||
return nil
|
||||
}
|
||||
return invalid("post-recreation phase has no mutation marker")
|
||||
default:
|
||||
return invalid("phase is not valid for restart")
|
||||
}
|
||||
}
|
||||
|
||||
func validateRestartStatePaths(restartStatePath, updateStatePath string) error {
|
||||
if restartStatePath == "" {
|
||||
return errors.New("restart state path is required")
|
||||
}
|
||||
if updateStatePath == "" {
|
||||
return errors.New("update state path is required")
|
||||
}
|
||||
if filepath.Clean(restartStatePath) == filepath.Clean(updateStatePath) {
|
||||
return fmt.Errorf("%w: restart and update state paths must remain separate", ErrInvalidRequest)
|
||||
}
|
||||
if lifecycleLockPath(restartStatePath) != lifecycleLockPath(updateStatePath) {
|
||||
return fmt.Errorf("%w: restart and update state paths must share one lifecycle control directory", ErrInvalidRequest)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func pairedRestartStatePath(updateStatePath string) string {
|
||||
return filepath.Join(filepath.Dir(updateStatePath), "restart-state.json")
|
||||
}
|
||||
|
||||
func prepareLifecycleMutation(restartStatePath string, removeFile func(string) error) error {
|
||||
state, err := readState(restartStatePath)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("restart recovery state could not be validated: %w", err)
|
||||
}
|
||||
if err := validateRestartRecoveryState(state); err != nil {
|
||||
return err
|
||||
}
|
||||
if state.Phase != PhaseVerified || !state.MutationStarted {
|
||||
return ErrInterruptedRestart
|
||||
}
|
||||
if err := removeFile(lifecycleOverridePath(restartStatePath, state.Transaction)); err != nil {
|
||||
return recoveryRequired("verified restart override could not be cleaned up", err)
|
||||
}
|
||||
if err := removeFile(restartStatePath); err != nil {
|
||||
return recoveryRequired("verified restart state could not be cleaned up", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,599 @@
|
||||
package pi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRestartRequiresConfirmationWithoutInvokingCompose(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
_, err := Restart(context.Background(), fake, RestartRequest{
|
||||
StatePath: filepath.Join(dir, "restart-state.json"),
|
||||
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
||||
})
|
||||
if !errors.Is(err, ErrConfirmationRequired) {
|
||||
t.Fatalf("Restart() error = %v, want ErrConfirmationRequired", err)
|
||||
}
|
||||
if got, want := err.Error(), "restart requires --yes after reviewing the planned Pi core recreation"; got != want {
|
||||
t.Fatalf("Restart() error text = %q, want %q", got, want)
|
||||
}
|
||||
assertNotCalled(t, fake.calls, "compose")
|
||||
}
|
||||
|
||||
func TestRestartDrainsRecreatesOnlyCoreAndRetainsImage(t *testing.T) {
|
||||
fake := newFakeRunner()
|
||||
fake.sessionsWire = `[{"status":"open","archived":false}]`
|
||||
dir := t.TempDir()
|
||||
hooks := defaultLifecycleHooks
|
||||
sleepCalls := 0
|
||||
hooks.sleep = func(duration time.Duration) {
|
||||
sleepCalls++
|
||||
if duration != time.Second {
|
||||
t.Fatalf("drain sleep = %s, want %s", duration, time.Second)
|
||||
}
|
||||
fake.sessionsWire = `[]`
|
||||
}
|
||||
|
||||
result, err := restartWithHooks(context.Background(), fake, RestartRequest{
|
||||
StatePath: filepath.Join(dir, "restart-state.json"),
|
||||
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
||||
Confirm: true,
|
||||
Drain: true,
|
||||
}, hooks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Version != fake.version {
|
||||
t.Fatalf("version = %q, want %q", result.Version, fake.version)
|
||||
}
|
||||
if sleepCalls != 1 {
|
||||
t.Fatalf("drain sleep calls = %d, want 1", sleepCalls)
|
||||
}
|
||||
assertCalled(t, fake.calls, "up --detach --wait --wait-timeout 45 --no-deps --force-recreate --no-build --pull never core")
|
||||
assertNotCalled(t, fake.calls, "compose build --pull")
|
||||
for _, call := range fake.calls {
|
||||
if strings.HasPrefix(call, "pull ") {
|
||||
t.Fatalf("restart invoked direct image pull: %s", call)
|
||||
}
|
||||
}
|
||||
assertNotCalled(t, fake.calls, "frontend")
|
||||
if _, err := os.Stat(result.StatePath); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("successful restart state still exists: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestartPinsCapturedImageWhenConfiguredTagMovesBeforeRecreate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
hooks := defaultLifecycleHooks
|
||||
write := hooks.writeState
|
||||
hooks.writeState = func(path string, state State) error {
|
||||
if state.Phase == PhasePreflight && state.MutationStarted {
|
||||
fake.tags[fake.configuredImage] = "sha256:moved-configured-tag"
|
||||
fake.imageVersions["sha256:moved-configured-tag"] = "9.99.0"
|
||||
}
|
||||
return write(path, state)
|
||||
}
|
||||
restartStatePath := filepath.Join(dir, "restart-state.json")
|
||||
|
||||
result, err := restartWithHooks(context.Background(), fake, RestartRequest{
|
||||
StatePath: restartStatePath,
|
||||
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
||||
Confirm: true,
|
||||
}, hooks)
|
||||
if err != nil {
|
||||
t.Fatalf("restartWithHooks() error = %v", err)
|
||||
}
|
||||
if result.Version != "0.80.3" || fake.currentImage != "sha256:old" {
|
||||
t.Fatalf("restart result=%+v image=%q; want captured 0.80.3 / sha256:old", result, fake.currentImage)
|
||||
}
|
||||
assertCalled(t, fake.calls, "image tag sha256:old thothii-core:tht-")
|
||||
assertCalled(t, fake.calls, "pi-lifecycle-")
|
||||
if matches, globErr := filepath.Glob(filepath.Join(dir, "pi-lifecycle-*.yaml")); globErr != nil || len(matches) != 0 {
|
||||
t.Fatalf("successful restart overrides = %v, error = %v; want safe cleanup", matches, globErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestartRefusesActiveSessionsWithoutDrain(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
fake.activeSessions = true
|
||||
|
||||
_, err := Restart(context.Background(), fake, RestartRequest{
|
||||
StatePath: filepath.Join(dir, "restart-state.json"),
|
||||
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
||||
Confirm: true,
|
||||
})
|
||||
if !errors.Is(err, ErrActiveSessions) {
|
||||
t.Fatalf("Restart() error = %v, want ErrActiveSessions", err)
|
||||
}
|
||||
if got, want := err.Error(), "active sessions must be drained before restarting Pi; use --drain only after they are complete"; got != want {
|
||||
t.Fatalf("Restart() error text = %q, want %q", got, want)
|
||||
}
|
||||
if fake.maintenance {
|
||||
t.Fatal("maintenance remained active after refusing pre-mutation restart")
|
||||
}
|
||||
if fake.recreated {
|
||||
t.Fatal("core was recreated with active sessions")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestartActivationFailureClearsPreMutationMaintenance(t *testing.T) {
|
||||
for _, failure := range []string{
|
||||
"maintenance-activate-durability",
|
||||
"maintenance-activate-durability-without-status-flag",
|
||||
} {
|
||||
t.Run(failure, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
fake.fail = failure
|
||||
|
||||
_, err := Restart(context.Background(), fake, RestartRequest{
|
||||
StatePath: filepath.Join(dir, "restart-state.json"),
|
||||
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
||||
Confirm: true,
|
||||
})
|
||||
var recovery *RecoveryRequiredError
|
||||
if !errors.As(err, &recovery) {
|
||||
t.Fatalf("Restart() error = %v, want RecoveryRequiredError", err)
|
||||
}
|
||||
if fake.maintenance {
|
||||
t.Fatal("pre-mutation activation failure left maintenance active")
|
||||
}
|
||||
if fake.recreated {
|
||||
t.Fatal("pre-mutation activation failure recreated core")
|
||||
}
|
||||
assertCalled(t, fake.calls, "/internal/maintenance/activate")
|
||||
assertCalled(t, fake.calls, "/internal/maintenance/deactivate")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestartRefusesInterruptedUpdateOrRestartState(t *testing.T) {
|
||||
for _, stateFile := range []string{"update-state.json", "restart-state.json"} {
|
||||
t.Run(stateFile, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
previous := stateImageForTest(t, fake)
|
||||
writeStateForTest(t, filepath.Join(dir, stateFile), State{
|
||||
Transaction: "interrupted",
|
||||
Phase: PhaseRecreated,
|
||||
Target: Target{Version: fake.version, Source: "restart"},
|
||||
Previous: previous,
|
||||
MutationStarted: true,
|
||||
})
|
||||
fake.calls = nil
|
||||
|
||||
_, err := Restart(context.Background(), fake, RestartRequest{
|
||||
StatePath: filepath.Join(dir, "restart-state.json"),
|
||||
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
||||
Confirm: true,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Restart() accepted interrupted lifecycle state")
|
||||
}
|
||||
assertNotCalled(t, fake.calls, "compose")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestartPreflightFailureNeverRecreatesCoreAndClearsMaintenance(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
fake.fail = "preflight"
|
||||
|
||||
_, err := Restart(context.Background(), fake, RestartRequest{
|
||||
StatePath: filepath.Join(dir, "restart-state.json"),
|
||||
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
||||
Confirm: true,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Restart() error = nil, want preflight failure")
|
||||
}
|
||||
if fake.recreated || fake.currentImage != "sha256:old" {
|
||||
t.Fatalf("preflight failure mutated core: recreated=%t image=%q", fake.recreated, fake.currentImage)
|
||||
}
|
||||
if fake.maintenance {
|
||||
t.Fatal("maintenance remained active after preflight failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestartPostRecreateFailureKeepsMaintenanceAndRecoveryState(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
fake.fail = "health"
|
||||
statePath := filepath.Join(dir, "restart-state.json")
|
||||
|
||||
_, err := Restart(context.Background(), fake, RestartRequest{
|
||||
StatePath: statePath,
|
||||
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
||||
Confirm: true,
|
||||
})
|
||||
var recovery *RecoveryRequiredError
|
||||
if !errors.As(err, &recovery) {
|
||||
t.Fatalf("Restart() error = %v, want RecoveryRequiredError", err)
|
||||
}
|
||||
if !fake.recreated {
|
||||
t.Fatal("post-recreate failure occurred before core recreation")
|
||||
}
|
||||
if !fake.maintenance {
|
||||
t.Fatal("maintenance was cleared after post-recreate failure")
|
||||
}
|
||||
state, stateErr := readState(statePath)
|
||||
if stateErr != nil || !state.MutationStarted {
|
||||
t.Fatalf("restart recovery state = %+v, %v; want durable mutation state", state, stateErr)
|
||||
}
|
||||
overridePath := lifecycleOverridePath(statePath, state.Transaction)
|
||||
selected, overrideErr := readLifecycleOverride(overridePath)
|
||||
if overrideErr != nil || selected != state.Previous.Reference || fake.tags[selected] != state.Previous.ID {
|
||||
t.Fatalf("restart override = %q, %v; want retained exact image %q", selected, overrideErr, state.Previous.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestartMaintenanceClearFailureRestoresRecoveryState(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
fake.fail = "maintenance-clear"
|
||||
statePath := filepath.Join(dir, "restart-state.json")
|
||||
|
||||
_, err := Restart(context.Background(), fake, RestartRequest{
|
||||
StatePath: statePath,
|
||||
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
||||
Confirm: true,
|
||||
})
|
||||
var recovery *RecoveryRequiredError
|
||||
if !errors.As(err, &recovery) {
|
||||
t.Fatalf("Restart() error = %v, want RecoveryRequiredError", err)
|
||||
}
|
||||
if !fake.maintenance {
|
||||
t.Fatal("maintenance was cleared despite deactivation failure")
|
||||
}
|
||||
state, stateErr := readState(statePath)
|
||||
if stateErr != nil || state.Phase != PhaseVerified || !state.MutationStarted {
|
||||
t.Fatalf("restart recovery state = %+v, %v; want durable verified mutation state", state, stateErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoverLifecycleMaintenanceVerifiesAndClearsRestartState(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
fake.maintenance = true
|
||||
previous := stateImageForTest(t, fake)
|
||||
restartStatePath := filepath.Join(dir, "restart-state.json")
|
||||
updateStatePath := filepath.Join(dir, "update-state.json")
|
||||
writeStateForTest(t, restartStatePath, State{
|
||||
Transaction: "restart-recovery",
|
||||
Phase: PhaseRecreated,
|
||||
Target: Target{Version: fake.version, Source: "restart"},
|
||||
Previous: previous,
|
||||
MutationStarted: true,
|
||||
})
|
||||
restartState, err := readState(restartStatePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
restartOverride := lifecycleOverridePath(restartStatePath, restartState.Transaction)
|
||||
if err := writeLifecycleOverride(restartOverride, restartState.Previous.Reference); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
delete(fake.tags, restartState.Previous.Reference)
|
||||
fake.tags[fake.configuredImage] = "sha256:moved-before-recovery"
|
||||
candidate := previous
|
||||
candidate.Reference = "thothii-core:tht-recover-candidate"
|
||||
writeStateForTest(t, updateStatePath, State{
|
||||
Transaction: "update-recovery",
|
||||
Phase: PhasePromoting,
|
||||
Target: Target{Version: fake.version, Source: string(BuildSource)},
|
||||
Previous: previous,
|
||||
Candidate: candidate,
|
||||
MutationStarted: true,
|
||||
})
|
||||
if err := writeLifecycleOverride(currentImageOverridePath(updateStatePath), candidate.Reference); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fake.calls = nil
|
||||
|
||||
if err := RecoverLifecycleMaintenance(context.Background(), fake, updateStatePath, restartStatePath, true); err != nil {
|
||||
t.Fatalf("RecoverLifecycleMaintenance() error = %v", err)
|
||||
}
|
||||
if _, err := os.Stat(restartStatePath); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("restart recovery state still exists: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(restartOverride); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("restart recovery override still exists: %v", err)
|
||||
}
|
||||
if fake.tags[restartState.Previous.Reference] != restartState.Previous.ID {
|
||||
t.Fatalf("restart recovery pin = %q, want %q", fake.tags[restartState.Previous.Reference], restartState.Previous.ID)
|
||||
}
|
||||
assertCalled(t, fake.calls, restartOverride)
|
||||
if fake.maintenance {
|
||||
t.Fatal("maintenance remained active after both lifecycle states were verified")
|
||||
}
|
||||
updateState, err := readState(updateStatePath)
|
||||
if err != nil || updateState.Phase != PhaseVerified {
|
||||
t.Fatalf("update recovery state = %+v, %v; want verified image rollback metadata", updateState, err)
|
||||
}
|
||||
deactivate := callIndex(fake.calls, "/internal/maintenance/deactivate")
|
||||
lastVerification := lastCallIndexBefore(fake.calls, "/pi-management/test", deactivate)
|
||||
if deactivate < 0 || lastVerification < 0 {
|
||||
t.Fatalf("calls = %v; want verification before maintenance deactivation", fake.calls)
|
||||
}
|
||||
verificationCount := 0
|
||||
for index := 0; index < deactivate; index++ {
|
||||
if strings.Contains(fake.calls[index], "/internal/maintenance/deactivate") {
|
||||
t.Fatalf("maintenance reopened before combined verification: %v", fake.calls)
|
||||
}
|
||||
if strings.Contains(fake.calls[index], "/pi-management/test") {
|
||||
verificationCount++
|
||||
}
|
||||
}
|
||||
if verificationCount < 3 {
|
||||
t.Fatalf("verification calls before maintenance deactivation = %d, want restart, update, and final proofs: %v", verificationCount, fake.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoverLifecycleMaintenanceRejectsMalformedRestartState(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
phase Phase
|
||||
source string
|
||||
}{
|
||||
{name: "recreated_without_mutation_marker", phase: PhaseRecreated, source: "restart"},
|
||||
{name: "non_restart_source", phase: PhasePreflight, source: string(BuildSource)},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
fake.maintenance = true
|
||||
restartStatePath := filepath.Join(dir, "restart-state.json")
|
||||
updateStatePath := filepath.Join(dir, "update-state.json")
|
||||
writeStateForTest(t, restartStatePath, State{
|
||||
Transaction: "malformed-restart",
|
||||
Phase: test.phase,
|
||||
Target: Target{Version: fake.version, Source: test.source},
|
||||
Previous: stateImageForTest(t, fake),
|
||||
})
|
||||
fake.calls = nil
|
||||
|
||||
err := RecoverLifecycleMaintenance(
|
||||
context.Background(), fake, updateStatePath, restartStatePath, true,
|
||||
)
|
||||
if !errors.Is(err, ErrInvalidRequest) || !strings.Contains(err.Error(), "restart recovery state is invalid") {
|
||||
t.Fatalf("RecoverLifecycleMaintenance() error = %v, want invalid restart recovery state", err)
|
||||
}
|
||||
if !fake.maintenance {
|
||||
t.Fatal("malformed restart state reopened admission")
|
||||
}
|
||||
if _, stateErr := os.Stat(restartStatePath); stateErr != nil {
|
||||
t.Fatalf("malformed restart state was removed: %v", stateErr)
|
||||
}
|
||||
assertNotCalled(t, fake.calls, "/internal/maintenance/deactivate")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestartRefusesMalformedRestartStateWithoutInvokingCompose(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
restartStatePath := filepath.Join(dir, "restart-state.json")
|
||||
writeStateForTest(t, restartStatePath, State{
|
||||
Transaction: "malformed-restart",
|
||||
Phase: PhasePreflight,
|
||||
Target: Target{Version: fake.version, Source: string(BuildSource)},
|
||||
Previous: stateImageForTest(t, fake),
|
||||
})
|
||||
fake.calls = nil
|
||||
|
||||
_, err := Restart(context.Background(), fake, RestartRequest{
|
||||
StatePath: restartStatePath,
|
||||
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
||||
Confirm: true,
|
||||
})
|
||||
if !errors.Is(err, ErrInvalidRequest) || !strings.Contains(err.Error(), "restart recovery state is invalid") {
|
||||
t.Fatalf("Restart() error = %v, want invalid restart recovery state", err)
|
||||
}
|
||||
if _, stateErr := os.Stat(restartStatePath); stateErr != nil {
|
||||
t.Fatalf("malformed restart state was removed: %v", stateErr)
|
||||
}
|
||||
assertNotCalled(t, fake.calls, "compose")
|
||||
}
|
||||
|
||||
func TestRestartDurabilityFailureBoundaries(t *testing.T) {
|
||||
injected := errors.New("injected restart durability failure")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
configure func(*fakeRunner, *lifecycleHooks)
|
||||
wantPhase Phase
|
||||
wantMutation bool
|
||||
wantRecreated bool
|
||||
wantMaintenance bool
|
||||
wantRecovery bool
|
||||
wantInjected bool
|
||||
}{
|
||||
{
|
||||
name: "mutation_marker_write",
|
||||
configure: func(_ *fakeRunner, hooks *lifecycleHooks) {
|
||||
write := hooks.writeState
|
||||
hooks.writeState = func(path string, state State) error {
|
||||
if state.Phase == PhasePreflight && state.MutationStarted {
|
||||
return injected
|
||||
}
|
||||
return write(path, state)
|
||||
}
|
||||
},
|
||||
wantPhase: PhasePreflight,
|
||||
wantInjected: true,
|
||||
},
|
||||
{
|
||||
name: "core_recreation",
|
||||
configure: func(fake *fakeRunner, _ *lifecycleHooks) {
|
||||
fake.fail = "recreate"
|
||||
},
|
||||
wantPhase: PhasePreflight,
|
||||
wantMutation: true,
|
||||
wantRecreated: true,
|
||||
wantMaintenance: true,
|
||||
wantRecovery: true,
|
||||
},
|
||||
{
|
||||
name: "maintenance_proof",
|
||||
configure: func(fake *fakeRunner, _ *lifecycleHooks) {
|
||||
fake.fail = "maintenance-proof"
|
||||
},
|
||||
wantPhase: PhasePreflight,
|
||||
wantMutation: true,
|
||||
wantRecreated: true,
|
||||
wantMaintenance: true,
|
||||
wantRecovery: true,
|
||||
},
|
||||
{
|
||||
name: "recreated_phase_write",
|
||||
configure: func(_ *fakeRunner, hooks *lifecycleHooks) {
|
||||
write := hooks.writeState
|
||||
hooks.writeState = func(path string, state State) error {
|
||||
if state.Phase == PhaseRecreated {
|
||||
return injected
|
||||
}
|
||||
return write(path, state)
|
||||
}
|
||||
},
|
||||
wantPhase: PhasePreflight,
|
||||
wantMutation: true,
|
||||
wantRecreated: true,
|
||||
wantMaintenance: true,
|
||||
wantRecovery: true,
|
||||
wantInjected: true,
|
||||
},
|
||||
{
|
||||
name: "verified_phase_write",
|
||||
configure: func(_ *fakeRunner, hooks *lifecycleHooks) {
|
||||
write := hooks.writeState
|
||||
hooks.writeState = func(path string, state State) error {
|
||||
if state.Phase == PhaseVerified {
|
||||
return injected
|
||||
}
|
||||
return write(path, state)
|
||||
}
|
||||
},
|
||||
wantPhase: PhaseRecreated,
|
||||
wantMutation: true,
|
||||
wantRecreated: true,
|
||||
wantMaintenance: true,
|
||||
wantRecovery: true,
|
||||
wantInjected: true,
|
||||
},
|
||||
{
|
||||
name: "restart_state_removal",
|
||||
configure: func(_ *fakeRunner, hooks *lifecycleHooks) {
|
||||
hooks.removeFile = func(string) error { return injected }
|
||||
},
|
||||
wantPhase: PhaseVerified,
|
||||
wantMutation: true,
|
||||
wantRecreated: true,
|
||||
wantMaintenance: true,
|
||||
wantRecovery: true,
|
||||
wantInjected: true,
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
hooks := defaultLifecycleHooks
|
||||
test.configure(fake, &hooks)
|
||||
statePath := filepath.Join(dir, "restart-state.json")
|
||||
|
||||
_, err := restartWithHooks(context.Background(), fake, RestartRequest{
|
||||
StatePath: statePath,
|
||||
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
||||
Confirm: true,
|
||||
}, hooks)
|
||||
if err == nil {
|
||||
t.Fatal("restartWithHooks() error = nil, want injected boundary failure")
|
||||
}
|
||||
var recovery *RecoveryRequiredError
|
||||
if got := errors.As(err, &recovery); got != test.wantRecovery {
|
||||
t.Fatalf("recovery-required = %t, want %t; error = %v", got, test.wantRecovery, err)
|
||||
}
|
||||
if test.wantInjected && !errors.Is(err, injected) {
|
||||
t.Fatalf("restartWithHooks() error = %v, want injected durability cause", err)
|
||||
}
|
||||
state, stateErr := readState(statePath)
|
||||
if stateErr != nil {
|
||||
t.Fatalf("readState() error = %v", stateErr)
|
||||
}
|
||||
if state.Phase != test.wantPhase || state.MutationStarted != test.wantMutation {
|
||||
t.Fatalf("state = %+v, want phase=%q mutation=%t", state, test.wantPhase, test.wantMutation)
|
||||
}
|
||||
if fake.recreated != test.wantRecreated {
|
||||
t.Fatalf("recreated = %t, want %t", fake.recreated, test.wantRecreated)
|
||||
}
|
||||
if fake.maintenance != test.wantMaintenance {
|
||||
t.Fatalf("maintenance = %t, want %t", fake.maintenance, test.wantMaintenance)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestartRejectsImageConfigurationAndMountDrift(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
failure string
|
||||
want error
|
||||
}{
|
||||
{failure: "image-drift", want: errRestartImageDrift},
|
||||
{failure: "config-drift", want: errRestartConfigurationDrift},
|
||||
{failure: "mount-drift", want: errRestartMountDrift},
|
||||
} {
|
||||
t.Run(test.failure, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
fake.fail = test.failure
|
||||
statePath := filepath.Join(dir, "restart-state.json")
|
||||
|
||||
_, err := Restart(context.Background(), fake, RestartRequest{
|
||||
StatePath: statePath,
|
||||
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
||||
Confirm: true,
|
||||
})
|
||||
if !errors.Is(err, test.want) {
|
||||
t.Fatalf("Restart() error = %v, want errors.Is(..., %v)", err, test.want)
|
||||
}
|
||||
if !fake.maintenance {
|
||||
t.Fatal("maintenance was cleared after restart identity drift")
|
||||
}
|
||||
if _, stateErr := os.Stat(statePath); stateErr != nil {
|
||||
t.Fatalf("restart recovery state missing after drift: %v", stateErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestartPreservesVerifiedUpdateState(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
updateStatePath := filepath.Join(dir, "update-state.json")
|
||||
writeStateForTest(t, updateStatePath, State{
|
||||
Transaction: "verified-update",
|
||||
Phase: PhaseVerified,
|
||||
Target: Target{Version: fake.version, Source: string(BuildSource)},
|
||||
Previous: stateImageForTest(t, fake),
|
||||
})
|
||||
before := readStateBytes(t, updateStatePath)
|
||||
|
||||
if _, err := Restart(context.Background(), fake, RestartRequest{
|
||||
StatePath: filepath.Join(dir, "restart-state.json"),
|
||||
UpdateStatePath: updateStatePath,
|
||||
Confirm: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after := readStateBytes(t, updateStatePath)
|
||||
if string(after) != string(before) {
|
||||
t.Fatal("restart changed verified update rollback metadata")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
// Package pi implements host-side lifecycle operations for the Pi bundled in core.
|
||||
package pi
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gofrs/flock"
|
||||
)
|
||||
|
||||
const stateFileVersion = 4
|
||||
|
||||
// Phase describes the durable point reached by a Pi update.
|
||||
type Phase string
|
||||
|
||||
const (
|
||||
PhasePreflight Phase = "preflight"
|
||||
PhaseBuilding Phase = "building"
|
||||
PhaseRecreated Phase = "recreated"
|
||||
PhasePromoting Phase = "promoting"
|
||||
PhaseVerified Phase = "verified"
|
||||
PhaseRolledBack Phase = "rolled_back"
|
||||
PhaseFailed Phase = "failed"
|
||||
PhaseNoop Phase = "noop"
|
||||
)
|
||||
|
||||
// Image is the non-secret recovery identity of a core image and its mounted volume names.
|
||||
type Image struct {
|
||||
ID string `json:"id"`
|
||||
Reference string `json:"reference"`
|
||||
Mounts []Mount `json:"mounts"`
|
||||
MountFingerprint string `json:"mount_fingerprint"`
|
||||
ConfigurationSHA string `json:"configuration_sha256,omitempty"`
|
||||
}
|
||||
|
||||
// Mount is the complete persistence identity relevant to safe core recreation.
|
||||
type Mount struct {
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name,omitempty"`
|
||||
SourceSHA256 string `json:"source_sha256"`
|
||||
SourceAliases []string `json:"-"`
|
||||
Destination string `json:"destination"`
|
||||
RW bool `json:"rw"`
|
||||
Options string `json:"options,omitempty"`
|
||||
}
|
||||
|
||||
// Target records the immutable input selected by the operator. Source is build, restart, or a
|
||||
// digest-pinned image reference; it intentionally never contains credentials.
|
||||
type Target struct {
|
||||
Version string `json:"version"`
|
||||
Source string `json:"source"`
|
||||
}
|
||||
|
||||
// State is recovery metadata stored below the installation project. It never stores environment
|
||||
// values, secret paths, credentials, or command output.
|
||||
type State struct {
|
||||
Version int `json:"version"`
|
||||
Transaction string `json:"transaction"`
|
||||
Phase Phase `json:"phase"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
Target Target `json:"target,omitempty"`
|
||||
Previous Image `json:"previous"`
|
||||
Candidate Image `json:"candidate,omitempty"`
|
||||
MutationStarted bool `json:"mutation_started,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func readState(path string) (State, error) {
|
||||
contents, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return State{}, err
|
||||
}
|
||||
var state State
|
||||
if err := json.Unmarshal(contents, &state); err != nil {
|
||||
return State{}, errors.New("update recovery state is invalid")
|
||||
}
|
||||
if state.Version != stateFileVersion || state.Previous.ID == "" || state.Previous.Reference == "" || state.Previous.MountFingerprint == "" {
|
||||
return State{}, errors.New("update recovery state is incomplete")
|
||||
}
|
||||
if mountFingerprint(state.Previous.Mounts) != state.Previous.MountFingerprint || (state.Candidate.ID != "" && mountFingerprint(state.Candidate.Mounts) != state.Candidate.MountFingerprint) {
|
||||
return State{}, errors.New("update recovery state mount fingerprint is invalid")
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func writeState(path string, state State) error {
|
||||
if state.Previous.ID == "" || state.Previous.Reference == "" || state.Previous.MountFingerprint == "" {
|
||||
return errors.New("refusing to write incomplete update recovery state")
|
||||
}
|
||||
state.Version = stateFileVersion
|
||||
state.UpdatedAt = time.Now().UTC()
|
||||
contents, err := json.MarshalIndent(state, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode update recovery state: %w", err)
|
||||
}
|
||||
contents = append(contents, '\n')
|
||||
if err := writeFileDurably(path, ".update-state-", contents); err != nil {
|
||||
return fmt.Errorf("could not durably write update recovery state: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeFileDurably(path, prefix string, contents []byte) error {
|
||||
directory := filepath.Dir(path)
|
||||
if err := os.MkdirAll(directory, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
temporary, err := os.CreateTemp(directory, prefix+"*.tmp")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
temporaryName := temporary.Name()
|
||||
defer os.Remove(temporaryName)
|
||||
if err := temporary.Chmod(0o600); err != nil {
|
||||
temporary.Close()
|
||||
return err
|
||||
}
|
||||
if _, err := temporary.Write(contents); err != nil {
|
||||
temporary.Close()
|
||||
return err
|
||||
}
|
||||
if err := temporary.Sync(); err != nil {
|
||||
temporary.Close()
|
||||
return err
|
||||
}
|
||||
if err := temporary.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return durableReplace(temporaryName, path, directory)
|
||||
}
|
||||
|
||||
func mountSourceHash(source string) string {
|
||||
sum := sha256.Sum256([]byte(source))
|
||||
return fmt.Sprintf("%x", sum[:])
|
||||
}
|
||||
|
||||
func mountSourceAliases(mountType, source, goos string) []string {
|
||||
if mountType != "bind" || goos != "darwin" {
|
||||
return nil
|
||||
}
|
||||
source = filepath.Clean(source)
|
||||
var alias string
|
||||
switch {
|
||||
case strings.HasPrefix(source, "/host_mnt/private/var/"), strings.HasPrefix(source, "/host_mnt/Users/"):
|
||||
alias = strings.TrimPrefix(source, "/host_mnt")
|
||||
case strings.HasPrefix(source, "/private/var/"), strings.HasPrefix(source, "/Users/"):
|
||||
alias = "/host_mnt" + source
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
return []string{mountSourceHash(alias)}
|
||||
}
|
||||
|
||||
func mountFingerprint(mounts []Mount) string {
|
||||
values := make([]string, len(mounts))
|
||||
for i, mount := range mounts {
|
||||
values[i] = strings.Join([]string{mount.Type, mount.Name, mount.SourceSHA256, mount.Destination, fmt.Sprint(mount.RW), mount.Options}, "\x00")
|
||||
}
|
||||
sort.Strings(values)
|
||||
sum := sha256.Sum256([]byte(strings.Join(values, "\n")))
|
||||
return fmt.Sprintf("%x", sum[:])
|
||||
}
|
||||
|
||||
type lockOwner struct {
|
||||
PID int `json:"pid"`
|
||||
Host string `json:"host"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
Transaction string `json:"transaction"`
|
||||
}
|
||||
|
||||
type updateLock struct {
|
||||
file *flock.Flock
|
||||
metadata string
|
||||
}
|
||||
|
||||
var ErrLockHeld = errors.New("another Pi update, restart, or rollback is already in progress")
|
||||
|
||||
func lifecycleLockPath(statePath string) string {
|
||||
return filepath.Join(filepath.Dir(statePath), "pi-lifecycle.lock")
|
||||
}
|
||||
|
||||
func acquireLock(statePath string) (*updateLock, error) {
|
||||
if err := os.MkdirAll(filepath.Dir(statePath), 0o700); err != nil {
|
||||
return nil, errors.New("could not create Pi update recovery directory")
|
||||
}
|
||||
path := lifecycleLockPath(statePath)
|
||||
file := flock.New(path, flock.SetPermissions(0o600))
|
||||
locked, err := file.TryLock()
|
||||
if err != nil {
|
||||
return nil, errors.New("could not acquire Pi update lock")
|
||||
}
|
||||
if !locked {
|
||||
return nil, ErrLockHeld
|
||||
}
|
||||
host, err := os.Hostname()
|
||||
if err != nil {
|
||||
_ = file.Unlock()
|
||||
return nil, errors.New("could not identify Pi update lock owner")
|
||||
}
|
||||
owner := lockOwner{PID: os.Getpid(), Host: host, StartedAt: time.Now().UTC(), Transaction: fmt.Sprintf("%d-%d", os.Getpid(), time.Now().UnixNano())}
|
||||
contents, err := json.Marshal(owner)
|
||||
if err != nil {
|
||||
_ = file.Unlock()
|
||||
return nil, errors.New("could not record Pi update lock owner")
|
||||
}
|
||||
metadata := path + ".owner.json"
|
||||
if err := writeFileDurably(metadata, ".lock-owner-", append(contents, '\n')); err != nil {
|
||||
_ = file.Unlock()
|
||||
return nil, errors.New("could not record Pi update lock owner")
|
||||
}
|
||||
return &updateLock{file: file, metadata: metadata}, nil
|
||||
}
|
||||
func (l *updateLock) Release() {
|
||||
_ = durableRemove(l.metadata)
|
||||
_ = l.file.Unlock()
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package pi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAdvisoryLockRejectsAConcurrentOwner(t *testing.T) {
|
||||
statePath := filepath.Join(t.TempDir(), "update-state.json")
|
||||
first, err := acquireLock(statePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer first.Release()
|
||||
second, err := acquireLock(statePath)
|
||||
if second != nil {
|
||||
second.Release()
|
||||
}
|
||||
if !errors.Is(err, ErrLockHeld) {
|
||||
t.Fatalf("second acquireLock() error = %v, want ErrLockHeld", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateAndRestartStatePathsShareOneLifecycleLock(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
first, err := acquireLock(filepath.Join(dir, "update-state.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer first.Release()
|
||||
|
||||
second, err := acquireLock(filepath.Join(dir, "restart-state.json"))
|
||||
if !errors.Is(err, ErrLockHeld) || second != nil {
|
||||
t.Fatalf("second lock = %#v, %v; want nil, ErrLockHeld", second, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdvisoryLockCrashReleasesAndReacquires(t *testing.T) {
|
||||
statePath := filepath.Join(t.TempDir(), "update-state.json")
|
||||
if os.Getenv("THT_LOCK_CRASH_HELPER") == "1" {
|
||||
lock, err := acquireLock(os.Getenv("THT_LOCK_STATE"))
|
||||
if err != nil || lock == nil {
|
||||
os.Exit(23)
|
||||
}
|
||||
os.Exit(0) // Deliberately bypass Release: the OS must release ownership.
|
||||
}
|
||||
command := exec.Command(os.Args[0], "-test.run=^TestAdvisoryLockCrashReleasesAndReacquires$")
|
||||
command.Env = append(os.Environ(), "THT_LOCK_CRASH_HELPER=1", "THT_LOCK_STATE="+statePath)
|
||||
if output, err := command.CombinedOutput(); err != nil {
|
||||
t.Fatalf("crash helper failed: %v: %s", err, output)
|
||||
}
|
||||
lock, err := acquireLock(statePath)
|
||||
if err != nil {
|
||||
t.Fatalf("acquireLock() after owner crash = %v", err)
|
||||
}
|
||||
lock.Release()
|
||||
}
|
||||
|
||||
func TestAdvisoryLockIgnoresPartialDiagnosticMetadata(t *testing.T) {
|
||||
statePath := filepath.Join(t.TempDir(), "update-state.json")
|
||||
lockPath := lifecycleLockPath(statePath)
|
||||
if err := os.WriteFile(lockPath, nil, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(lockPath+".owner.json", []byte("{partial"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lock, err := acquireLock(statePath)
|
||||
if err != nil {
|
||||
t.Fatalf("acquireLock() with partial diagnostics = %v", err)
|
||||
}
|
||||
lock.Release()
|
||||
}
|
||||
@@ -0,0 +1,967 @@
|
||||
package pi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/compose"
|
||||
"github.com/distribution/reference"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrConfirmationRequired = errors.New("update requires --yes after reviewing the planned Pi version")
|
||||
ErrActiveSessions = errors.New("active sessions must be drained before updating Pi; use --drain only after they are complete")
|
||||
ErrInterruptedUpdate = errors.New("a previous Pi update is incomplete; run pi rollback --yes before starting another update")
|
||||
ErrInvalidRequest = errors.New("invalid Pi lifecycle request")
|
||||
versionPattern = regexp.MustCompile(`^[0-9]+(?:\.[0-9]+){1,3}(?:[-+][0-9A-Za-z.-]+)?$`)
|
||||
)
|
||||
|
||||
// Source chooses whether the candidate is built from this checkout or pulled from an immutable image.
|
||||
type Source string
|
||||
|
||||
const (
|
||||
BuildSource Source = "build"
|
||||
PullSource Source = "pull"
|
||||
)
|
||||
|
||||
// Request contains only non-secret operator inputs.
|
||||
type Request struct {
|
||||
StatePath string
|
||||
RestartStatePath string
|
||||
Version string
|
||||
Source Source
|
||||
Image string
|
||||
Confirm bool
|
||||
Drain bool
|
||||
}
|
||||
|
||||
// Result summarizes the completed, failed, or recovered transaction without command output.
|
||||
type Result struct {
|
||||
Phase Phase
|
||||
StatePath string
|
||||
}
|
||||
|
||||
type lifecycleHooks struct {
|
||||
writeState func(string, State) error
|
||||
removeFile func(string) error
|
||||
sleep func(time.Duration)
|
||||
}
|
||||
|
||||
var defaultLifecycleHooks = lifecycleHooks{
|
||||
writeState: writeState,
|
||||
removeFile: durableRemove,
|
||||
sleep: time.Sleep,
|
||||
}
|
||||
|
||||
// Update performs a recoverable core-only Pi update using the default Compose command layout.
|
||||
func Update(ctx context.Context, runner Runner, request Request) (result Result, retErr error) {
|
||||
return updateWithHooks(ctx, runner, request, defaultLifecycleHooks)
|
||||
}
|
||||
|
||||
func updateWithHooks(ctx context.Context, runner Runner, request Request, hooks lifecycleHooks) (result Result, retErr error) {
|
||||
if request.StatePath == "" {
|
||||
return Result{}, errors.New("update state path is required")
|
||||
}
|
||||
if request.RestartStatePath == "" {
|
||||
request.RestartStatePath = pairedRestartStatePath(request.StatePath)
|
||||
}
|
||||
if err := validateRestartStatePaths(request.RestartStatePath, request.StatePath); err != nil {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
}
|
||||
lock, err := acquireLock(request.StatePath)
|
||||
if err != nil {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
}
|
||||
defer lock.Release()
|
||||
if !request.Confirm {
|
||||
return Result{StatePath: request.StatePath}, ErrConfirmationRequired
|
||||
}
|
||||
if !versionPattern.MatchString(request.Version) {
|
||||
return Result{StatePath: request.StatePath}, fmt.Errorf("%w: Pi version must be an explicit pinned version", ErrInvalidRequest)
|
||||
}
|
||||
if request.Source == "" {
|
||||
return Result{StatePath: request.StatePath}, fmt.Errorf("%w: Pi update requires an explicit source: build or pull", ErrInvalidRequest)
|
||||
}
|
||||
if request.Source != BuildSource && request.Source != PullSource {
|
||||
return Result{StatePath: request.StatePath}, fmt.Errorf("%w: Pi update source must be build or pull", ErrInvalidRequest)
|
||||
}
|
||||
if request.Source == PullSource {
|
||||
canonical, err := canonicalDigestReference(request.Image)
|
||||
if err != nil {
|
||||
return Result{StatePath: request.StatePath}, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
request.Image = canonical
|
||||
}
|
||||
if err := prepareLifecycleMutation(request.RestartStatePath, hooks.removeFile); err != nil {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
}
|
||||
if old, err := readState(request.StatePath); err == nil && stateNeedsRecovery(old) {
|
||||
return Result{StatePath: request.StatePath}, ErrInterruptedUpdate
|
||||
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
} else if err == nil && !old.MutationStarted {
|
||||
if cleanupErr := hooks.removeFile(lifecycleOverridePath(request.StatePath, old.Transaction)); cleanupErr != nil {
|
||||
return Result{StatePath: request.StatePath}, errors.New("safe prior preparation state could not be cleaned up")
|
||||
}
|
||||
}
|
||||
if err := setMaintenance(ctx, runner, true); err != nil {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
}
|
||||
clearMaintenance := true
|
||||
defer func() {
|
||||
if !clearMaintenance {
|
||||
return
|
||||
}
|
||||
if clearErr := setMaintenance(context.Background(), runner, false); clearErr != nil {
|
||||
result = Result{Phase: PhaseFailed, StatePath: request.StatePath}
|
||||
retErr = errors.Join(retErr, fmt.Errorf("maintenance admission gate could not be cleared: %w", clearErr))
|
||||
}
|
||||
}()
|
||||
|
||||
if err := waitForInactiveSessions(ctx, runner, request.Drain, hooks.sleep); err != nil {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
}
|
||||
if err := Doctor(ctx, runner); err != nil {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
}
|
||||
|
||||
currentVersion, err := Status(ctx, runner)
|
||||
if err != nil {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
}
|
||||
if currentVersion == request.Version {
|
||||
return Result{Phase: PhaseNoop, StatePath: request.StatePath}, nil
|
||||
}
|
||||
configured, err := renderedCore(ctx, runner)
|
||||
if err != nil {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
}
|
||||
previous, err := runningImage(ctx, runner, configured.Reference)
|
||||
if err != nil {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
}
|
||||
previous.ConfigurationSHA = configured.ConfigurationSHA
|
||||
transaction := lifecycleTransaction(request.StatePath)
|
||||
previous.Reference = lifecycleImageTag(transaction, "previous")
|
||||
candidateReference := lifecycleImageTag(transaction, "candidate")
|
||||
if err := tagImage(ctx, runner, previous.ID, previous.Reference, "previous Pi image pin"); err != nil {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
}
|
||||
state := State{
|
||||
Transaction: transaction,
|
||||
Phase: PhasePreflight,
|
||||
Target: Target{Version: request.Version, Source: sourceValue(request)},
|
||||
Previous: previous,
|
||||
Candidate: Image{Reference: candidateReference},
|
||||
}
|
||||
if err := hooks.writeState(request.StatePath, state); err != nil {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
}
|
||||
overridePath := lifecycleOverridePath(request.StatePath, transaction)
|
||||
if err := writeLifecycleOverride(overridePath, candidateReference); err != nil {
|
||||
return Result{StatePath: request.StatePath}, err
|
||||
}
|
||||
lifecycle := composeOverrideRunner{Runner: runner, path: overridePath}
|
||||
|
||||
state.Phase = PhaseBuilding
|
||||
if err := hooks.writeState(request.StatePath, state); err != nil {
|
||||
result, retErr, clearMaintenance = failPreparation(request.StatePath, overridePath, state, err, hooks)
|
||||
return result, retErr
|
||||
}
|
||||
if err := prepareCandidate(ctx, lifecycle, request, candidateReference); err != nil {
|
||||
result, retErr, clearMaintenance = failPreparation(request.StatePath, overridePath, state, err, hooks)
|
||||
return result, retErr
|
||||
}
|
||||
running, err := activeSessions(ctx, runner)
|
||||
if err != nil {
|
||||
result, retErr, clearMaintenance = failPreparation(request.StatePath, overridePath, state, err, hooks)
|
||||
return result, retErr
|
||||
}
|
||||
if running {
|
||||
result, retErr, clearMaintenance = failPreparation(request.StatePath, overridePath, state, ErrActiveSessions, hooks)
|
||||
return result, retErr
|
||||
}
|
||||
state.MutationStarted = true
|
||||
if err := hooks.writeState(request.StatePath, state); err != nil {
|
||||
state.MutationStarted = false
|
||||
result, retErr, clearMaintenance = failPreparation(request.StatePath, overridePath, state, err, hooks)
|
||||
return result, retErr
|
||||
}
|
||||
clearMaintenance = false
|
||||
if err := recreateCore(ctx, lifecycle); err != nil {
|
||||
result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
|
||||
return result, retErr
|
||||
}
|
||||
if err := ensureMaintenance(ctx, lifecycle); err != nil {
|
||||
result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
|
||||
return result, retErr
|
||||
}
|
||||
state.Phase = PhaseRecreated
|
||||
state.Candidate, err = runningImage(ctx, lifecycle, candidateReference)
|
||||
if err != nil {
|
||||
result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
|
||||
return result, retErr
|
||||
}
|
||||
if err := hooks.writeState(request.StatePath, state); err != nil {
|
||||
result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
|
||||
return result, retErr
|
||||
}
|
||||
if err := verifyCandidate(ctx, lifecycle, request.Version, previous); err != nil {
|
||||
result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
|
||||
return result, retErr
|
||||
}
|
||||
state.Phase, state.Error = PhasePromoting, ""
|
||||
if err := hooks.writeState(request.StatePath, state); err != nil {
|
||||
result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
|
||||
return result, retErr
|
||||
}
|
||||
if err := promoteLifecycleOverride(overridePath, currentImageOverridePath(request.StatePath), candidateReference); err != nil {
|
||||
result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
|
||||
return result, retErr
|
||||
}
|
||||
state.Phase = PhaseVerified
|
||||
if err := hooks.writeState(request.StatePath, state); err != nil {
|
||||
result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
|
||||
return result, retErr
|
||||
}
|
||||
clearMaintenance = true
|
||||
return Result{Phase: PhaseVerified, StatePath: request.StatePath}, nil
|
||||
}
|
||||
|
||||
func stateNeedsRecovery(state State) bool {
|
||||
switch state.Phase {
|
||||
case PhaseVerified, PhaseRolledBack, PhaseNoop:
|
||||
return false
|
||||
case PhaseFailed:
|
||||
return state.MutationStarted
|
||||
default:
|
||||
return state.MutationStarted
|
||||
}
|
||||
}
|
||||
|
||||
func failPreparation(statePath, overridePath string, state State, cause error, hooks lifecycleHooks) (Result, error, bool) {
|
||||
state.Phase = PhaseFailed
|
||||
state.MutationStarted = false
|
||||
state.Error = "candidate preparation failed before core mutation"
|
||||
writeErr := hooks.writeState(statePath, state)
|
||||
removeErr := hooks.removeFile(overridePath)
|
||||
message := "candidate preparation failed before core mutation"
|
||||
if writeErr != nil || removeErr != nil {
|
||||
message += "; safe preparation cleanup was incomplete"
|
||||
}
|
||||
return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("%s: %w", message, cause), true
|
||||
}
|
||||
|
||||
// Rollback restores the image recorded in durable update state. It is safe for interrupted runs.
|
||||
func Rollback(ctx context.Context, runner Runner, statePath, restartStatePath string, confirm bool) (result Result, retErr error) {
|
||||
return rollbackWithHooks(ctx, runner, statePath, restartStatePath, confirm, defaultLifecycleHooks)
|
||||
}
|
||||
|
||||
func rollbackWithHooks(ctx context.Context, runner Runner, statePath, restartStatePath string, confirm bool, hooks lifecycleHooks) (result Result, retErr error) {
|
||||
if restartStatePath == "" {
|
||||
restartStatePath = pairedRestartStatePath(statePath)
|
||||
}
|
||||
if err := validateRestartStatePaths(restartStatePath, statePath); err != nil {
|
||||
return Result{StatePath: statePath}, err
|
||||
}
|
||||
lock, err := acquireLock(statePath)
|
||||
if err != nil {
|
||||
return Result{StatePath: statePath}, err
|
||||
}
|
||||
defer lock.Release()
|
||||
if !confirm {
|
||||
return Result{StatePath: statePath}, ErrConfirmationRequired
|
||||
}
|
||||
if err := prepareLifecycleMutation(restartStatePath, hooks.removeFile); err != nil {
|
||||
return Result{StatePath: statePath}, err
|
||||
}
|
||||
maintenanceErr := ensureMaintenance(ctx, runner)
|
||||
clearMaintenance := maintenanceErr == nil
|
||||
defer func() {
|
||||
if !clearMaintenance {
|
||||
return
|
||||
}
|
||||
if clearErr := setMaintenance(context.Background(), runner, false); clearErr != nil {
|
||||
result = Result{Phase: PhaseFailed, StatePath: statePath}
|
||||
retErr = errors.Join(retErr, fmt.Errorf("maintenance admission gate could not be cleared: %w", clearErr))
|
||||
}
|
||||
}()
|
||||
if maintenanceErr == nil {
|
||||
if active, err := activeSessions(ctx, runner); err != nil {
|
||||
return Result{StatePath: statePath}, err
|
||||
} else if active {
|
||||
return Result{StatePath: statePath}, ErrActiveSessions
|
||||
}
|
||||
}
|
||||
state, err := readState(statePath)
|
||||
if err != nil {
|
||||
if maintenanceErr == nil {
|
||||
clearMaintenance = false
|
||||
}
|
||||
return Result{StatePath: statePath}, err
|
||||
}
|
||||
overridePath := lifecycleOverridePath(statePath, state.Transaction)
|
||||
if err := writeLifecycleOverride(overridePath, state.Previous.Reference); err != nil {
|
||||
clearMaintenance = false
|
||||
return Result{StatePath: statePath}, err
|
||||
}
|
||||
lifecycle := composeOverrideRunner{Runner: runner, path: overridePath}
|
||||
if maintenanceErr != nil {
|
||||
stopped, stopErr := coreIsStopped(ctx, runner)
|
||||
if stopErr != nil || !stopped {
|
||||
return Result{StatePath: statePath}, maintenanceErr
|
||||
}
|
||||
if err := persistMaintenanceWithoutLiveCore(ctx, lifecycle); err != nil {
|
||||
return Result{Phase: PhaseFailed, StatePath: statePath}, err
|
||||
}
|
||||
}
|
||||
clearMaintenance = false
|
||||
if err := restore(ctx, lifecycle, state.Previous); err != nil {
|
||||
state.Phase, state.Error = PhaseFailed, "rollback failed"
|
||||
if writeErr := hooks.writeState(statePath, state); writeErr != nil {
|
||||
return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("rollback failed and recovery state could not be persisted")
|
||||
}
|
||||
return Result{Phase: PhaseFailed, StatePath: statePath}, err
|
||||
}
|
||||
if active, err := activeSessions(ctx, runner); err != nil {
|
||||
return Result{Phase: PhaseFailed, StatePath: statePath}, err
|
||||
} else if active {
|
||||
return Result{Phase: PhaseFailed, StatePath: statePath}, ErrActiveSessions
|
||||
}
|
||||
if err := promoteLifecycleOverride(overridePath, currentImageOverridePath(statePath), state.Previous.Reference); err != nil {
|
||||
return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("rollback restored the core but durable current-image promotion failed: %w", err)
|
||||
}
|
||||
state.Phase, state.Error = PhaseRolledBack, ""
|
||||
if err := hooks.writeState(statePath, state); err != nil {
|
||||
return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("rollback restored the core but recovery state could not be persisted")
|
||||
}
|
||||
clearMaintenance = true
|
||||
return Result{Phase: PhaseRolledBack, StatePath: statePath}, nil
|
||||
}
|
||||
|
||||
func compensate(ctx context.Context, runner Runner, statePath, overridePath string, state State, cause error, hooks lifecycleHooks) (Result, error, bool) {
|
||||
if err := writeLifecycleOverride(overridePath, state.Previous.Reference); err != nil {
|
||||
return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("update failed and rollback override could not be prepared: recovery required"), false
|
||||
}
|
||||
lifecycle := composeOverrideRunner{Runner: runner, path: overridePath}
|
||||
if err := ensureMaintenance(context.Background(), runner); err != nil {
|
||||
stopped, stopErr := coreIsStopped(context.Background(), runner)
|
||||
if stopErr != nil || !stopped {
|
||||
state.Phase, state.Error = PhaseFailed, "maintenance recovery failed"
|
||||
_ = hooks.writeState(statePath, state)
|
||||
return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("update failed and maintenance could not be reactivated: recovery required"), false
|
||||
}
|
||||
if markerErr := persistMaintenanceWithoutLiveCore(context.Background(), lifecycle); markerErr != nil {
|
||||
state.Phase, state.Error = PhaseFailed, "maintenance recovery failed"
|
||||
_ = hooks.writeState(statePath, state)
|
||||
return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("update failed and durable maintenance could not be established: recovery required"), false
|
||||
}
|
||||
} else if active, err := activeSessions(context.Background(), runner); err != nil || active {
|
||||
state.Phase, state.Error = PhaseFailed, "rollback inventory failed"
|
||||
_ = hooks.writeState(statePath, state)
|
||||
return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("update failed and rollback inventory is not quiescent: recovery required"), false
|
||||
}
|
||||
if restoreErr := restore(ctx, lifecycle, state.Previous); restoreErr != nil {
|
||||
state.Phase, state.Error = PhaseFailed, "candidate verification and automatic rollback failed"
|
||||
if writeErr := hooks.writeState(statePath, state); writeErr != nil {
|
||||
return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("update failed and rollback proof failed; recovery state could not be persisted"), false
|
||||
}
|
||||
return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("update failed; automatic rollback also failed: recovery required"), false
|
||||
}
|
||||
if active, err := activeSessions(context.Background(), runner); err != nil || active {
|
||||
state.Phase, state.Error = PhaseFailed, "restored rollback inventory failed"
|
||||
_ = hooks.writeState(statePath, state)
|
||||
return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("previous core image was restored but rollback inventory is not quiescent: recovery required"), false
|
||||
}
|
||||
if err := promoteLifecycleOverride(overridePath, currentImageOverridePath(statePath), state.Previous.Reference); err != nil {
|
||||
return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("previous core image was restored but durable selector promotion failed: %w", err), false
|
||||
}
|
||||
state.Phase, state.Error = PhaseRolledBack, ""
|
||||
if writeErr := hooks.writeState(statePath, state); writeErr != nil {
|
||||
return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("previous core image was restored but recovery state write failed: recovery required"), false
|
||||
}
|
||||
return Result{Phase: PhaseRolledBack, StatePath: statePath}, fmt.Errorf("update failed; previous core image was restored: %w", cause), true
|
||||
}
|
||||
|
||||
func coreIsStopped(ctx context.Context, runner Runner) (bool, error) {
|
||||
result, err := runCompose(ctx, runner, "ps", "--status", "running", "-q", "core")
|
||||
if err != nil {
|
||||
return false, commandError("core running-state check", result, err)
|
||||
}
|
||||
return strings.TrimSpace(result.Stdout) == "", nil
|
||||
}
|
||||
|
||||
const maintenanceMarkerScript = `
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const marker = process.env.THT_MAINTENANCE_FILE;
|
||||
if (!marker) throw new Error("THT_MAINTENANCE_FILE is required");
|
||||
const directory = path.dirname(marker);
|
||||
fs.mkdirSync(directory, { recursive: true });
|
||||
const temporary = marker + ".rollback-" + process.pid + "-" + Date.now();
|
||||
let file;
|
||||
try {
|
||||
file = fs.openSync(temporary, "wx", 0o600);
|
||||
fs.writeFileSync(file, "{\"version\":1,\"active\":true}\n", "utf8");
|
||||
fs.fsyncSync(file);
|
||||
fs.closeSync(file);
|
||||
file = undefined;
|
||||
fs.renameSync(temporary, marker);
|
||||
const directoryFile = fs.openSync(directory, "r");
|
||||
try { fs.fsyncSync(directoryFile); } finally { fs.closeSync(directoryFile); }
|
||||
} catch (error) {
|
||||
if (file !== undefined) try { fs.closeSync(file); } catch {}
|
||||
try { fs.unlinkSync(temporary); } catch {}
|
||||
throw error;
|
||||
}
|
||||
`
|
||||
|
||||
func persistMaintenanceWithoutLiveCore(ctx context.Context, runner Runner) error {
|
||||
result, err := runCompose(ctx, runner,
|
||||
"run", "--rm", "--no-deps", "--entrypoint", "node", "core", "-e", maintenanceMarkerScript,
|
||||
)
|
||||
if err != nil {
|
||||
return commandError("durable maintenance recovery", result, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sourceValue(request Request) string {
|
||||
if request.Source == PullSource {
|
||||
return request.Image
|
||||
}
|
||||
return string(BuildSource)
|
||||
}
|
||||
|
||||
func canonicalDigestReference(value string) (string, error) {
|
||||
if strings.Contains(value, "://") || strings.ContainsAny(value, "?#") || strings.Contains(value, "@") && strings.Contains(strings.Split(value, "@")[0], ":") && strings.Contains(strings.Split(value, "@")[0], "//") {
|
||||
return "", errors.New("pulled Pi image must be a credential-free canonical sha256 digest reference")
|
||||
}
|
||||
parsed, err := reference.ParseAnyReference(value)
|
||||
if err != nil {
|
||||
return "", errors.New("pulled Pi image must be a valid canonical sha256 digest reference")
|
||||
}
|
||||
canonical, ok := parsed.(reference.Canonical)
|
||||
if !ok || canonical.Digest().Algorithm().String() != "sha256" || len(canonical.Digest().Encoded()) != 64 {
|
||||
return "", errors.New("pulled Pi image must use an immutable sha256 digest")
|
||||
}
|
||||
return reference.FamiliarString(canonical), nil
|
||||
}
|
||||
|
||||
func setMaintenance(ctx context.Context, runner Runner, enabled bool) error {
|
||||
path := "deactivate"
|
||||
if enabled {
|
||||
path = "activate"
|
||||
}
|
||||
args := []string{"exec", "-T", "core", "curl", "-fsS", "-X", "POST", "http://127.0.0.1:8787/internal/maintenance/" + path}
|
||||
result, err := runCompose(ctx, runner, args...)
|
||||
status, valid := parseMaintenanceStatus(result.Stdout)
|
||||
if err == nil && valid && status.Active == enabled && status.Admissions == 0 && !status.RecoveryRequired {
|
||||
return nil
|
||||
}
|
||||
// Status identifies the safest immediate state after an ambiguous response. It cannot
|
||||
// acknowledge durability for an operation whose command returned an error.
|
||||
observed, statusErr := MaintenanceStatus(ctx, runner)
|
||||
if valid && status.RecoveryRequired {
|
||||
return recoveryRequired("maintenance durability was explicitly not acknowledged", err)
|
||||
}
|
||||
if statusErr == nil && observed.RecoveryRequired {
|
||||
return recoveryRequired("maintenance durability was explicitly not acknowledged", err)
|
||||
}
|
||||
if err != nil {
|
||||
if statusErr == nil && observed.Active == enabled && observed.Admissions == 0 {
|
||||
return recoveryRequired("maintenance durability was not acknowledged after a failed command", err)
|
||||
}
|
||||
return commandError("maintenance admission gate", result, err)
|
||||
}
|
||||
if statusErr == nil && observed.Active == enabled && observed.Admissions == 0 {
|
||||
return nil
|
||||
}
|
||||
return errors.New("maintenance admission gate did not acknowledge a quiescent state")
|
||||
}
|
||||
|
||||
type MaintenanceState struct {
|
||||
Active bool `json:"active"`
|
||||
Admissions int `json:"admissions"`
|
||||
RecoveryRequired bool `json:"recoveryRequired"`
|
||||
}
|
||||
|
||||
func parseMaintenanceStatus(value string) (MaintenanceState, bool) {
|
||||
var status MaintenanceState
|
||||
err := json.Unmarshal([]byte(value), &status)
|
||||
return status, err == nil && status.Admissions >= 0
|
||||
}
|
||||
|
||||
func MaintenanceStatus(ctx context.Context, runner Runner) (MaintenanceState, error) {
|
||||
result, err := runCompose(ctx, runner, "exec", "-T", "core", "curl", "-fsS", "http://127.0.0.1:8787/internal/maintenance/status")
|
||||
if err != nil {
|
||||
return MaintenanceState{}, commandError("maintenance status check", result, err)
|
||||
}
|
||||
status, valid := parseMaintenanceStatus(result.Stdout)
|
||||
if !valid {
|
||||
return MaintenanceState{}, errors.New("maintenance status check returned invalid data")
|
||||
}
|
||||
return status, nil
|
||||
}
|
||||
|
||||
func ensureMaintenance(ctx context.Context, runner Runner) error {
|
||||
status, err := MaintenanceStatus(ctx, runner)
|
||||
if err == nil && status.Active && status.Admissions == 0 && !status.RecoveryRequired {
|
||||
return nil
|
||||
}
|
||||
if err == nil && status.RecoveryRequired {
|
||||
return recoveryRequired("maintenance durability was explicitly not acknowledged", nil)
|
||||
}
|
||||
return setMaintenance(ctx, runner, true)
|
||||
}
|
||||
|
||||
func activeSessions(ctx context.Context, runner Runner) (bool, error) {
|
||||
scope := "all"
|
||||
if scoped, ok := runner.(interface{ SessionInventoryScope() string }); ok {
|
||||
if requested := scoped.SessionInventoryScope(); requested == "mine" || requested == "all" {
|
||||
scope = requested
|
||||
}
|
||||
}
|
||||
args := append([]string{"exec", "-T", "core", "curl", "-fsS"}, internalIdentityHeaders...)
|
||||
args = append(args, "http://127.0.0.1:8787/sessions?scope="+scope)
|
||||
result, err := runCompose(ctx, runner, args...)
|
||||
if err != nil {
|
||||
return false, commandError("active-session check", result, err)
|
||||
}
|
||||
var payload []struct {
|
||||
Status string `json:"status"`
|
||||
Archived bool `json:"archived"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(result.Stdout), &payload); err != nil {
|
||||
return false, errors.New("active-session check returned invalid session data")
|
||||
}
|
||||
for _, session := range payload {
|
||||
if !session.Archived && session.Status != "finalized" && session.Status != "closed" {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func waitForInactiveSessions(ctx context.Context, runner Runner, drain bool, sleep func(time.Duration)) error {
|
||||
running, err := activeSessions(ctx, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !running {
|
||||
return nil
|
||||
}
|
||||
if !drain {
|
||||
return ErrActiveSessions
|
||||
}
|
||||
for attempts := 0; attempts < 30; attempts++ {
|
||||
running, err = activeSessions(ctx, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !running {
|
||||
return nil
|
||||
}
|
||||
sleep(time.Second)
|
||||
}
|
||||
return ErrActiveSessions
|
||||
}
|
||||
|
||||
func runningImage(ctx context.Context, runner Runner, reference string) (Image, error) {
|
||||
container, err := runCompose(ctx, runner, "ps", "-q", "core")
|
||||
if err != nil || strings.TrimSpace(container.Stdout) == "" {
|
||||
return Image{}, commandError("running core image check", container, err)
|
||||
}
|
||||
id := strings.TrimSpace(container.Stdout)
|
||||
image, err := runner.Run(ctx, []string{"inspect", "--format", "{{.Image}}", id}, nil)
|
||||
if err != nil || strings.TrimSpace(image.Stdout) == "" {
|
||||
return Image{}, commandError("running core image check", image, err)
|
||||
}
|
||||
mounts, err := runner.Run(ctx, []string{"inspect", "--format", "{{json .Mounts}}", id}, nil)
|
||||
if err != nil {
|
||||
return Image{}, commandError("core volume check", mounts, err)
|
||||
}
|
||||
var raw []struct {
|
||||
Type string `json:"Type"`
|
||||
Name string `json:"Name"`
|
||||
Source string `json:"Source"`
|
||||
Destination string `json:"Destination"`
|
||||
RW bool `json:"RW"`
|
||||
Mode string `json:"Mode"`
|
||||
Propagation string `json:"Propagation"`
|
||||
Driver string `json:"Driver"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(mounts.Stdout), &raw); err != nil {
|
||||
return Image{}, errors.New("core returned invalid persistence mount data")
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return Image{}, errors.New("core has no persistence mounts to preserve")
|
||||
}
|
||||
contract := make([]Mount, 0, len(raw))
|
||||
for _, mount := range raw {
|
||||
if mount.Type == "" || mount.Source == "" || mount.Destination == "" {
|
||||
return Image{}, errors.New("core returned incomplete persistence mount data")
|
||||
}
|
||||
contract = append(contract, Mount{Type: mount.Type, Name: mount.Name, SourceSHA256: mountSourceHash(mount.Source), SourceAliases: mountSourceAliases(mount.Type, mount.Source, runtime.GOOS), Destination: mount.Destination, RW: mount.RW, Options: strings.Join([]string{mount.Mode, mount.Propagation, mount.Driver}, "\x00")})
|
||||
}
|
||||
return Image{ID: strings.TrimSpace(image.Stdout), Reference: reference, Mounts: contract, MountFingerprint: mountFingerprint(contract)}, nil
|
||||
}
|
||||
|
||||
func prepareCandidate(ctx context.Context, runner Runner, request Request, candidateReference string) error {
|
||||
if request.Source == BuildSource {
|
||||
result, err := runCompose(ctx, runner, "build", "--pull", "--build-arg", "PI_VERSION="+request.Version, "core")
|
||||
if err != nil {
|
||||
return commandError("Pi image build", result, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
pull, err := runner.Run(ctx, []string{"pull", request.Image}, nil)
|
||||
if err != nil {
|
||||
return commandError("Pi image pull", pull, err)
|
||||
}
|
||||
return tagImage(ctx, runner, request.Image, candidateReference, "Pi image tag")
|
||||
}
|
||||
|
||||
func recreateCore(ctx context.Context, runner Runner) error {
|
||||
result, err := runCompose(ctx, runner, "up", "--detach", "--wait", "--wait-timeout", "45", "--no-deps", "--force-recreate", "core")
|
||||
if err != nil {
|
||||
return commandError("core recreation", result, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyCandidate(ctx context.Context, runner Runner, wanted string, previous Image) error {
|
||||
health, err := runCompose(ctx, runner, "exec", "-T", "core", "curl", "-fsS", "http://127.0.0.1:8787/health")
|
||||
if err != nil {
|
||||
return commandError("core health check", health, err)
|
||||
}
|
||||
if err := verifyCandidateVersionIdentity(ctx, runner, wanted); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := Test(ctx, runner); err != nil {
|
||||
return err
|
||||
}
|
||||
configured, err := renderedCore(ctx, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if configured.ConfigurationSHA != previous.ConfigurationSHA {
|
||||
return errors.New("external endpoint configuration changed during Pi update")
|
||||
}
|
||||
after, err := runningImage(ctx, runner, configured.Reference)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !sameMounts(previous.Mounts, after.Mounts) {
|
||||
return errors.New("core persistence mount contract changed during Pi update")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyCandidateVersionIdentity(ctx context.Context, runner Runner, wanted string) error {
|
||||
executable, err := Status(ctx, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
environment, label, err := expectedVersions(ctx, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if executable != wanted || environment != wanted || label != wanted {
|
||||
return errors.New("candidate Pi executable, PI_VERSION, and image label do not all match the requested pinned version")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func restore(ctx context.Context, runner Runner, previous Image) error {
|
||||
if err := tagImage(ctx, runner, previous.ID, previous.Reference, "rollback image restore"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := recreateCore(ctx, runner); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureMaintenance(ctx, runner); err != nil {
|
||||
return err
|
||||
}
|
||||
configured, err := renderedCore(ctx, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
after, err := runningImage(ctx, runner, configured.Reference)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if after.ID != previous.ID {
|
||||
return errors.New("rollback core image does not match recorded previous image")
|
||||
}
|
||||
if configured.ConfigurationSHA != previous.ConfigurationSHA {
|
||||
return errors.New("external endpoint configuration drift prevents rollback proof")
|
||||
}
|
||||
if !sameMounts(previous.Mounts, after.Mounts) {
|
||||
return errors.New("core persistence mount contract changed during rollback")
|
||||
}
|
||||
if err := Doctor(ctx, runner); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := Test(ctx, runner); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func tagImage(ctx context.Context, runner Runner, source, target, label string) error {
|
||||
result, err := runner.Run(ctx, []string{"image", "tag", source, target}, nil)
|
||||
if err != nil {
|
||||
return commandError(label, result, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type composeOverrideRunner struct {
|
||||
Runner
|
||||
path string
|
||||
}
|
||||
|
||||
func (r composeOverrideRunner) Run(ctx context.Context, args []string, stdin io.Reader) (compose.Result, error) {
|
||||
if len(args) > 0 && args[0] == "compose" {
|
||||
withOverride := append([]string{"compose", "-f", r.path}, args[1:]...)
|
||||
return r.Runner.Run(ctx, withOverride, stdin)
|
||||
}
|
||||
return r.Runner.Run(ctx, args, stdin)
|
||||
}
|
||||
|
||||
func lifecycleTransaction(statePath string) string {
|
||||
value := fmt.Sprintf("%s\x00%d\x00%d", filepath.Clean(statePath), os.Getpid(), time.Now().UnixNano())
|
||||
sum := sha256.Sum256([]byte(value))
|
||||
return fmt.Sprintf("%x", sum[:8])
|
||||
}
|
||||
|
||||
func lifecycleImageTag(transaction, role string) string {
|
||||
return "thothii-core:tht-" + transaction + "-" + role
|
||||
}
|
||||
|
||||
func lifecycleOverridePath(statePath, transaction string) string {
|
||||
if transaction == "" {
|
||||
transaction = "recovery"
|
||||
}
|
||||
return filepath.Join(filepath.Dir(statePath), "pi-lifecycle-"+transaction+".yaml")
|
||||
}
|
||||
|
||||
func currentImageOverridePath(statePath string) string {
|
||||
return filepath.Join(filepath.Dir(statePath), "current-image.yaml")
|
||||
}
|
||||
|
||||
func writeLifecycleOverride(path, image string) error {
|
||||
quoted, err := json.Marshal(image)
|
||||
if err != nil {
|
||||
return errors.New("lifecycle image override could not be encoded")
|
||||
}
|
||||
contents := []byte(`services:
|
||||
core:
|
||||
image: ` + string(quoted) + `
|
||||
workspace-maintenance:
|
||||
image: ` + string(quoted) + `
|
||||
`)
|
||||
if err := writeFileDurably(path, ".pi-lifecycle-", contents); err != nil {
|
||||
return errors.New("lifecycle image override could not be written durably")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func promoteLifecycleOverride(source, destination, expectedImage string) error {
|
||||
return promoteLifecycleOverrideWith(source, destination, expectedImage, durableReplace)
|
||||
}
|
||||
|
||||
func promoteLifecycleOverrideWith(
|
||||
source, destination, expectedImage string,
|
||||
replace func(string, string, string) error,
|
||||
) error {
|
||||
if err := replace(source, destination, filepath.Dir(destination)); err != nil {
|
||||
selected, readErr := readLifecycleOverride(destination)
|
||||
if readErr == nil && selected == expectedImage {
|
||||
return recoveryRequired("lifecycle image override changed but durability was not acknowledged", err)
|
||||
}
|
||||
return recoveryRequired("lifecycle image override could not be promoted durably", err)
|
||||
}
|
||||
selected, err := readLifecycleOverride(destination)
|
||||
if err != nil || selected != expectedImage {
|
||||
return errors.New("promoted lifecycle image override could not be verified")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func readLifecycleOverride(path string) (string, error) {
|
||||
contents, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, line := range strings.Split(string(contents), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if !strings.HasPrefix(line, "image:") {
|
||||
continue
|
||||
}
|
||||
encoded := strings.TrimSpace(strings.TrimPrefix(line, "image:"))
|
||||
var image string
|
||||
if json.Unmarshal([]byte(encoded), &image) != nil || image == "" || strings.ContainsAny(image, "\r\n") {
|
||||
return "", errors.New("lifecycle image override is invalid")
|
||||
}
|
||||
return image, nil
|
||||
}
|
||||
return "", errors.New("lifecycle image override has no core image")
|
||||
}
|
||||
|
||||
// RecoverMaintenance clears a stale durable gate only after the running core and terminal
|
||||
// recovery metadata prove that no rollback is still required.
|
||||
func RecoverMaintenance(ctx context.Context, runner Runner, statePath string, confirm bool) error {
|
||||
if !confirm {
|
||||
return ErrConfirmationRequired
|
||||
}
|
||||
lock, err := acquireLock(statePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer lock.Release()
|
||||
return recoverMaintenanceLocked(ctx, runner, statePath)
|
||||
}
|
||||
|
||||
func recoverMaintenanceLocked(ctx context.Context, runner Runner, statePath string) error {
|
||||
state, stateErr := readState(statePath)
|
||||
if stateErr == nil {
|
||||
transactionOverride := lifecycleOverridePath(statePath, state.Transaction)
|
||||
switch {
|
||||
case state.Phase == PhasePromoting:
|
||||
if err := recoverPromotion(ctx, runner, statePath, transactionOverride, &state); err != nil {
|
||||
return err
|
||||
}
|
||||
case !state.MutationStarted && state.Phase != PhaseVerified && state.Phase != PhaseRolledBack && state.Phase != PhaseNoop:
|
||||
state.Phase, state.Error = PhaseFailed, "candidate preparation interrupted before core mutation"
|
||||
if err := writeState(statePath, state); err != nil {
|
||||
return errors.New("maintenance recovery could not finalize safe preparation state")
|
||||
}
|
||||
if err := durableRemove(transactionOverride); err != nil {
|
||||
return errors.New("maintenance recovery could not remove the safe preparation override")
|
||||
}
|
||||
case stateNeedsRecovery(state):
|
||||
return ErrInterruptedUpdate
|
||||
default:
|
||||
if err := durableRemove(transactionOverride); err != nil {
|
||||
return errors.New("maintenance recovery could not remove the lifecycle override")
|
||||
}
|
||||
}
|
||||
} else if !errors.Is(stateErr, os.ErrNotExist) {
|
||||
return stateErr
|
||||
}
|
||||
status, err := MaintenanceStatus(ctx, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !status.Active {
|
||||
return nil
|
||||
}
|
||||
if err := Doctor(ctx, runner); err != nil {
|
||||
return err
|
||||
}
|
||||
return setMaintenance(ctx, runner, false)
|
||||
}
|
||||
|
||||
func recoverPromotion(ctx context.Context, runner Runner, statePath, transactionOverride string, state *State) error {
|
||||
currentOverride := currentImageOverridePath(statePath)
|
||||
selected, currentErr := readLifecycleOverride(currentOverride)
|
||||
if currentErr != nil || selected != state.Candidate.Reference {
|
||||
pending, pendingErr := readLifecycleOverride(transactionOverride)
|
||||
if pendingErr != nil || pending != state.Candidate.Reference {
|
||||
if currentErr == nil && selected == state.Previous.Reference {
|
||||
if err := verifyRestoredCurrent(ctx, runner, state.Previous); err != nil {
|
||||
return ErrInterruptedUpdate
|
||||
}
|
||||
state.Phase, state.Error = PhaseRolledBack, ""
|
||||
return writeState(statePath, *state)
|
||||
}
|
||||
return ErrInterruptedUpdate
|
||||
}
|
||||
if err := promoteLifecycleOverride(transactionOverride, currentOverride, state.Candidate.Reference); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := verifyCandidate(ctx, runner, state.Target.Version, state.Previous); err != nil {
|
||||
return err
|
||||
}
|
||||
state.Phase, state.Error = PhaseVerified, ""
|
||||
return writeState(statePath, *state)
|
||||
}
|
||||
|
||||
func verifyRestoredCurrent(ctx context.Context, runner Runner, previous Image) error {
|
||||
configured, err := renderedCore(ctx, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
after, err := runningImage(ctx, runner, configured.Reference)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if after.ID != previous.ID || configured.ConfigurationSHA != previous.ConfigurationSHA || !sameMounts(previous.Mounts, after.Mounts) {
|
||||
return errors.New("running core does not match the durable previous-image selector")
|
||||
}
|
||||
if err := Doctor(ctx, runner); err != nil {
|
||||
return err
|
||||
}
|
||||
return Test(ctx, runner)
|
||||
}
|
||||
|
||||
func sameStrings(left, right []string) bool {
|
||||
left, right = append([]string(nil), left...), append([]string(nil), right...)
|
||||
sort.Strings(left)
|
||||
sort.Strings(right)
|
||||
return strings.Join(left, "\x00") == strings.Join(right, "\x00")
|
||||
}
|
||||
|
||||
func sameMounts(left, right []Mount) bool {
|
||||
if len(left) != len(right) {
|
||||
return false
|
||||
}
|
||||
identityWithoutSource := func(m Mount) string {
|
||||
return m.Type + "\x00" + m.Name + "\x00" + m.Destination + "\x00" + fmt.Sprint(m.RW) + "\x00" + m.Options
|
||||
}
|
||||
sourceMatches := func(a, b Mount) bool {
|
||||
if a.SourceSHA256 == b.SourceSHA256 {
|
||||
return true
|
||||
}
|
||||
for _, alias := range a.SourceAliases {
|
||||
if alias == b.SourceSHA256 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, alias := range b.SourceAliases {
|
||||
if alias == a.SourceSHA256 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
matched := make([]bool, len(right))
|
||||
for _, candidate := range left {
|
||||
found := false
|
||||
for index, observed := range right {
|
||||
if matched[index] || identityWithoutSource(candidate) != identityWithoutSource(observed) || !sourceMatches(candidate, observed) {
|
||||
continue
|
||||
}
|
||||
matched[index], found = true, true
|
||||
break
|
||||
}
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user