feat(thothctl): add safe Pi core restart
This commit is contained in:
@@ -0,0 +1,292 @@
|
||||
package pi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRestartRequiresConfirmationWithoutInvokingCompose(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
_, err := Restart(context.Background(), fake, RestartRequest{
|
||||
StatePath: filepath.Join(dir, "restart-state.json"),
|
||||
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
||||
})
|
||||
if !errors.Is(err, ErrConfirmationRequired) {
|
||||
t.Fatalf("Restart() error = %v, want ErrConfirmationRequired", err)
|
||||
}
|
||||
assertNotCalled(t, fake.calls, "compose")
|
||||
}
|
||||
|
||||
func TestRestartDrainsRecreatesOnlyCoreAndRetainsImage(t *testing.T) {
|
||||
fake := newFakeRunner()
|
||||
fake.activeSessions = true
|
||||
dir := t.TempDir()
|
||||
hooks := defaultLifecycleHooks
|
||||
hooks.sleep = func(time.Duration) { fake.activeSessions = false }
|
||||
|
||||
result, err := restartWithHooks(context.Background(), fake, RestartRequest{
|
||||
StatePath: filepath.Join(dir, "restart-state.json"),
|
||||
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
||||
Confirm: true,
|
||||
Drain: true,
|
||||
}, hooks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Version != fake.version {
|
||||
t.Fatalf("version = %q, want %q", result.Version, fake.version)
|
||||
}
|
||||
assertCalled(t, fake.calls, "up --detach --wait --wait-timeout 45 --no-deps --force-recreate core")
|
||||
assertNotCalled(t, fake.calls, "build --pull")
|
||||
assertNotCalled(t, fake.calls, "pull ")
|
||||
assertNotCalled(t, fake.calls, "frontend")
|
||||
if _, err := os.Stat(result.StatePath); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("successful restart state still exists: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestartRefusesActiveSessionsWithoutDrain(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
fake.activeSessions = true
|
||||
|
||||
_, err := Restart(context.Background(), fake, RestartRequest{
|
||||
StatePath: filepath.Join(dir, "restart-state.json"),
|
||||
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
||||
Confirm: true,
|
||||
})
|
||||
if !errors.Is(err, ErrActiveSessions) {
|
||||
t.Fatalf("Restart() error = %v, want ErrActiveSessions", err)
|
||||
}
|
||||
if fake.maintenance {
|
||||
t.Fatal("maintenance remained active after refusing pre-mutation restart")
|
||||
}
|
||||
if fake.recreated {
|
||||
t.Fatal("core was recreated with active sessions")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestartRefusesInterruptedUpdateOrRestartState(t *testing.T) {
|
||||
for _, stateFile := range []string{"update-state.json", "restart-state.json"} {
|
||||
t.Run(stateFile, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
previous := stateImageForTest(t, fake)
|
||||
writeStateForTest(t, filepath.Join(dir, stateFile), State{
|
||||
Transaction: "interrupted",
|
||||
Phase: PhaseRecreated,
|
||||
Target: Target{Version: fake.version, Source: "restart"},
|
||||
Previous: previous,
|
||||
MutationStarted: true,
|
||||
})
|
||||
fake.calls = nil
|
||||
|
||||
_, err := Restart(context.Background(), fake, RestartRequest{
|
||||
StatePath: filepath.Join(dir, "restart-state.json"),
|
||||
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
||||
Confirm: true,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Restart() accepted interrupted lifecycle state")
|
||||
}
|
||||
assertNotCalled(t, fake.calls, "compose")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestartPreflightFailureNeverRecreatesCoreAndClearsMaintenance(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
fake.fail = "preflight"
|
||||
|
||||
_, err := Restart(context.Background(), fake, RestartRequest{
|
||||
StatePath: filepath.Join(dir, "restart-state.json"),
|
||||
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
||||
Confirm: true,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Restart() error = nil, want preflight failure")
|
||||
}
|
||||
if fake.recreated || fake.currentImage != "sha256:old" {
|
||||
t.Fatalf("preflight failure mutated core: recreated=%t image=%q", fake.recreated, fake.currentImage)
|
||||
}
|
||||
if fake.maintenance {
|
||||
t.Fatal("maintenance remained active after preflight failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestartPostRecreateFailureKeepsMaintenanceAndRecoveryState(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
fake.fail = "health"
|
||||
statePath := filepath.Join(dir, "restart-state.json")
|
||||
|
||||
_, err := Restart(context.Background(), fake, RestartRequest{
|
||||
StatePath: statePath,
|
||||
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
||||
Confirm: true,
|
||||
})
|
||||
var recovery *RecoveryRequiredError
|
||||
if !errors.As(err, &recovery) {
|
||||
t.Fatalf("Restart() error = %v, want RecoveryRequiredError", err)
|
||||
}
|
||||
if !fake.recreated {
|
||||
t.Fatal("post-recreate failure occurred before core recreation")
|
||||
}
|
||||
if !fake.maintenance {
|
||||
t.Fatal("maintenance was cleared after post-recreate failure")
|
||||
}
|
||||
state, stateErr := readState(statePath)
|
||||
if stateErr != nil || !state.MutationStarted {
|
||||
t.Fatalf("restart recovery state = %+v, %v; want durable mutation state", state, stateErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestartMaintenanceClearFailureRestoresRecoveryState(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
fake.fail = "maintenance-clear"
|
||||
statePath := filepath.Join(dir, "restart-state.json")
|
||||
|
||||
_, err := Restart(context.Background(), fake, RestartRequest{
|
||||
StatePath: statePath,
|
||||
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
||||
Confirm: true,
|
||||
})
|
||||
var recovery *RecoveryRequiredError
|
||||
if !errors.As(err, &recovery) {
|
||||
t.Fatalf("Restart() error = %v, want RecoveryRequiredError", err)
|
||||
}
|
||||
if !fake.maintenance {
|
||||
t.Fatal("maintenance was cleared despite deactivation failure")
|
||||
}
|
||||
state, stateErr := readState(statePath)
|
||||
if stateErr != nil || state.Phase != PhaseVerified || !state.MutationStarted {
|
||||
t.Fatalf("restart recovery state = %+v, %v; want durable verified mutation state", state, stateErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoverLifecycleMaintenanceVerifiesAndClearsRestartState(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
fake.maintenance = true
|
||||
previous := stateImageForTest(t, fake)
|
||||
restartStatePath := filepath.Join(dir, "restart-state.json")
|
||||
updateStatePath := filepath.Join(dir, "update-state.json")
|
||||
writeStateForTest(t, restartStatePath, State{
|
||||
Transaction: "restart-recovery",
|
||||
Phase: PhaseRecreated,
|
||||
Target: Target{Version: fake.version, Source: "restart"},
|
||||
Previous: previous,
|
||||
MutationStarted: true,
|
||||
})
|
||||
candidate := previous
|
||||
candidate.Reference = "thothii-core:thothctl-recover-candidate"
|
||||
writeStateForTest(t, updateStatePath, State{
|
||||
Transaction: "update-recovery",
|
||||
Phase: PhasePromoting,
|
||||
Target: Target{Version: fake.version, Source: string(BuildSource)},
|
||||
Previous: previous,
|
||||
Candidate: candidate,
|
||||
MutationStarted: true,
|
||||
})
|
||||
if err := writeLifecycleOverride(currentImageOverridePath(updateStatePath), candidate.Reference); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fake.calls = nil
|
||||
|
||||
if err := RecoverLifecycleMaintenance(context.Background(), fake, updateStatePath, restartStatePath, true); err != nil {
|
||||
t.Fatalf("RecoverLifecycleMaintenance() error = %v", err)
|
||||
}
|
||||
if _, err := os.Stat(restartStatePath); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("restart recovery state still exists: %v", err)
|
||||
}
|
||||
if fake.maintenance {
|
||||
t.Fatal("maintenance remained active after both lifecycle states were verified")
|
||||
}
|
||||
updateState, err := readState(updateStatePath)
|
||||
if err != nil || updateState.Phase != PhaseVerified {
|
||||
t.Fatalf("update recovery state = %+v, %v; want verified image rollback metadata", updateState, err)
|
||||
}
|
||||
deactivate := callIndex(fake.calls, "/internal/maintenance/deactivate")
|
||||
lastVerification := lastCallIndexBefore(fake.calls, "/pi-management/test", deactivate)
|
||||
if deactivate < 0 || lastVerification < 0 {
|
||||
t.Fatalf("calls = %v; want verification before maintenance deactivation", fake.calls)
|
||||
}
|
||||
verificationCount := 0
|
||||
for index := 0; index < deactivate; index++ {
|
||||
if strings.Contains(fake.calls[index], "/internal/maintenance/deactivate") {
|
||||
t.Fatalf("maintenance reopened before combined verification: %v", fake.calls)
|
||||
}
|
||||
if strings.Contains(fake.calls[index], "/pi-management/test") {
|
||||
verificationCount++
|
||||
}
|
||||
}
|
||||
if verificationCount < 3 {
|
||||
t.Fatalf("verification calls before maintenance deactivation = %d, want restart, update, and final proofs: %v", verificationCount, fake.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestartRejectsImageConfigurationAndMountDrift(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
failure string
|
||||
want error
|
||||
}{
|
||||
{failure: "image-drift", want: errRestartImageDrift},
|
||||
{failure: "config-drift", want: errRestartConfigurationDrift},
|
||||
{failure: "mount-drift", want: errRestartMountDrift},
|
||||
} {
|
||||
t.Run(test.failure, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
fake.fail = test.failure
|
||||
statePath := filepath.Join(dir, "restart-state.json")
|
||||
|
||||
_, err := Restart(context.Background(), fake, RestartRequest{
|
||||
StatePath: statePath,
|
||||
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
||||
Confirm: true,
|
||||
})
|
||||
if !errors.Is(err, test.want) {
|
||||
t.Fatalf("Restart() error = %v, want errors.Is(..., %v)", err, test.want)
|
||||
}
|
||||
if !fake.maintenance {
|
||||
t.Fatal("maintenance was cleared after restart identity drift")
|
||||
}
|
||||
if _, stateErr := os.Stat(statePath); stateErr != nil {
|
||||
t.Fatalf("restart recovery state missing after drift: %v", stateErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestartPreservesVerifiedUpdateState(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
fake := newFakeRunner()
|
||||
updateStatePath := filepath.Join(dir, "update-state.json")
|
||||
writeStateForTest(t, updateStatePath, State{
|
||||
Transaction: "verified-update",
|
||||
Phase: PhaseVerified,
|
||||
Target: Target{Version: fake.version, Source: string(BuildSource)},
|
||||
Previous: stateImageForTest(t, fake),
|
||||
})
|
||||
before := readStateBytes(t, updateStatePath)
|
||||
|
||||
if _, err := Restart(context.Background(), fake, RestartRequest{
|
||||
StatePath: filepath.Join(dir, "restart-state.json"),
|
||||
UpdateStatePath: updateStatePath,
|
||||
Confirm: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after := readStateBytes(t, updateStatePath)
|
||||
if string(after) != string(before) {
|
||||
t.Fatal("restart changed verified update rollback metadata")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user