Files
ThothII/tools/tht/internal/pi/restart_test.go
T

600 lines
20 KiB
Go

package pi
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func TestRestartRequiresConfirmationWithoutInvokingCompose(t *testing.T) {
dir := t.TempDir()
fake := newFakeRunner()
_, err := Restart(context.Background(), fake, RestartRequest{
StatePath: filepath.Join(dir, "restart-state.json"),
UpdateStatePath: filepath.Join(dir, "update-state.json"),
})
if !errors.Is(err, ErrConfirmationRequired) {
t.Fatalf("Restart() error = %v, want ErrConfirmationRequired", err)
}
if got, want := err.Error(), "restart requires --yes after reviewing the planned Pi core recreation"; got != want {
t.Fatalf("Restart() error text = %q, want %q", got, want)
}
assertNotCalled(t, fake.calls, "compose")
}
func TestRestartDrainsRecreatesOnlyCoreAndRetainsImage(t *testing.T) {
fake := newFakeRunner()
fake.sessionsWire = `[{"status":"open","archived":false}]`
dir := t.TempDir()
hooks := defaultLifecycleHooks
sleepCalls := 0
hooks.sleep = func(duration time.Duration) {
sleepCalls++
if duration != time.Second {
t.Fatalf("drain sleep = %s, want %s", duration, time.Second)
}
fake.sessionsWire = `[]`
}
result, err := restartWithHooks(context.Background(), fake, RestartRequest{
StatePath: filepath.Join(dir, "restart-state.json"),
UpdateStatePath: filepath.Join(dir, "update-state.json"),
Confirm: true,
Drain: true,
}, hooks)
if err != nil {
t.Fatal(err)
}
if result.Version != fake.version {
t.Fatalf("version = %q, want %q", result.Version, fake.version)
}
if sleepCalls != 1 {
t.Fatalf("drain sleep calls = %d, want 1", sleepCalls)
}
assertCalled(t, fake.calls, "up --detach --wait --wait-timeout 45 --no-deps --force-recreate --no-build --pull never core")
assertNotCalled(t, fake.calls, "compose build --pull")
for _, call := range fake.calls {
if strings.HasPrefix(call, "pull ") {
t.Fatalf("restart invoked direct image pull: %s", call)
}
}
assertNotCalled(t, fake.calls, "frontend")
if _, err := os.Stat(result.StatePath); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("successful restart state still exists: %v", err)
}
}
func TestRestartPinsCapturedImageWhenConfiguredTagMovesBeforeRecreate(t *testing.T) {
dir := t.TempDir()
fake := newFakeRunner()
hooks := defaultLifecycleHooks
write := hooks.writeState
hooks.writeState = func(path string, state State) error {
if state.Phase == PhasePreflight && state.MutationStarted {
fake.tags[fake.configuredImage] = "sha256:moved-configured-tag"
fake.imageVersions["sha256:moved-configured-tag"] = "9.99.0"
}
return write(path, state)
}
restartStatePath := filepath.Join(dir, "restart-state.json")
result, err := restartWithHooks(context.Background(), fake, RestartRequest{
StatePath: restartStatePath,
UpdateStatePath: filepath.Join(dir, "update-state.json"),
Confirm: true,
}, hooks)
if err != nil {
t.Fatalf("restartWithHooks() error = %v", err)
}
if result.Version != "0.80.3" || fake.currentImage != "sha256:old" {
t.Fatalf("restart result=%+v image=%q; want captured 0.80.3 / sha256:old", result, fake.currentImage)
}
assertCalled(t, fake.calls, "image tag sha256:old thothii-core:tht-")
assertCalled(t, fake.calls, "pi-lifecycle-")
if matches, globErr := filepath.Glob(filepath.Join(dir, "pi-lifecycle-*.yaml")); globErr != nil || len(matches) != 0 {
t.Fatalf("successful restart overrides = %v, error = %v; want safe cleanup", matches, globErr)
}
}
func TestRestartRefusesActiveSessionsWithoutDrain(t *testing.T) {
dir := t.TempDir()
fake := newFakeRunner()
fake.activeSessions = true
_, err := Restart(context.Background(), fake, RestartRequest{
StatePath: filepath.Join(dir, "restart-state.json"),
UpdateStatePath: filepath.Join(dir, "update-state.json"),
Confirm: true,
})
if !errors.Is(err, ErrActiveSessions) {
t.Fatalf("Restart() error = %v, want ErrActiveSessions", err)
}
if got, want := err.Error(), "active sessions must be drained before restarting Pi; use --drain only after they are complete"; got != want {
t.Fatalf("Restart() error text = %q, want %q", got, want)
}
if fake.maintenance {
t.Fatal("maintenance remained active after refusing pre-mutation restart")
}
if fake.recreated {
t.Fatal("core was recreated with active sessions")
}
}
func TestRestartActivationFailureClearsPreMutationMaintenance(t *testing.T) {
for _, failure := range []string{
"maintenance-activate-durability",
"maintenance-activate-durability-without-status-flag",
} {
t.Run(failure, func(t *testing.T) {
dir := t.TempDir()
fake := newFakeRunner()
fake.fail = failure
_, err := Restart(context.Background(), fake, RestartRequest{
StatePath: filepath.Join(dir, "restart-state.json"),
UpdateStatePath: filepath.Join(dir, "update-state.json"),
Confirm: true,
})
var recovery *RecoveryRequiredError
if !errors.As(err, &recovery) {
t.Fatalf("Restart() error = %v, want RecoveryRequiredError", err)
}
if fake.maintenance {
t.Fatal("pre-mutation activation failure left maintenance active")
}
if fake.recreated {
t.Fatal("pre-mutation activation failure recreated core")
}
assertCalled(t, fake.calls, "operator-command.js maintenance-activate")
assertCalled(t, fake.calls, "operator-command.js maintenance-deactivate")
})
}
}
func TestRestartRefusesInterruptedUpdateOrRestartState(t *testing.T) {
for _, stateFile := range []string{"update-state.json", "restart-state.json"} {
t.Run(stateFile, func(t *testing.T) {
dir := t.TempDir()
fake := newFakeRunner()
previous := stateImageForTest(t, fake)
writeStateForTest(t, filepath.Join(dir, stateFile), State{
Transaction: "interrupted",
Phase: PhaseRecreated,
Target: Target{Version: fake.version, Source: "restart"},
Previous: previous,
MutationStarted: true,
})
fake.calls = nil
_, err := Restart(context.Background(), fake, RestartRequest{
StatePath: filepath.Join(dir, "restart-state.json"),
UpdateStatePath: filepath.Join(dir, "update-state.json"),
Confirm: true,
})
if err == nil {
t.Fatal("Restart() accepted interrupted lifecycle state")
}
assertNotCalled(t, fake.calls, "compose")
})
}
}
func TestRestartPreflightFailureNeverRecreatesCoreAndClearsMaintenance(t *testing.T) {
dir := t.TempDir()
fake := newFakeRunner()
fake.fail = "preflight"
_, err := Restart(context.Background(), fake, RestartRequest{
StatePath: filepath.Join(dir, "restart-state.json"),
UpdateStatePath: filepath.Join(dir, "update-state.json"),
Confirm: true,
})
if err == nil {
t.Fatal("Restart() error = nil, want preflight failure")
}
if fake.recreated || fake.currentImage != "sha256:old" {
t.Fatalf("preflight failure mutated core: recreated=%t image=%q", fake.recreated, fake.currentImage)
}
if fake.maintenance {
t.Fatal("maintenance remained active after preflight failure")
}
}
func TestRestartPostRecreateFailureKeepsMaintenanceAndRecoveryState(t *testing.T) {
dir := t.TempDir()
fake := newFakeRunner()
fake.fail = "health"
statePath := filepath.Join(dir, "restart-state.json")
_, err := Restart(context.Background(), fake, RestartRequest{
StatePath: statePath,
UpdateStatePath: filepath.Join(dir, "update-state.json"),
Confirm: true,
})
var recovery *RecoveryRequiredError
if !errors.As(err, &recovery) {
t.Fatalf("Restart() error = %v, want RecoveryRequiredError", err)
}
if !fake.recreated {
t.Fatal("post-recreate failure occurred before core recreation")
}
if !fake.maintenance {
t.Fatal("maintenance was cleared after post-recreate failure")
}
state, stateErr := readState(statePath)
if stateErr != nil || !state.MutationStarted {
t.Fatalf("restart recovery state = %+v, %v; want durable mutation state", state, stateErr)
}
overridePath := lifecycleOverridePath(statePath, state.Transaction)
selected, overrideErr := readLifecycleOverride(overridePath)
if overrideErr != nil || selected != state.Previous.Reference || fake.tags[selected] != state.Previous.ID {
t.Fatalf("restart override = %q, %v; want retained exact image %q", selected, overrideErr, state.Previous.ID)
}
}
func TestRestartMaintenanceClearFailureRestoresRecoveryState(t *testing.T) {
dir := t.TempDir()
fake := newFakeRunner()
fake.fail = "maintenance-clear"
statePath := filepath.Join(dir, "restart-state.json")
_, err := Restart(context.Background(), fake, RestartRequest{
StatePath: statePath,
UpdateStatePath: filepath.Join(dir, "update-state.json"),
Confirm: true,
})
var recovery *RecoveryRequiredError
if !errors.As(err, &recovery) {
t.Fatalf("Restart() error = %v, want RecoveryRequiredError", err)
}
if !fake.maintenance {
t.Fatal("maintenance was cleared despite deactivation failure")
}
state, stateErr := readState(statePath)
if stateErr != nil || state.Phase != PhaseVerified || !state.MutationStarted {
t.Fatalf("restart recovery state = %+v, %v; want durable verified mutation state", state, stateErr)
}
}
func TestRecoverLifecycleMaintenanceVerifiesAndClearsRestartState(t *testing.T) {
dir := t.TempDir()
fake := newFakeRunner()
fake.maintenance = true
previous := stateImageForTest(t, fake)
restartStatePath := filepath.Join(dir, "restart-state.json")
updateStatePath := filepath.Join(dir, "update-state.json")
writeStateForTest(t, restartStatePath, State{
Transaction: "restart-recovery",
Phase: PhaseRecreated,
Target: Target{Version: fake.version, Source: "restart"},
Previous: previous,
MutationStarted: true,
})
restartState, err := readState(restartStatePath)
if err != nil {
t.Fatal(err)
}
restartOverride := lifecycleOverridePath(restartStatePath, restartState.Transaction)
if err := writeLifecycleOverride(restartOverride, restartState.Previous.Reference); err != nil {
t.Fatal(err)
}
delete(fake.tags, restartState.Previous.Reference)
fake.tags[fake.configuredImage] = "sha256:moved-before-recovery"
candidate := previous
candidate.Reference = "thothii-core:tht-recover-candidate"
writeStateForTest(t, updateStatePath, State{
Transaction: "update-recovery",
Phase: PhasePromoting,
Target: Target{Version: fake.version, Source: string(BuildSource)},
Previous: previous,
Candidate: candidate,
MutationStarted: true,
})
if err := writeLifecycleOverride(currentImageOverridePath(updateStatePath), candidate.Reference); err != nil {
t.Fatal(err)
}
fake.calls = nil
if err := RecoverLifecycleMaintenance(context.Background(), fake, updateStatePath, restartStatePath, true); err != nil {
t.Fatalf("RecoverLifecycleMaintenance() error = %v", err)
}
if _, err := os.Stat(restartStatePath); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("restart recovery state still exists: %v", err)
}
if _, err := os.Stat(restartOverride); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("restart recovery override still exists: %v", err)
}
if fake.tags[restartState.Previous.Reference] != restartState.Previous.ID {
t.Fatalf("restart recovery pin = %q, want %q", fake.tags[restartState.Previous.Reference], restartState.Previous.ID)
}
assertCalled(t, fake.calls, restartOverride)
if fake.maintenance {
t.Fatal("maintenance remained active after both lifecycle states were verified")
}
updateState, err := readState(updateStatePath)
if err != nil || updateState.Phase != PhaseVerified {
t.Fatalf("update recovery state = %+v, %v; want verified image rollback metadata", updateState, err)
}
deactivate := callIndex(fake.calls, "operator-command.js maintenance-deactivate")
lastVerification := lastCallIndexBefore(fake.calls, "operator-command.js pi-test", deactivate)
if deactivate < 0 || lastVerification < 0 {
t.Fatalf("calls = %v; want verification before maintenance deactivation", fake.calls)
}
verificationCount := 0
for index := 0; index < deactivate; index++ {
if strings.Contains(fake.calls[index], "operator-command.js maintenance-deactivate") {
t.Fatalf("maintenance reopened before combined verification: %v", fake.calls)
}
if strings.Contains(fake.calls[index], "operator-command.js pi-test") {
verificationCount++
}
}
if verificationCount < 3 {
t.Fatalf("verification calls before maintenance deactivation = %d, want restart, update, and final proofs: %v", verificationCount, fake.calls)
}
}
func TestRecoverLifecycleMaintenanceRejectsMalformedRestartState(t *testing.T) {
for _, test := range []struct {
name string
phase Phase
source string
}{
{name: "recreated_without_mutation_marker", phase: PhaseRecreated, source: "restart"},
{name: "non_restart_source", phase: PhasePreflight, source: string(BuildSource)},
} {
t.Run(test.name, func(t *testing.T) {
dir := t.TempDir()
fake := newFakeRunner()
fake.maintenance = true
restartStatePath := filepath.Join(dir, "restart-state.json")
updateStatePath := filepath.Join(dir, "update-state.json")
writeStateForTest(t, restartStatePath, State{
Transaction: "malformed-restart",
Phase: test.phase,
Target: Target{Version: fake.version, Source: test.source},
Previous: stateImageForTest(t, fake),
})
fake.calls = nil
err := RecoverLifecycleMaintenance(
context.Background(), fake, updateStatePath, restartStatePath, true,
)
if !errors.Is(err, ErrInvalidRequest) || !strings.Contains(err.Error(), "restart recovery state is invalid") {
t.Fatalf("RecoverLifecycleMaintenance() error = %v, want invalid restart recovery state", err)
}
if !fake.maintenance {
t.Fatal("malformed restart state reopened admission")
}
if _, stateErr := os.Stat(restartStatePath); stateErr != nil {
t.Fatalf("malformed restart state was removed: %v", stateErr)
}
assertNotCalled(t, fake.calls, "operator-command.js maintenance-deactivate")
})
}
}
func TestRestartRefusesMalformedRestartStateWithoutInvokingCompose(t *testing.T) {
dir := t.TempDir()
fake := newFakeRunner()
restartStatePath := filepath.Join(dir, "restart-state.json")
writeStateForTest(t, restartStatePath, State{
Transaction: "malformed-restart",
Phase: PhasePreflight,
Target: Target{Version: fake.version, Source: string(BuildSource)},
Previous: stateImageForTest(t, fake),
})
fake.calls = nil
_, err := Restart(context.Background(), fake, RestartRequest{
StatePath: restartStatePath,
UpdateStatePath: filepath.Join(dir, "update-state.json"),
Confirm: true,
})
if !errors.Is(err, ErrInvalidRequest) || !strings.Contains(err.Error(), "restart recovery state is invalid") {
t.Fatalf("Restart() error = %v, want invalid restart recovery state", err)
}
if _, stateErr := os.Stat(restartStatePath); stateErr != nil {
t.Fatalf("malformed restart state was removed: %v", stateErr)
}
assertNotCalled(t, fake.calls, "compose")
}
func TestRestartDurabilityFailureBoundaries(t *testing.T) {
injected := errors.New("injected restart durability failure")
for _, test := range []struct {
name string
configure func(*fakeRunner, *lifecycleHooks)
wantPhase Phase
wantMutation bool
wantRecreated bool
wantMaintenance bool
wantRecovery bool
wantInjected bool
}{
{
name: "mutation_marker_write",
configure: func(_ *fakeRunner, hooks *lifecycleHooks) {
write := hooks.writeState
hooks.writeState = func(path string, state State) error {
if state.Phase == PhasePreflight && state.MutationStarted {
return injected
}
return write(path, state)
}
},
wantPhase: PhasePreflight,
wantInjected: true,
},
{
name: "core_recreation",
configure: func(fake *fakeRunner, _ *lifecycleHooks) {
fake.fail = "recreate"
},
wantPhase: PhasePreflight,
wantMutation: true,
wantRecreated: true,
wantMaintenance: true,
wantRecovery: true,
},
{
name: "maintenance_proof",
configure: func(fake *fakeRunner, _ *lifecycleHooks) {
fake.fail = "maintenance-proof"
},
wantPhase: PhasePreflight,
wantMutation: true,
wantRecreated: true,
wantMaintenance: true,
wantRecovery: true,
},
{
name: "recreated_phase_write",
configure: func(_ *fakeRunner, hooks *lifecycleHooks) {
write := hooks.writeState
hooks.writeState = func(path string, state State) error {
if state.Phase == PhaseRecreated {
return injected
}
return write(path, state)
}
},
wantPhase: PhasePreflight,
wantMutation: true,
wantRecreated: true,
wantMaintenance: true,
wantRecovery: true,
wantInjected: true,
},
{
name: "verified_phase_write",
configure: func(_ *fakeRunner, hooks *lifecycleHooks) {
write := hooks.writeState
hooks.writeState = func(path string, state State) error {
if state.Phase == PhaseVerified {
return injected
}
return write(path, state)
}
},
wantPhase: PhaseRecreated,
wantMutation: true,
wantRecreated: true,
wantMaintenance: true,
wantRecovery: true,
wantInjected: true,
},
{
name: "restart_state_removal",
configure: func(_ *fakeRunner, hooks *lifecycleHooks) {
hooks.removeFile = func(string) error { return injected }
},
wantPhase: PhaseVerified,
wantMutation: true,
wantRecreated: true,
wantMaintenance: true,
wantRecovery: true,
wantInjected: true,
},
} {
t.Run(test.name, func(t *testing.T) {
dir := t.TempDir()
fake := newFakeRunner()
hooks := defaultLifecycleHooks
test.configure(fake, &hooks)
statePath := filepath.Join(dir, "restart-state.json")
_, err := restartWithHooks(context.Background(), fake, RestartRequest{
StatePath: statePath,
UpdateStatePath: filepath.Join(dir, "update-state.json"),
Confirm: true,
}, hooks)
if err == nil {
t.Fatal("restartWithHooks() error = nil, want injected boundary failure")
}
var recovery *RecoveryRequiredError
if got := errors.As(err, &recovery); got != test.wantRecovery {
t.Fatalf("recovery-required = %t, want %t; error = %v", got, test.wantRecovery, err)
}
if test.wantInjected && !errors.Is(err, injected) {
t.Fatalf("restartWithHooks() error = %v, want injected durability cause", err)
}
state, stateErr := readState(statePath)
if stateErr != nil {
t.Fatalf("readState() error = %v", stateErr)
}
if state.Phase != test.wantPhase || state.MutationStarted != test.wantMutation {
t.Fatalf("state = %+v, want phase=%q mutation=%t", state, test.wantPhase, test.wantMutation)
}
if fake.recreated != test.wantRecreated {
t.Fatalf("recreated = %t, want %t", fake.recreated, test.wantRecreated)
}
if fake.maintenance != test.wantMaintenance {
t.Fatalf("maintenance = %t, want %t", fake.maintenance, test.wantMaintenance)
}
})
}
}
func TestRestartRejectsImageConfigurationAndMountDrift(t *testing.T) {
for _, test := range []struct {
failure string
want error
}{
{failure: "image-drift", want: errRestartImageDrift},
{failure: "config-drift", want: errRestartConfigurationDrift},
{failure: "mount-drift", want: errRestartMountDrift},
} {
t.Run(test.failure, func(t *testing.T) {
dir := t.TempDir()
fake := newFakeRunner()
fake.fail = test.failure
statePath := filepath.Join(dir, "restart-state.json")
_, err := Restart(context.Background(), fake, RestartRequest{
StatePath: statePath,
UpdateStatePath: filepath.Join(dir, "update-state.json"),
Confirm: true,
})
if !errors.Is(err, test.want) {
t.Fatalf("Restart() error = %v, want errors.Is(..., %v)", err, test.want)
}
if !fake.maintenance {
t.Fatal("maintenance was cleared after restart identity drift")
}
if _, stateErr := os.Stat(statePath); stateErr != nil {
t.Fatalf("restart recovery state missing after drift: %v", stateErr)
}
})
}
}
func TestRestartPreservesVerifiedUpdateState(t *testing.T) {
dir := t.TempDir()
fake := newFakeRunner()
updateStatePath := filepath.Join(dir, "update-state.json")
writeStateForTest(t, updateStatePath, State{
Transaction: "verified-update",
Phase: PhaseVerified,
Target: Target{Version: fake.version, Source: string(BuildSource)},
Previous: stateImageForTest(t, fake),
})
before := readStateBytes(t, updateStatePath)
if _, err := Restart(context.Background(), fake, RestartRequest{
StatePath: filepath.Join(dir, "restart-state.json"),
UpdateStatePath: updateStatePath,
Confirm: true,
}); err != nil {
t.Fatal(err)
}
after := readStateBytes(t, updateStatePath)
if string(after) != string(before) {
t.Fatal("restart changed verified update rollback metadata")
}
}