package pi import ( "context" "errors" "os" "path/filepath" "strings" "testing" "time" ) func TestRestartRequiresConfirmationWithoutInvokingCompose(t *testing.T) { dir := t.TempDir() fake := newFakeRunner() _, err := Restart(context.Background(), fake, RestartRequest{ StatePath: filepath.Join(dir, "restart-state.json"), UpdateStatePath: filepath.Join(dir, "update-state.json"), }) if !errors.Is(err, ErrConfirmationRequired) { t.Fatalf("Restart() error = %v, want ErrConfirmationRequired", err) } if got, want := err.Error(), "restart requires --yes after reviewing the planned Pi core recreation"; got != want { t.Fatalf("Restart() error text = %q, want %q", got, want) } assertNotCalled(t, fake.calls, "compose") } func TestRestartDrainsRecreatesOnlyCoreAndRetainsImage(t *testing.T) { fake := newFakeRunner() fake.sessionsWire = `[{"status":"open","archived":false}]` dir := t.TempDir() hooks := defaultLifecycleHooks sleepCalls := 0 hooks.sleep = func(duration time.Duration) { sleepCalls++ if duration != time.Second { t.Fatalf("drain sleep = %s, want %s", duration, time.Second) } fake.sessionsWire = `[]` } result, err := restartWithHooks(context.Background(), fake, RestartRequest{ StatePath: filepath.Join(dir, "restart-state.json"), UpdateStatePath: filepath.Join(dir, "update-state.json"), Confirm: true, Drain: true, }, hooks) if err != nil { t.Fatal(err) } if result.Version != fake.version { t.Fatalf("version = %q, want %q", result.Version, fake.version) } if sleepCalls != 1 { t.Fatalf("drain sleep calls = %d, want 1", sleepCalls) } assertCalled(t, fake.calls, "up --detach --wait --wait-timeout 45 --no-deps --force-recreate --no-build --pull never core") assertNotCalled(t, fake.calls, "compose build --pull") for _, call := range fake.calls { if strings.HasPrefix(call, "pull ") { t.Fatalf("restart invoked direct image pull: %s", call) } } assertNotCalled(t, fake.calls, "frontend") if _, err := os.Stat(result.StatePath); !errors.Is(err, os.ErrNotExist) { t.Fatalf("successful restart state still exists: %v", err) } } func TestRestartPinsCapturedImageWhenConfiguredTagMovesBeforeRecreate(t *testing.T) { dir := t.TempDir() fake := newFakeRunner() hooks := defaultLifecycleHooks write := hooks.writeState hooks.writeState = func(path string, state State) error { if state.Phase == PhasePreflight && state.MutationStarted { fake.tags[fake.configuredImage] = "sha256:moved-configured-tag" fake.imageVersions["sha256:moved-configured-tag"] = "9.99.0" } return write(path, state) } restartStatePath := filepath.Join(dir, "restart-state.json") result, err := restartWithHooks(context.Background(), fake, RestartRequest{ StatePath: restartStatePath, UpdateStatePath: filepath.Join(dir, "update-state.json"), Confirm: true, }, hooks) if err != nil { t.Fatalf("restartWithHooks() error = %v", err) } if result.Version != "0.80.3" || fake.currentImage != "sha256:old" { t.Fatalf("restart result=%+v image=%q; want captured 0.80.3 / sha256:old", result, fake.currentImage) } assertCalled(t, fake.calls, "image tag sha256:old thothii-core:tht-") assertCalled(t, fake.calls, "pi-lifecycle-") if matches, globErr := filepath.Glob(filepath.Join(dir, "pi-lifecycle-*.yaml")); globErr != nil || len(matches) != 0 { t.Fatalf("successful restart overrides = %v, error = %v; want safe cleanup", matches, globErr) } } func TestRestartRefusesActiveSessionsWithoutDrain(t *testing.T) { dir := t.TempDir() fake := newFakeRunner() fake.activeSessions = true _, err := Restart(context.Background(), fake, RestartRequest{ StatePath: filepath.Join(dir, "restart-state.json"), UpdateStatePath: filepath.Join(dir, "update-state.json"), Confirm: true, }) if !errors.Is(err, ErrActiveSessions) { t.Fatalf("Restart() error = %v, want ErrActiveSessions", err) } if got, want := err.Error(), "active sessions must be drained before restarting Pi; use --drain only after they are complete"; got != want { t.Fatalf("Restart() error text = %q, want %q", got, want) } if fake.maintenance { t.Fatal("maintenance remained active after refusing pre-mutation restart") } if fake.recreated { t.Fatal("core was recreated with active sessions") } } func TestRestartActivationFailureClearsPreMutationMaintenance(t *testing.T) { for _, failure := range []string{ "maintenance-activate-durability", "maintenance-activate-durability-without-status-flag", } { t.Run(failure, func(t *testing.T) { dir := t.TempDir() fake := newFakeRunner() fake.fail = failure _, err := Restart(context.Background(), fake, RestartRequest{ StatePath: filepath.Join(dir, "restart-state.json"), UpdateStatePath: filepath.Join(dir, "update-state.json"), Confirm: true, }) var recovery *RecoveryRequiredError if !errors.As(err, &recovery) { t.Fatalf("Restart() error = %v, want RecoveryRequiredError", err) } if fake.maintenance { t.Fatal("pre-mutation activation failure left maintenance active") } if fake.recreated { t.Fatal("pre-mutation activation failure recreated core") } assertCalled(t, fake.calls, "operator-command.js maintenance-activate") assertCalled(t, fake.calls, "operator-command.js maintenance-deactivate") }) } } func TestRestartRefusesInterruptedUpdateOrRestartState(t *testing.T) { for _, stateFile := range []string{"update-state.json", "restart-state.json"} { t.Run(stateFile, func(t *testing.T) { dir := t.TempDir() fake := newFakeRunner() previous := stateImageForTest(t, fake) writeStateForTest(t, filepath.Join(dir, stateFile), State{ Transaction: "interrupted", Phase: PhaseRecreated, Target: Target{Version: fake.version, Source: "restart"}, Previous: previous, MutationStarted: true, }) fake.calls = nil _, err := Restart(context.Background(), fake, RestartRequest{ StatePath: filepath.Join(dir, "restart-state.json"), UpdateStatePath: filepath.Join(dir, "update-state.json"), Confirm: true, }) if err == nil { t.Fatal("Restart() accepted interrupted lifecycle state") } assertNotCalled(t, fake.calls, "compose") }) } } func TestRestartPreflightFailureNeverRecreatesCoreAndClearsMaintenance(t *testing.T) { dir := t.TempDir() fake := newFakeRunner() fake.fail = "preflight" _, err := Restart(context.Background(), fake, RestartRequest{ StatePath: filepath.Join(dir, "restart-state.json"), UpdateStatePath: filepath.Join(dir, "update-state.json"), Confirm: true, }) if err == nil { t.Fatal("Restart() error = nil, want preflight failure") } if fake.recreated || fake.currentImage != "sha256:old" { t.Fatalf("preflight failure mutated core: recreated=%t image=%q", fake.recreated, fake.currentImage) } if fake.maintenance { t.Fatal("maintenance remained active after preflight failure") } } func TestRestartPostRecreateFailureKeepsMaintenanceAndRecoveryState(t *testing.T) { dir := t.TempDir() fake := newFakeRunner() fake.fail = "health" statePath := filepath.Join(dir, "restart-state.json") _, err := Restart(context.Background(), fake, RestartRequest{ StatePath: statePath, UpdateStatePath: filepath.Join(dir, "update-state.json"), Confirm: true, }) var recovery *RecoveryRequiredError if !errors.As(err, &recovery) { t.Fatalf("Restart() error = %v, want RecoveryRequiredError", err) } if !fake.recreated { t.Fatal("post-recreate failure occurred before core recreation") } if !fake.maintenance { t.Fatal("maintenance was cleared after post-recreate failure") } state, stateErr := readState(statePath) if stateErr != nil || !state.MutationStarted { t.Fatalf("restart recovery state = %+v, %v; want durable mutation state", state, stateErr) } overridePath := lifecycleOverridePath(statePath, state.Transaction) selected, overrideErr := readLifecycleOverride(overridePath) if overrideErr != nil || selected != state.Previous.Reference || fake.tags[selected] != state.Previous.ID { t.Fatalf("restart override = %q, %v; want retained exact image %q", selected, overrideErr, state.Previous.ID) } } func TestRestartMaintenanceClearFailureRestoresRecoveryState(t *testing.T) { dir := t.TempDir() fake := newFakeRunner() fake.fail = "maintenance-clear" statePath := filepath.Join(dir, "restart-state.json") _, err := Restart(context.Background(), fake, RestartRequest{ StatePath: statePath, UpdateStatePath: filepath.Join(dir, "update-state.json"), Confirm: true, }) var recovery *RecoveryRequiredError if !errors.As(err, &recovery) { t.Fatalf("Restart() error = %v, want RecoveryRequiredError", err) } if !fake.maintenance { t.Fatal("maintenance was cleared despite deactivation failure") } state, stateErr := readState(statePath) if stateErr != nil || state.Phase != PhaseVerified || !state.MutationStarted { t.Fatalf("restart recovery state = %+v, %v; want durable verified mutation state", state, stateErr) } } func TestRecoverLifecycleMaintenanceVerifiesAndClearsRestartState(t *testing.T) { dir := t.TempDir() fake := newFakeRunner() fake.maintenance = true previous := stateImageForTest(t, fake) restartStatePath := filepath.Join(dir, "restart-state.json") updateStatePath := filepath.Join(dir, "update-state.json") writeStateForTest(t, restartStatePath, State{ Transaction: "restart-recovery", Phase: PhaseRecreated, Target: Target{Version: fake.version, Source: "restart"}, Previous: previous, MutationStarted: true, }) restartState, err := readState(restartStatePath) if err != nil { t.Fatal(err) } restartOverride := lifecycleOverridePath(restartStatePath, restartState.Transaction) if err := writeLifecycleOverride(restartOverride, restartState.Previous.Reference); err != nil { t.Fatal(err) } delete(fake.tags, restartState.Previous.Reference) fake.tags[fake.configuredImage] = "sha256:moved-before-recovery" candidate := previous candidate.Reference = "thothii-core:tht-recover-candidate" writeStateForTest(t, updateStatePath, State{ Transaction: "update-recovery", Phase: PhasePromoting, Target: Target{Version: fake.version, Source: string(BuildSource)}, Previous: previous, Candidate: candidate, MutationStarted: true, }) if err := writeLifecycleOverride(currentImageOverridePath(updateStatePath), candidate.Reference); err != nil { t.Fatal(err) } fake.calls = nil if err := RecoverLifecycleMaintenance(context.Background(), fake, updateStatePath, restartStatePath, true); err != nil { t.Fatalf("RecoverLifecycleMaintenance() error = %v", err) } if _, err := os.Stat(restartStatePath); !errors.Is(err, os.ErrNotExist) { t.Fatalf("restart recovery state still exists: %v", err) } if _, err := os.Stat(restartOverride); !errors.Is(err, os.ErrNotExist) { t.Fatalf("restart recovery override still exists: %v", err) } if fake.tags[restartState.Previous.Reference] != restartState.Previous.ID { t.Fatalf("restart recovery pin = %q, want %q", fake.tags[restartState.Previous.Reference], restartState.Previous.ID) } assertCalled(t, fake.calls, restartOverride) if fake.maintenance { t.Fatal("maintenance remained active after both lifecycle states were verified") } updateState, err := readState(updateStatePath) if err != nil || updateState.Phase != PhaseVerified { t.Fatalf("update recovery state = %+v, %v; want verified image rollback metadata", updateState, err) } deactivate := callIndex(fake.calls, "operator-command.js maintenance-deactivate") lastVerification := lastCallIndexBefore(fake.calls, "operator-command.js pi-test", deactivate) if deactivate < 0 || lastVerification < 0 { t.Fatalf("calls = %v; want verification before maintenance deactivation", fake.calls) } verificationCount := 0 for index := 0; index < deactivate; index++ { if strings.Contains(fake.calls[index], "operator-command.js maintenance-deactivate") { t.Fatalf("maintenance reopened before combined verification: %v", fake.calls) } if strings.Contains(fake.calls[index], "operator-command.js pi-test") { verificationCount++ } } if verificationCount < 3 { t.Fatalf("verification calls before maintenance deactivation = %d, want restart, update, and final proofs: %v", verificationCount, fake.calls) } } func TestRecoverLifecycleMaintenanceRejectsMalformedRestartState(t *testing.T) { for _, test := range []struct { name string phase Phase source string }{ {name: "recreated_without_mutation_marker", phase: PhaseRecreated, source: "restart"}, {name: "non_restart_source", phase: PhasePreflight, source: string(BuildSource)}, } { t.Run(test.name, func(t *testing.T) { dir := t.TempDir() fake := newFakeRunner() fake.maintenance = true restartStatePath := filepath.Join(dir, "restart-state.json") updateStatePath := filepath.Join(dir, "update-state.json") writeStateForTest(t, restartStatePath, State{ Transaction: "malformed-restart", Phase: test.phase, Target: Target{Version: fake.version, Source: test.source}, Previous: stateImageForTest(t, fake), }) fake.calls = nil err := RecoverLifecycleMaintenance( context.Background(), fake, updateStatePath, restartStatePath, true, ) if !errors.Is(err, ErrInvalidRequest) || !strings.Contains(err.Error(), "restart recovery state is invalid") { t.Fatalf("RecoverLifecycleMaintenance() error = %v, want invalid restart recovery state", err) } if !fake.maintenance { t.Fatal("malformed restart state reopened admission") } if _, stateErr := os.Stat(restartStatePath); stateErr != nil { t.Fatalf("malformed restart state was removed: %v", stateErr) } assertNotCalled(t, fake.calls, "operator-command.js maintenance-deactivate") }) } } func TestRestartRefusesMalformedRestartStateWithoutInvokingCompose(t *testing.T) { dir := t.TempDir() fake := newFakeRunner() restartStatePath := filepath.Join(dir, "restart-state.json") writeStateForTest(t, restartStatePath, State{ Transaction: "malformed-restart", Phase: PhasePreflight, Target: Target{Version: fake.version, Source: string(BuildSource)}, Previous: stateImageForTest(t, fake), }) fake.calls = nil _, err := Restart(context.Background(), fake, RestartRequest{ StatePath: restartStatePath, UpdateStatePath: filepath.Join(dir, "update-state.json"), Confirm: true, }) if !errors.Is(err, ErrInvalidRequest) || !strings.Contains(err.Error(), "restart recovery state is invalid") { t.Fatalf("Restart() error = %v, want invalid restart recovery state", err) } if _, stateErr := os.Stat(restartStatePath); stateErr != nil { t.Fatalf("malformed restart state was removed: %v", stateErr) } assertNotCalled(t, fake.calls, "compose") } func TestRestartDurabilityFailureBoundaries(t *testing.T) { injected := errors.New("injected restart durability failure") for _, test := range []struct { name string configure func(*fakeRunner, *lifecycleHooks) wantPhase Phase wantMutation bool wantRecreated bool wantMaintenance bool wantRecovery bool wantInjected bool }{ { name: "mutation_marker_write", configure: func(_ *fakeRunner, hooks *lifecycleHooks) { write := hooks.writeState hooks.writeState = func(path string, state State) error { if state.Phase == PhasePreflight && state.MutationStarted { return injected } return write(path, state) } }, wantPhase: PhasePreflight, wantInjected: true, }, { name: "core_recreation", configure: func(fake *fakeRunner, _ *lifecycleHooks) { fake.fail = "recreate" }, wantPhase: PhasePreflight, wantMutation: true, wantRecreated: true, wantMaintenance: true, wantRecovery: true, }, { name: "maintenance_proof", configure: func(fake *fakeRunner, _ *lifecycleHooks) { fake.fail = "maintenance-proof" }, wantPhase: PhasePreflight, wantMutation: true, wantRecreated: true, wantMaintenance: true, wantRecovery: true, }, { name: "recreated_phase_write", configure: func(_ *fakeRunner, hooks *lifecycleHooks) { write := hooks.writeState hooks.writeState = func(path string, state State) error { if state.Phase == PhaseRecreated { return injected } return write(path, state) } }, wantPhase: PhasePreflight, wantMutation: true, wantRecreated: true, wantMaintenance: true, wantRecovery: true, wantInjected: true, }, { name: "verified_phase_write", configure: func(_ *fakeRunner, hooks *lifecycleHooks) { write := hooks.writeState hooks.writeState = func(path string, state State) error { if state.Phase == PhaseVerified { return injected } return write(path, state) } }, wantPhase: PhaseRecreated, wantMutation: true, wantRecreated: true, wantMaintenance: true, wantRecovery: true, wantInjected: true, }, { name: "restart_state_removal", configure: func(_ *fakeRunner, hooks *lifecycleHooks) { hooks.removeFile = func(string) error { return injected } }, wantPhase: PhaseVerified, wantMutation: true, wantRecreated: true, wantMaintenance: true, wantRecovery: true, wantInjected: true, }, } { t.Run(test.name, func(t *testing.T) { dir := t.TempDir() fake := newFakeRunner() hooks := defaultLifecycleHooks test.configure(fake, &hooks) statePath := filepath.Join(dir, "restart-state.json") _, err := restartWithHooks(context.Background(), fake, RestartRequest{ StatePath: statePath, UpdateStatePath: filepath.Join(dir, "update-state.json"), Confirm: true, }, hooks) if err == nil { t.Fatal("restartWithHooks() error = nil, want injected boundary failure") } var recovery *RecoveryRequiredError if got := errors.As(err, &recovery); got != test.wantRecovery { t.Fatalf("recovery-required = %t, want %t; error = %v", got, test.wantRecovery, err) } if test.wantInjected && !errors.Is(err, injected) { t.Fatalf("restartWithHooks() error = %v, want injected durability cause", err) } state, stateErr := readState(statePath) if stateErr != nil { t.Fatalf("readState() error = %v", stateErr) } if state.Phase != test.wantPhase || state.MutationStarted != test.wantMutation { t.Fatalf("state = %+v, want phase=%q mutation=%t", state, test.wantPhase, test.wantMutation) } if fake.recreated != test.wantRecreated { t.Fatalf("recreated = %t, want %t", fake.recreated, test.wantRecreated) } if fake.maintenance != test.wantMaintenance { t.Fatalf("maintenance = %t, want %t", fake.maintenance, test.wantMaintenance) } }) } } func TestRestartRejectsImageConfigurationAndMountDrift(t *testing.T) { for _, test := range []struct { failure string want error }{ {failure: "image-drift", want: errRestartImageDrift}, {failure: "config-drift", want: errRestartConfigurationDrift}, {failure: "mount-drift", want: errRestartMountDrift}, } { t.Run(test.failure, func(t *testing.T) { dir := t.TempDir() fake := newFakeRunner() fake.fail = test.failure statePath := filepath.Join(dir, "restart-state.json") _, err := Restart(context.Background(), fake, RestartRequest{ StatePath: statePath, UpdateStatePath: filepath.Join(dir, "update-state.json"), Confirm: true, }) if !errors.Is(err, test.want) { t.Fatalf("Restart() error = %v, want errors.Is(..., %v)", err, test.want) } if !fake.maintenance { t.Fatal("maintenance was cleared after restart identity drift") } if _, stateErr := os.Stat(statePath); stateErr != nil { t.Fatalf("restart recovery state missing after drift: %v", stateErr) } }) } } func TestRestartPreservesVerifiedUpdateState(t *testing.T) { dir := t.TempDir() fake := newFakeRunner() updateStatePath := filepath.Join(dir, "update-state.json") writeStateForTest(t, updateStatePath, State{ Transaction: "verified-update", Phase: PhaseVerified, Target: Target{Version: fake.version, Source: string(BuildSource)}, Previous: stateImageForTest(t, fake), }) before := readStateBytes(t, updateStatePath) if _, err := Restart(context.Background(), fake, RestartRequest{ StatePath: filepath.Join(dir, "restart-state.json"), UpdateStatePath: updateStatePath, Confirm: true, }); err != nil { t.Fatal(err) } after := readStateBytes(t, updateStatePath) if string(after) != string(before) { t.Fatal("restart changed verified update rollback metadata") } }