293 lines
9.8 KiB
Go
293 lines
9.8 KiB
Go
package pi
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestRestartRequiresConfirmationWithoutInvokingCompose(t *testing.T) {
|
|
dir := t.TempDir()
|
|
fake := newFakeRunner()
|
|
_, err := Restart(context.Background(), fake, RestartRequest{
|
|
StatePath: filepath.Join(dir, "restart-state.json"),
|
|
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
|
})
|
|
if !errors.Is(err, ErrConfirmationRequired) {
|
|
t.Fatalf("Restart() error = %v, want ErrConfirmationRequired", err)
|
|
}
|
|
assertNotCalled(t, fake.calls, "compose")
|
|
}
|
|
|
|
func TestRestartDrainsRecreatesOnlyCoreAndRetainsImage(t *testing.T) {
|
|
fake := newFakeRunner()
|
|
fake.activeSessions = true
|
|
dir := t.TempDir()
|
|
hooks := defaultLifecycleHooks
|
|
hooks.sleep = func(time.Duration) { fake.activeSessions = false }
|
|
|
|
result, err := restartWithHooks(context.Background(), fake, RestartRequest{
|
|
StatePath: filepath.Join(dir, "restart-state.json"),
|
|
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
|
Confirm: true,
|
|
Drain: true,
|
|
}, hooks)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.Version != fake.version {
|
|
t.Fatalf("version = %q, want %q", result.Version, fake.version)
|
|
}
|
|
assertCalled(t, fake.calls, "up --detach --wait --wait-timeout 45 --no-deps --force-recreate core")
|
|
assertNotCalled(t, fake.calls, "build --pull")
|
|
assertNotCalled(t, fake.calls, "pull ")
|
|
assertNotCalled(t, fake.calls, "frontend")
|
|
if _, err := os.Stat(result.StatePath); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("successful restart state still exists: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestRestartRefusesActiveSessionsWithoutDrain(t *testing.T) {
|
|
dir := t.TempDir()
|
|
fake := newFakeRunner()
|
|
fake.activeSessions = true
|
|
|
|
_, err := Restart(context.Background(), fake, RestartRequest{
|
|
StatePath: filepath.Join(dir, "restart-state.json"),
|
|
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
|
Confirm: true,
|
|
})
|
|
if !errors.Is(err, ErrActiveSessions) {
|
|
t.Fatalf("Restart() error = %v, want ErrActiveSessions", err)
|
|
}
|
|
if fake.maintenance {
|
|
t.Fatal("maintenance remained active after refusing pre-mutation restart")
|
|
}
|
|
if fake.recreated {
|
|
t.Fatal("core was recreated with active sessions")
|
|
}
|
|
}
|
|
|
|
func TestRestartRefusesInterruptedUpdateOrRestartState(t *testing.T) {
|
|
for _, stateFile := range []string{"update-state.json", "restart-state.json"} {
|
|
t.Run(stateFile, func(t *testing.T) {
|
|
dir := t.TempDir()
|
|
fake := newFakeRunner()
|
|
previous := stateImageForTest(t, fake)
|
|
writeStateForTest(t, filepath.Join(dir, stateFile), State{
|
|
Transaction: "interrupted",
|
|
Phase: PhaseRecreated,
|
|
Target: Target{Version: fake.version, Source: "restart"},
|
|
Previous: previous,
|
|
MutationStarted: true,
|
|
})
|
|
fake.calls = nil
|
|
|
|
_, err := Restart(context.Background(), fake, RestartRequest{
|
|
StatePath: filepath.Join(dir, "restart-state.json"),
|
|
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
|
Confirm: true,
|
|
})
|
|
if err == nil {
|
|
t.Fatal("Restart() accepted interrupted lifecycle state")
|
|
}
|
|
assertNotCalled(t, fake.calls, "compose")
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRestartPreflightFailureNeverRecreatesCoreAndClearsMaintenance(t *testing.T) {
|
|
dir := t.TempDir()
|
|
fake := newFakeRunner()
|
|
fake.fail = "preflight"
|
|
|
|
_, err := Restart(context.Background(), fake, RestartRequest{
|
|
StatePath: filepath.Join(dir, "restart-state.json"),
|
|
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
|
Confirm: true,
|
|
})
|
|
if err == nil {
|
|
t.Fatal("Restart() error = nil, want preflight failure")
|
|
}
|
|
if fake.recreated || fake.currentImage != "sha256:old" {
|
|
t.Fatalf("preflight failure mutated core: recreated=%t image=%q", fake.recreated, fake.currentImage)
|
|
}
|
|
if fake.maintenance {
|
|
t.Fatal("maintenance remained active after preflight failure")
|
|
}
|
|
}
|
|
|
|
func TestRestartPostRecreateFailureKeepsMaintenanceAndRecoveryState(t *testing.T) {
|
|
dir := t.TempDir()
|
|
fake := newFakeRunner()
|
|
fake.fail = "health"
|
|
statePath := filepath.Join(dir, "restart-state.json")
|
|
|
|
_, err := Restart(context.Background(), fake, RestartRequest{
|
|
StatePath: statePath,
|
|
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
|
Confirm: true,
|
|
})
|
|
var recovery *RecoveryRequiredError
|
|
if !errors.As(err, &recovery) {
|
|
t.Fatalf("Restart() error = %v, want RecoveryRequiredError", err)
|
|
}
|
|
if !fake.recreated {
|
|
t.Fatal("post-recreate failure occurred before core recreation")
|
|
}
|
|
if !fake.maintenance {
|
|
t.Fatal("maintenance was cleared after post-recreate failure")
|
|
}
|
|
state, stateErr := readState(statePath)
|
|
if stateErr != nil || !state.MutationStarted {
|
|
t.Fatalf("restart recovery state = %+v, %v; want durable mutation state", state, stateErr)
|
|
}
|
|
}
|
|
|
|
func TestRestartMaintenanceClearFailureRestoresRecoveryState(t *testing.T) {
|
|
dir := t.TempDir()
|
|
fake := newFakeRunner()
|
|
fake.fail = "maintenance-clear"
|
|
statePath := filepath.Join(dir, "restart-state.json")
|
|
|
|
_, err := Restart(context.Background(), fake, RestartRequest{
|
|
StatePath: statePath,
|
|
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
|
Confirm: true,
|
|
})
|
|
var recovery *RecoveryRequiredError
|
|
if !errors.As(err, &recovery) {
|
|
t.Fatalf("Restart() error = %v, want RecoveryRequiredError", err)
|
|
}
|
|
if !fake.maintenance {
|
|
t.Fatal("maintenance was cleared despite deactivation failure")
|
|
}
|
|
state, stateErr := readState(statePath)
|
|
if stateErr != nil || state.Phase != PhaseVerified || !state.MutationStarted {
|
|
t.Fatalf("restart recovery state = %+v, %v; want durable verified mutation state", state, stateErr)
|
|
}
|
|
}
|
|
|
|
func TestRecoverLifecycleMaintenanceVerifiesAndClearsRestartState(t *testing.T) {
|
|
dir := t.TempDir()
|
|
fake := newFakeRunner()
|
|
fake.maintenance = true
|
|
previous := stateImageForTest(t, fake)
|
|
restartStatePath := filepath.Join(dir, "restart-state.json")
|
|
updateStatePath := filepath.Join(dir, "update-state.json")
|
|
writeStateForTest(t, restartStatePath, State{
|
|
Transaction: "restart-recovery",
|
|
Phase: PhaseRecreated,
|
|
Target: Target{Version: fake.version, Source: "restart"},
|
|
Previous: previous,
|
|
MutationStarted: true,
|
|
})
|
|
candidate := previous
|
|
candidate.Reference = "thothii-core:thothctl-recover-candidate"
|
|
writeStateForTest(t, updateStatePath, State{
|
|
Transaction: "update-recovery",
|
|
Phase: PhasePromoting,
|
|
Target: Target{Version: fake.version, Source: string(BuildSource)},
|
|
Previous: previous,
|
|
Candidate: candidate,
|
|
MutationStarted: true,
|
|
})
|
|
if err := writeLifecycleOverride(currentImageOverridePath(updateStatePath), candidate.Reference); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fake.calls = nil
|
|
|
|
if err := RecoverLifecycleMaintenance(context.Background(), fake, updateStatePath, restartStatePath, true); err != nil {
|
|
t.Fatalf("RecoverLifecycleMaintenance() error = %v", err)
|
|
}
|
|
if _, err := os.Stat(restartStatePath); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("restart recovery state still exists: %v", err)
|
|
}
|
|
if fake.maintenance {
|
|
t.Fatal("maintenance remained active after both lifecycle states were verified")
|
|
}
|
|
updateState, err := readState(updateStatePath)
|
|
if err != nil || updateState.Phase != PhaseVerified {
|
|
t.Fatalf("update recovery state = %+v, %v; want verified image rollback metadata", updateState, err)
|
|
}
|
|
deactivate := callIndex(fake.calls, "/internal/maintenance/deactivate")
|
|
lastVerification := lastCallIndexBefore(fake.calls, "/pi-management/test", deactivate)
|
|
if deactivate < 0 || lastVerification < 0 {
|
|
t.Fatalf("calls = %v; want verification before maintenance deactivation", fake.calls)
|
|
}
|
|
verificationCount := 0
|
|
for index := 0; index < deactivate; index++ {
|
|
if strings.Contains(fake.calls[index], "/internal/maintenance/deactivate") {
|
|
t.Fatalf("maintenance reopened before combined verification: %v", fake.calls)
|
|
}
|
|
if strings.Contains(fake.calls[index], "/pi-management/test") {
|
|
verificationCount++
|
|
}
|
|
}
|
|
if verificationCount < 3 {
|
|
t.Fatalf("verification calls before maintenance deactivation = %d, want restart, update, and final proofs: %v", verificationCount, fake.calls)
|
|
}
|
|
}
|
|
|
|
func TestRestartRejectsImageConfigurationAndMountDrift(t *testing.T) {
|
|
for _, test := range []struct {
|
|
failure string
|
|
want error
|
|
}{
|
|
{failure: "image-drift", want: errRestartImageDrift},
|
|
{failure: "config-drift", want: errRestartConfigurationDrift},
|
|
{failure: "mount-drift", want: errRestartMountDrift},
|
|
} {
|
|
t.Run(test.failure, func(t *testing.T) {
|
|
dir := t.TempDir()
|
|
fake := newFakeRunner()
|
|
fake.fail = test.failure
|
|
statePath := filepath.Join(dir, "restart-state.json")
|
|
|
|
_, err := Restart(context.Background(), fake, RestartRequest{
|
|
StatePath: statePath,
|
|
UpdateStatePath: filepath.Join(dir, "update-state.json"),
|
|
Confirm: true,
|
|
})
|
|
if !errors.Is(err, test.want) {
|
|
t.Fatalf("Restart() error = %v, want errors.Is(..., %v)", err, test.want)
|
|
}
|
|
if !fake.maintenance {
|
|
t.Fatal("maintenance was cleared after restart identity drift")
|
|
}
|
|
if _, stateErr := os.Stat(statePath); stateErr != nil {
|
|
t.Fatalf("restart recovery state missing after drift: %v", stateErr)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRestartPreservesVerifiedUpdateState(t *testing.T) {
|
|
dir := t.TempDir()
|
|
fake := newFakeRunner()
|
|
updateStatePath := filepath.Join(dir, "update-state.json")
|
|
writeStateForTest(t, updateStatePath, State{
|
|
Transaction: "verified-update",
|
|
Phase: PhaseVerified,
|
|
Target: Target{Version: fake.version, Source: string(BuildSource)},
|
|
Previous: stateImageForTest(t, fake),
|
|
})
|
|
before := readStateBytes(t, updateStatePath)
|
|
|
|
if _, err := Restart(context.Background(), fake, RestartRequest{
|
|
StatePath: filepath.Join(dir, "restart-state.json"),
|
|
UpdateStatePath: updateStatePath,
|
|
Confirm: true,
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
after := readStateBytes(t, updateStatePath)
|
|
if string(after) != string(before) {
|
|
t.Fatal("restart changed verified update rollback metadata")
|
|
}
|
|
}
|