fix: finalize durable Pi lifecycle

2026-08-04 21:34:08 +02:00
parent 5ba2821a1b
commit a368889838
20 changed files with 1016 additions and 147 deletions
@@ -54,6 +54,9 @@ func TestUpdateBuildsPinnedVersionRecreatesOnlyCoreAndPersistsRecoveryState(t *t
 	if got := string(readStateBytes(t, result.StatePath)); strings.Contains(got, "llm.example.invalid") {
 		t.Fatalf("state = %q, want an endpoint-free configuration digest", got)
 	}
+	if selected := readSelectorReference(t, currentImageOverridePath(result.StatePath)); selected != fake.buildReference {
+		t.Fatalf("durable selector = %q, want verified candidate %q", selected, fake.buildReference)
+	}
 }

 func TestUpdateUsesATransactionScopedComposeOverrideWithoutMutatingTheConfiguredImage(t *testing.T) {
@@ -69,16 +72,46 @@ func TestUpdateUsesATransactionScopedComposeOverrideWithoutMutatingTheConfigured
 	if matches, err := filepath.Glob(filepath.Join(filepath.Dir(statePath), "pi-lifecycle-*.yaml")); err != nil || len(matches) != 0 {
 		t.Fatalf("terminal lifecycle overrides = %v, error = %v; want none", matches, err)
 	}
+	if _, err := os.Stat(currentImageOverridePath(statePath)); err != nil {
+		t.Fatalf("durable current-image override missing: %v", err)
+	}
+}
+
+func TestSuccessfulUpdateAndRollbackRemainSelectedOnFreshRecreate(t *testing.T) {
+	fake := newFakeRunner()
+	statePath := filepath.Join(t.TempDir(), ".thothctl", "update-state.json")
+	if _, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true}); err != nil {
+		t.Fatal(err)
+	}
+	fake.currentImage = "sha256:old"
+	if err := recreateCore(context.Background(), composeOverrideRunner{Runner: fake, path: currentImageOverridePath(statePath)}); err != nil {
+		t.Fatal(err)
+	}
+	if fake.currentImage != "sha256:candidate" {
+		t.Fatalf("fresh recreate image = %q, want verified candidate", fake.currentImage)
+	}
+	if _, err := Rollback(context.Background(), fake, statePath, true); err != nil {
+		t.Fatal(err)
+	}
+	fake.currentImage = "sha256:candidate"
+	if err := recreateCore(context.Background(), composeOverrideRunner{Runner: fake, path: currentImageOverridePath(statePath)}); err != nil {
+		t.Fatal(err)
+	}
+	if fake.currentImage != "sha256:old" {
+		t.Fatalf("fresh recreate after rollback image = %q, want previous image", fake.currentImage)
+	}
 }

 func TestTwoInstallationsSharingAConfiguredTagUseDifferentLifecycleTags(t *testing.T) {
 	first, second := newFakeRunner(), newFakeRunner()
+	firstPath := filepath.Join(t.TempDir(), "one", "state.json")
+	secondPath := filepath.Join(t.TempDir(), "two", "state.json")
 	for _, item := range []struct {
 		fake *fakeRunner
 		path string
 	}{
-		{first, filepath.Join(t.TempDir(), "one", "state.json")},
-		{second, filepath.Join(t.TempDir(), "two", "state.json")},
+		{first, firstPath},
+		{second, secondPath},
 	} {
 		if _, err := Update(context.Background(), item.fake, Request{StatePath: item.path, Version: "0.81.0", Source: BuildSource, Confirm: true}); err != nil {
 			t.Fatal(err)
@@ -87,15 +120,28 @@ func TestTwoInstallationsSharingAConfiguredTagUseDifferentLifecycleTags(t *testi
 	if first.buildReference == second.buildReference {
 		t.Fatalf("installations reused lifecycle tag %q", first.buildReference)
 	}
+	firstSelector := readSelectorReference(t, currentImageOverridePath(firstPath))
+	secondSelector := readSelectorReference(t, currentImageOverridePath(secondPath))
+	if firstSelector == secondSelector || firstSelector != first.buildReference || secondSelector != second.buildReference {
+		t.Fatalf("installation selectors = %q / %q, want isolated lifecycle references", firstSelector, secondSelector)
+	}
 }

 func TestDigestPinnedConfiguredImageIsNeverUsedAsARollbackTagTarget(t *testing.T) {
 	fake := newFakeRunner()
 	fake.configuredImage = "registry.example.invalid/core@sha256:" + strings.Repeat("b", 64)
 	fake.tags = map[string]string{fake.configuredImage: "sha256:old"}
-	fake.fail = "health"
-	_, _ = Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true})
+	statePath := filepath.Join(t.TempDir(), ".thothctl", "state.json")
+	if _, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := Rollback(context.Background(), fake, statePath, true); err != nil {
+		t.Fatal(err)
+	}
 	assertNotCalled(t, fake.calls, "image tag sha256:old "+fake.configuredImage)
+	if selected := readSelectorReference(t, currentImageOverridePath(statePath)); !strings.Contains(selected, "-previous") {
+		t.Fatalf("rollback selector = %q, want transaction previous tag for digest-pinned base", selected)
+	}
 }

 func TestMaintenanceLostResponsesAreResolvedByStatusAndEveryRecreateStartsGated(t *testing.T) {
@@ -189,7 +235,7 @@ func TestUpdateRollsBackAfterPostRecreateFailures(t *testing.T) {
 }

 func TestEveryRecoveryStateWriteFailureIsHandledTransactionally(t *testing.T) {
-	for failAt := 1; failAt <= 4; failAt++ {
+	for failAt := 1; failAt <= 6; failAt++ {
 		t.Run(fmt.Sprintf("write-%d", failAt), func(t *testing.T) {
 			fake := newFakeRunner()
 			writes := 0
@@ -210,8 +256,14 @@ func TestEveryRecoveryStateWriteFailureIsHandledTransactionally(t *testing.T) {
 			if fake.currentImage != "sha256:old" {
 				t.Fatalf("current image = %q, want restored previous", fake.currentImage)
 			}
-			if failAt > 1 && result.Phase != PhaseRolledBack {
-				t.Fatalf("phase = %q, want rolled_back", result.Phase)
+			wantPhase := Phase("")
+			if failAt == 2 || failAt == 3 {
+				wantPhase = PhaseFailed
+			} else if failAt >= 4 {
+				wantPhase = PhaseRolledBack
+			}
+			if result.Phase != wantPhase {
+				t.Fatalf("phase = %q, want %q for write %d", result.Phase, wantPhase, failAt)
 			}
 			if fake.maintenance {
 				t.Fatal("maintenance remained active after proven stable recovery")
@@ -227,7 +279,7 @@ func TestCompensationWriteFailureKeepsMaintenanceActiveForExplicitRecovery(t *te
 	hooks := defaultLifecycleHooks
 	hooks.writeState = func(path string, state State) error {
 		writes++
-		if writes == 4 {
+		if writes == 5 {
 			return errors.New("injected compensation state write failure")
 		}
 		return writeState(path, state)
@@ -264,12 +316,15 @@ func TestRecoverMaintenanceClearsOnlyAfterTerminalStateAndVerifiedSmoke(t *testi
 	fake.maintenance = true
 	statePath := filepath.Join(t.TempDir(), "state.json")
 	previous := stateImageForTest(t, fake)
-	state := State{Transaction: "recover-test", Phase: PhaseVerified, Previous: previous}
+	state := State{Transaction: "recover-test", Phase: PhaseRolledBack, MutationStarted: true, Previous: previous}
 	writeStateForTest(t, statePath, state)
 	overridePath := lifecycleOverridePath(statePath, state.Transaction)
 	if err := writeLifecycleOverride(overridePath, previous.Reference); err != nil {
 		t.Fatal(err)
 	}
+	if err := writeLifecycleOverride(currentImageOverridePath(statePath), previous.Reference); err != nil {
+		t.Fatal(err)
+	}

 	if err := RecoverMaintenance(context.Background(), fake, statePath, true); err != nil {
 		t.Fatalf("RecoverMaintenance() error = %v", err)
@@ -280,6 +335,9 @@ func TestRecoverMaintenanceClearsOnlyAfterTerminalStateAndVerifiedSmoke(t *testi
 	if _, err := os.Stat(overridePath); !errors.Is(err, os.ErrNotExist) {
 		t.Fatalf("lifecycle override still exists: %v", err)
 	}
+	if selected := readSelectorReference(t, currentImageOverridePath(statePath)); selected != previous.Reference {
+		t.Fatalf("maintenance cleanup changed durable selector to %q", selected)
+	}
 	assertCalled(t, fake.calls, "/models")
 	assertCalled(t, fake.calls, "/settings")
 }
@@ -288,7 +346,7 @@ func TestRecoverMaintenanceRefusesPendingTransaction(t *testing.T) {
 	fake := newFakeRunner()
 	fake.maintenance = true
 	statePath := filepath.Join(t.TempDir(), "state.json")
-	writeStateForTest(t, statePath, State{Phase: PhaseRecreated, Previous: stateImageForTest(t, fake)})
+	writeStateForTest(t, statePath, State{Phase: PhaseRecreated, MutationStarted: true, Previous: stateImageForTest(t, fake)})

 	err := RecoverMaintenance(context.Background(), fake, statePath, true)
 	if !errors.Is(err, ErrInterruptedUpdate) {
@@ -299,6 +357,43 @@ func TestRecoverMaintenanceRefusesPendingTransaction(t *testing.T) {
 	}
 }

+func TestRecoverMaintenanceCompletesAnInterruptedDurablePromotion(t *testing.T) {
+	fake := newFakeRunner()
+	fake.maintenance = true
+	fake.currentImage = "sha256:candidate"
+	fake.version = "0.81.0"
+	fake.expectedVersion = "0.81.0"
+	fake.labelVersion = "0.81.0"
+	statePath := filepath.Join(t.TempDir(), ".thothctl", "update-state.json")
+	previous := stateImageForTest(t, newFakeRunner())
+	candidate := previous
+	candidate.ID = "sha256:candidate"
+	candidate.Reference = "thothii-core:thothctl-recover-candidate"
+	fake.tags[candidate.Reference] = candidate.ID
+	state := State{
+		Transaction:     "promotion-recovery",
+		Phase:           PhasePromoting,
+		MutationStarted: true,
+		Target:          Target{Version: "0.81.0", Source: string(BuildSource)},
+		Previous:        previous,
+		Candidate:       candidate,
+	}
+	writeStateForTest(t, statePath, state)
+	if err := writeLifecycleOverride(lifecycleOverridePath(statePath, state.Transaction), candidate.Reference); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := RecoverMaintenance(context.Background(), fake, statePath, true); err != nil {
+		t.Fatalf("RecoverMaintenance() promotion error = %v", err)
+	}
+	if selected := readSelectorReference(t, currentImageOverridePath(statePath)); selected != candidate.Reference {
+		t.Fatalf("recovered selector = %q, want %q", selected, candidate.Reference)
+	}
+	if recovered, err := readState(statePath); err != nil || recovered.Phase != PhaseVerified {
+		t.Fatalf("recovered state = %+v, %v; want verified", recovered, err)
+	}
+}
+
 func TestRollbackFinalStateWriteFailureKeepsMaintenanceAndOverrideForRecovery(t *testing.T) {
 	fake := newFakeRunner()
 	statePath := filepath.Join(t.TempDir(), "state.json")
@@ -320,13 +415,13 @@ func TestRollbackFinalStateWriteFailureKeepsMaintenanceAndOverrideForRecovery(t
 	if !fake.maintenance {
 		t.Fatal("maintenance was cleared without durable rollback finalization")
 	}
-	if _, err := os.Stat(lifecycleOverridePath(statePath, "rollback-test")); err != nil {
-		t.Fatalf("recovery override was not preserved: %v", err)
+	if selected := readSelectorReference(t, currentImageOverridePath(statePath)); selected != previous.Reference {
+		t.Fatalf("durable rollback selector = %q, want %q", selected, previous.Reference)
 	}
 }

-func TestUpdateDoesNotRecreateWhenPreflightOrBuildFails(t *testing.T) {
-	for _, failure := range []string{"preflight", "build"} {
+func TestUpdateDoesNotRecreateWhenPreflightFails(t *testing.T) {
+	for _, failure := range []string{"preflight"} {
 		t.Run(failure, func(t *testing.T) {
 			fake := newFakeRunner()
 			fake.fail = failure
@@ -337,16 +432,67 @@ func TestUpdateDoesNotRecreateWhenPreflightOrBuildFails(t *testing.T) {
 			if failure == "preflight" && result.Phase == PhaseRolledBack {
 				t.Fatalf("preflight failure unexpectedly rolled back: %+v", result)
 			}
-			if failure == "build" && result.Phase != PhaseRolledBack {
-				t.Fatalf("candidate build failure must compensate: %+v", result)
+			assertNotCalled(t, fake.calls, "force-recreate")
+		})
+	}
+}
+
+func TestCandidateBuildAndPullFailuresRemainPreMutationAndNeverRecreateCore(t *testing.T) {
+	for _, testCase := range []struct {
+		name    string
+		source  Source
+		image   string
+		failure string
+	}{
+		{name: "build", source: BuildSource, failure: "build"},
+		{name: "pull", source: PullSource, image: "registry.example.invalid/core@sha256:" + strings.Repeat("a", 64), failure: "pull"},
+		{name: "candidate tag", source: PullSource, image: "registry.example.invalid/core@sha256:" + strings.Repeat("b", 64), failure: "tag"},
+	} {
+		t.Run(testCase.name, func(t *testing.T) {
+			fake := newFakeRunner()
+			fake.fail = testCase.failure
+			statePath := filepath.Join(t.TempDir(), ".thothctl", "update-state.json")
+			result, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: testCase.source, Image: testCase.image, Confirm: true})
+			if err == nil {
+				t.Fatal("Update() error = nil, want preparation failure")
 			}
-			if failure == "preflight" {
-				assertNotCalled(t, fake.calls, "force-recreate")
+			if result.Phase != PhaseFailed {
+				t.Fatalf("phase = %q, want safe failed preparation", result.Phase)
+			}
+			state, stateErr := readState(statePath)
+			if stateErr != nil {
+				t.Fatal(stateErr)
+			}
+			if state.MutationStarted {
+				t.Fatal("preparation failure recorded mutationStarted")
+			}
+			assertNotCalled(t, fake.calls, "force-recreate")
+			if fake.maintenance {
+				t.Fatal("maintenance remained active after safe preparation failure")
 			}
 		})
 	}
 }

+func TestSuccessfulCompensationPreservesTheOriginalTypedCause(t *testing.T) {
+	for _, cause := range []error{ErrActiveSessions, ErrInterruptedUpdate} {
+		fake := newFakeRunner()
+		fake.maintenance = true
+		statePath := filepath.Join(t.TempDir(), ".thothctl", "update-state.json")
+		state := State{Transaction: "typed-cause", Phase: PhaseRecreated, MutationStarted: true, Previous: stateImageForTest(t, fake)}
+		result, err, clear := compensate(context.Background(), fake, statePath, lifecycleOverridePath(statePath, state.Transaction), state, cause, defaultLifecycleHooks)
+		if result.Phase != PhaseRolledBack || !clear {
+			t.Fatalf("compensation = %+v, clear=%t; want successful rollback", result, clear)
+		}
+		if !errors.Is(err, cause) {
+			t.Fatalf("compensation error = %v, want errors.Is(..., %v)", err, cause)
+		}
+		if !strings.Contains(err.Error(), "previous core image was restored") {
+			t.Fatalf("compensation error = %v, want rollback-success report", err)
+		}
+	}
+}
+
 func TestUpdateRequiresConfirmationAndDrainsActiveSessions(t *testing.T) {
 	fake := newFakeRunner()
 	_, err := Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource})
@@ -402,7 +548,7 @@ func TestRollbackRestoresInterruptedOrPreviouslyRecordedState(t *testing.T) {
 func TestUpdateRefusesToOverwriteInterruptedRecoveryState(t *testing.T) {
 	fake := newFakeRunner()
 	statePath := filepath.Join(t.TempDir(), "state.json")
-	writeStateForTest(t, statePath, State{Phase: PhaseRecreated, Previous: Image{ID: "sha256:old", Reference: "thothii-core:local", MountFingerprint: mountFingerprint(nil)}})
+	writeStateForTest(t, statePath, State{Phase: PhaseRecreated, MutationStarted: true, Previous: Image{ID: "sha256:old", Reference: "thothii-core:local", MountFingerprint: mountFingerprint(nil)}})
 	_, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true})
 	if !errors.Is(err, ErrInterruptedUpdate) {
 		t.Fatalf("Update() error = %v, want interrupted update error", err)
@@ -441,6 +587,8 @@ type fakeRunner struct {
 	calls                         []string
 	fail                          string
 	version                       string
+	expectedVersion               string
+	labelVersion                  string
 	activeSessions                bool
 	built                         bool
 	currentImage                  string
@@ -460,7 +608,7 @@ type fakeRunner struct {

 func newFakeRunner() *fakeRunner {
 	return &fakeRunner{
-		version: "0.80.3", currentImage: "sha256:old", configuredImage: "thothii-core:local",
+		version: "0.80.3", expectedVersion: "0.80.3", labelVersion: "0.80.3", currentImage: "sha256:old", configuredImage: "thothii-core:local",
 		tags:          map[string]string{"thothii-core:local": "sha256:old"},
 		imageVersions: map[string]string{"sha256:old": "0.80.3"},
 	}
@@ -478,6 +626,12 @@ func (f *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose
 	if f.fail == "build" && containsArg(args, "build") {
 		return compose.Result{ExitCode: 1}, errors.New("build token=secret")
 	}
+	if f.fail == "pull" && len(args) > 0 && args[0] == "pull" {
+		return compose.Result{ExitCode: 1}, errors.New("pull token=secret")
+	}
+	if f.fail == "tag" && len(args) >= 4 && args[0] == "image" && args[1] == "tag" && strings.Contains(args[3], "-candidate") {
+		return compose.Result{ExitCode: 1}, errors.New("tag token=secret")
+	}
 	if f.fail == "health" && f.built && strings.Contains(call, "curl -fsS http://127.0.0.1:8787/health") {
 		return compose.Result{ExitCode: 1}, errors.New("health token=secret")
 	}
@@ -501,6 +655,8 @@ func (f *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose
 		return compose.Result{Stdout: "core-container\n"}, nil
 	case strings.Contains(call, "inspect --format {{.Image}}"):
 		return compose.Result{Stdout: f.currentImage + "\n"}, nil
+	case strings.Contains(call, "io.thothii.pi.version"):
+		return compose.Result{Stdout: f.labelVersion + "\n"}, nil
 	case strings.Contains(call, "inspect --format {{json .Mounts}}"):
 		if f.fail == "mount-drift" && f.currentImage == "sha256:candidate" {
 			return compose.Result{Stdout: `[{"Type":"volume","Name":"wrong-settings","Source":"wrong-settings","Destination":"/data/settings","RW":true}]`}, nil
@@ -580,6 +736,8 @@ func (f *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose
 		return compose.Result{}, nil
 	case strings.Contains(call, "pi --version"):
 		return compose.Result{Stdout: f.version + "\n"}, nil
+	case strings.Contains(call, "PI_VERSION"):
+		return compose.Result{Stdout: f.expectedVersion + "\n"}, nil
 	case strings.Contains(call, "/models"):
 		if f.modelsWire != "" {
 			return compose.Result{Stdout: f.modelsWire}, nil
@@ -604,7 +762,7 @@ func containsArg(args []string, wanted string) bool {

 func selectedCoreReference(args []string, fallback string) string {
 	for index := 0; index+1 < len(args); index++ {
-		if args[index] != "-f" || !strings.Contains(filepath.Base(args[index+1]), "pi-lifecycle-") {
+		if args[index] != "-f" || (!strings.Contains(filepath.Base(args[index+1]), "pi-lifecycle-") && filepath.Base(args[index+1]) != "current-image.yaml") {
 			continue
 		}
 		contents, err := os.ReadFile(args[index+1])
@@ -626,6 +784,27 @@ func selectedCoreReference(args []string, fallback string) string {
 	return fallback
 }

+func readSelectorReference(t *testing.T, path string) string {
+	t.Helper()
+	contents, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, line := range strings.Split(string(contents), "\n") {
+		line = strings.TrimSpace(line)
+		if !strings.HasPrefix(line, "image:") {
+			continue
+		}
+		value := strings.TrimSpace(strings.TrimPrefix(line, "image:"))
+		if decoded, err := strconv.Unquote(value); err == nil {
+			return decoded
+		}
+		return value
+	}
+	t.Fatalf("selector %s has no image", path)
+	return ""
+}
+
 func callIndex(calls []string, contains string) int {
 	for index, call := range calls {
 		if strings.Contains(call, contains) {