fix: harden embedded Pi lifecycle recovery

2026-08-04 23:14:47 +02:00
parent a368889838
commit 935bb1db0e
15 changed files with 572 additions and 101 deletions
@@ -102,6 +102,38 @@ func TestSuccessfulUpdateAndRollbackRemainSelectedOnFreshRecreate(t *testing.T)
 	}
 }

+func TestSelectorPromotionDoesNotMaskPostRenameDirectoryFsyncFailure(t *testing.T) {
+	directory := t.TempDir()
+	source := filepath.Join(directory, "candidate.yaml")
+	destination := filepath.Join(directory, "current-image.yaml")
+	if err := writeLifecycleOverride(source, "thothii-core:candidate"); err != nil {
+		t.Fatal(err)
+	}
+	if err := writeLifecycleOverride(destination, "thothii-core:old"); err != nil {
+		t.Fatal(err)
+	}
+
+	err := promoteLifecycleOverrideWith(
+		source,
+		destination,
+		"thothii-core:candidate",
+		func(source, destination, _ string) error {
+			if err := os.Rename(source, destination); err != nil {
+				return err
+			}
+			return errors.New("injected post-rename directory fsync failure")
+		},
+	)
+
+	var recovery interface{ RecoveryRequired() bool }
+	if err == nil || !errors.As(err, &recovery) || !recovery.RecoveryRequired() {
+		t.Fatalf("promotion error = %v; want typed recovery-required result", err)
+	}
+	if selected := readSelectorReference(t, destination); selected != "thothii-core:candidate" {
+		t.Fatalf("immediate selector = %q, want landed candidate bytes", selected)
+	}
+}
+
 func TestTwoInstallationsSharingAConfiguredTagUseDifferentLifecycleTags(t *testing.T) {
 	first, second := newFakeRunner(), newFakeRunner()
 	firstPath := filepath.Join(t.TempDir(), "one", "state.json")
@@ -162,6 +194,21 @@ func TestMaintenanceLostResponsesAreResolvedByStatusAndEveryRecreateStartsGated(
 	}
 }

+func TestMaintenanceReconciliationDoesNotMaskExplicitDurabilityFailure(t *testing.T) {
+	fake := newFakeRunner()
+	fake.fail = "maintenance-activate-durability"
+
+	err := setMaintenance(context.Background(), fake, true)
+
+	var recovery interface{ RecoveryRequired() bool }
+	if err == nil || !errors.As(err, &recovery) || !recovery.RecoveryRequired() {
+		t.Fatalf("maintenance error = %v; want typed recovery-required result", err)
+	}
+	if !fake.maintenance {
+		t.Fatal("safe marker state was not retained after activation durability failure")
+	}
+}
+
 func TestCompensationReactivatesMaintenanceAndRescansBeforeRollback(t *testing.T) {
 	fake := newFakeRunner()
 	fake.fail = "version"
@@ -179,6 +226,63 @@ func TestCompensationReactivatesMaintenanceAndRescansBeforeRollback(t *testing.T
 	}
 }

+func TestAutomaticRollbackSurvivesADeadCandidateCore(t *testing.T) {
+	fake := newFakeRunner()
+	fake.fail = "dead-candidate"
+	statePath := filepath.Join(t.TempDir(), ".thothctl", "update-state.json")
+
+	result, err := Update(context.Background(), fake, Request{
+		StatePath: statePath,
+		Version:   "0.81.0",
+		Source:    BuildSource,
+		Confirm:   true,
+	})
+
+	if err == nil || result.Phase != PhaseRolledBack {
+		t.Fatalf("Update() = %+v, %v; want automatic rollback after dead candidate", result, err)
+	}
+	if fake.currentImage != "sha256:old" || !fake.coreRunning {
+		t.Fatalf("restored core = image:%q running:%t; want previous running image", fake.currentImage, fake.coreRunning)
+	}
+	if fake.execFailuresWhileStopped == 0 {
+		t.Fatal("fake did not exercise candidate exec failure")
+	}
+	assertCalled(t, fake.calls, "run --rm --no-deps --entrypoint node")
+	assertMaintenanceClearedAfterRestoredProof(t, fake)
+}
+
+func TestManualRollbackSurvivesADeadCandidateCore(t *testing.T) {
+	fake := newFakeRunner()
+	statePath := filepath.Join(t.TempDir(), ".thothctl", "update-state.json")
+	previous := stateImageForTest(t, fake)
+	previous.Reference = "thothii-core:thothctl-dead-candidate-previous"
+	fake.tags[previous.Reference] = previous.ID
+	writeStateForTest(t, statePath, State{
+		Transaction:     "dead-candidate",
+		Phase:           PhaseRecreated,
+		MutationStarted: true,
+		Previous:        previous,
+	})
+	fake.currentImage = "sha256:candidate"
+	fake.version = "0.81.0"
+	fake.coreRunning = false
+	fake.maintenance = false
+
+	result, err := Rollback(context.Background(), fake, statePath, true)
+
+	if err != nil || result.Phase != PhaseRolledBack {
+		t.Fatalf("Rollback() = %+v, %v; want restored previous core", result, err)
+	}
+	if fake.currentImage != "sha256:old" || !fake.coreRunning {
+		t.Fatalf("restored core = image:%q running:%t; want previous running image", fake.currentImage, fake.coreRunning)
+	}
+	if fake.execFailuresWhileStopped == 0 {
+		t.Fatal("fake did not exercise candidate exec failure")
+	}
+	assertCalled(t, fake.calls, "run --rm --no-deps --entrypoint node")
+	assertMaintenanceClearedAfterRestoredProof(t, fake)
+}
+
 func TestUpdatePullsOnlyDigestPinnedSource(t *testing.T) {
 	fake := newFakeRunner()
 	digest := "registry.example.invalid/thothii-core@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
@@ -234,6 +338,36 @@ func TestUpdateRollsBackAfterPostRecreateFailures(t *testing.T) {
 	}
 }

+func TestCandidateVerificationRejectsEveryDeclaredVersionBoundaryMismatch(t *testing.T) {
+	for _, boundary := range []string{"executable", "environment", "image-label"} {
+		t.Run(boundary, func(t *testing.T) {
+			fake := newFakeRunner()
+			switch boundary {
+			case "executable":
+				fake.candidateVersion = "0.80.9"
+			case "environment":
+				fake.candidateExpectedVersion = "0.80.9"
+			case "image-label":
+				fake.candidateLabelVersion = "0.80.9"
+			}
+
+			result, err := Update(context.Background(), fake, Request{
+				StatePath: filepath.Join(t.TempDir(), "state.json"),
+				Version:   "0.81.0",
+				Source:    BuildSource,
+				Confirm:   true,
+			})
+
+			if err == nil || result.Phase != PhaseRolledBack {
+				t.Fatalf("Update() = %+v, %v; want rollback for candidate %s mismatch", result, err, boundary)
+			}
+			if fake.currentImage != "sha256:old" {
+				t.Fatalf("current image = %q, want restored previous image", fake.currentImage)
+			}
+		})
+	}
+}
+
 func TestEveryRecoveryStateWriteFailureIsHandledTransactionally(t *testing.T) {
 	for failAt := 1; failAt <= 6; failAt++ {
 		t.Run(fmt.Sprintf("write-%d", failAt), func(t *testing.T) {
@@ -604,19 +738,35 @@ type fakeRunner struct {
 	dropMaintenanceAfterCandidate bool
 	modelsWire                    string
 	rollbackPrepared              bool
+	coreRunning                   bool
+	execFailuresWhileStopped      int
+	maintenanceHelperImages       []string
+	maintenanceClearImages        []string
+	restoredProofComplete         bool
+	candidateVersion              string
+	candidateExpectedVersion      string
+	candidateLabelVersion         string
 }

 func newFakeRunner() *fakeRunner {
 	return &fakeRunner{
 		version: "0.80.3", expectedVersion: "0.80.3", labelVersion: "0.80.3", currentImage: "sha256:old", configuredImage: "thothii-core:local",
-		tags:          map[string]string{"thothii-core:local": "sha256:old"},
-		imageVersions: map[string]string{"sha256:old": "0.80.3"},
+		tags:                     map[string]string{"thothii-core:local": "sha256:old"},
+		imageVersions:            map[string]string{"sha256:old": "0.80.3"},
+		coreRunning:              true,
+		candidateVersion:         "0.81.0",
+		candidateExpectedVersion: "0.81.0",
+		candidateLabelVersion:    "0.81.0",
 	}
 }

 func (f *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
 	call := strings.Join(args, " ")
 	f.calls = append(f.calls, call)
+	if !f.coreRunning && containsArg(args, "exec") {
+		f.execFailuresWhileStopped++
+		return compose.Result{ExitCode: 1}, errors.New("core service is not running")
+	}
 	if f.built && f.fail != "compensation" && strings.Contains(call, "image tag sha256:old") {
 		f.fail = ""
 	}
@@ -651,6 +801,11 @@ func (f *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose
 			endpoint = "https://drift.example.invalid"
 		}
 		return compose.Result{Stdout: `{"services":{"core":{"image":"` + selectedCoreReference(args, f.configuredImage) + `","environment":{"THT_LLM_URL":"` + endpoint + `"}}}}`}, nil
+	case strings.Contains(call, "ps --status running -q core"):
+		if f.coreRunning {
+			return compose.Result{Stdout: "core-container\n"}, nil
+		}
+		return compose.Result{}, nil
 	case strings.Contains(call, "ps -q core"):
 		return compose.Result{Stdout: "core-container\n"}, nil
 	case strings.Contains(call, "inspect --format {{.Image}}"):
@@ -667,6 +822,9 @@ func (f *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose
 		return compose.Result{Stdout: `[{"Type":"volume","Name":"settings","Source":"settings","Destination":"/data/settings","RW":true},{"Type":"volume","Name":"pi-state","Source":"pi-state","Destination":"/home/thoth/.pi","RW":true},{"Type":"volume","Name":"sessions","Source":"sessions","Destination":"/data/sessions","RW":true},{"Type":"volume","Name":"workspace-registry","Source":"workspace-registry","Destination":"/data/workspace-registry","RW":true}]`}, nil
 	case strings.Contains(call, "/internal/maintenance/activate"):
 		f.maintenance = true
+		if f.fail == "maintenance-activate-durability" {
+			return compose.Result{ExitCode: 22}, errors.New("maintenance activation durability was not acknowledged")
+		}
 		if f.lostMaintenanceResponse == "activate" {
 			f.lostMaintenanceResponse = ""
 			return compose.Result{ExitCode: 52}, errors.New("lost activation response")
@@ -677,12 +835,16 @@ func (f *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose
 			return compose.Result{ExitCode: 53}, errors.New("maintenance clear failure")
 		}
 		f.maintenance = false
+		f.maintenanceClearImages = append(f.maintenanceClearImages, f.currentImage)
 		if f.lostMaintenanceResponse == "deactivate" {
 			f.lostMaintenanceResponse = ""
 			return compose.Result{ExitCode: 52}, errors.New("lost deactivation response")
 		}
 		return compose.Result{Stdout: `{"active":false,"admissions":0}`}, nil
 	case strings.Contains(call, "/internal/maintenance/status"):
+		if f.fail == "maintenance-activate-durability" {
+			return compose.Result{Stdout: fmt.Sprintf(`{"active":%t,"admissions":0,"recoveryRequired":true}`, f.maintenance)}, nil
+		}
 		return compose.Result{Stdout: fmt.Sprintf(`{"active":%t,"admissions":0}`, f.maintenance)}, nil
 	case strings.Contains(call, "/sessions?scope=all"):
 		if f.sessionsWire != "" {
@@ -693,6 +855,17 @@ func (f *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose
 			return compose.Result{Stdout: `[{"status":"open","archived":false}]`}, nil
 		}
 		return compose.Result{Stdout: `[]`}, nil
+	case containsArg(args, "run") && containsArg(args, "--entrypoint") && containsArg(args, "node"):
+		reference := selectedCoreReference(args, f.configuredImage)
+		f.maintenanceHelperImages = append(f.maintenanceHelperImages, reference)
+		if reference == "" {
+			return compose.Result{ExitCode: 1}, errors.New("maintenance helper has no selected image")
+		}
+		if f.tags[reference] != "sha256:old" {
+			return compose.Result{ExitCode: 1}, errors.New("maintenance helper did not select the previous image")
+		}
+		f.maintenance = true
+		return compose.Result{}, nil
 	case containsArg(args, "build"):
 		f.built = true
 		f.buildReference = selectedCoreReference(args, f.configuredImage)
@@ -723,6 +896,21 @@ func (f *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose
 		if version, ok := f.imageVersions[f.currentImage]; ok {
 			f.version = version
 		}
+		if f.currentImage == "sha256:candidate" {
+			f.version = f.candidateVersion
+			f.expectedVersion = f.candidateExpectedVersion
+			f.labelVersion = f.candidateLabelVersion
+		} else if f.currentImage == "sha256:old" {
+			f.expectedVersion = "0.80.3"
+			f.labelVersion = "0.80.3"
+		}
+		f.coreRunning = true
+		if f.currentImage == "sha256:candidate" {
+			f.restoredProofComplete = false
+		}
+		if f.fail == "dead-candidate" && f.currentImage == "sha256:candidate" {
+			f.coreRunning = false
+		}
 		if f.dropMaintenanceAfterCandidate && f.currentImage == "sha256:candidate" {
 			f.maintenance = false
 			f.dropMaintenanceAfterCandidate = false
@@ -744,6 +932,9 @@ func (f *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose
 		}
 		return compose.Result{Stdout: `{"models":[{"id":"model","provider":"provider"}]}`}, nil
 	case strings.Contains(call, "/settings"):
+		if f.currentImage == "sha256:old" {
+			f.restoredProofComplete = true
+		}
 		return compose.Result{Stdout: `{"provider":"provider","model":"model","thinking":"medium"}`}, nil
 	case strings.Contains(call, "/health"):
 		return compose.Result{Stdout: `{"status":"ok"}`}, nil
@@ -840,6 +1031,25 @@ func assertNotCalled(t *testing.T, calls []string, prohibited string) {
 		}
 	}
 }
+
+func assertMaintenanceClearedAfterRestoredProof(t *testing.T, fake *fakeRunner) {
+	t.Helper()
+	if fake.maintenance {
+		t.Fatal("maintenance remained active after restored-core proof")
+	}
+	if !fake.restoredProofComplete {
+		t.Fatal("maintenance cleared before restored settings smoke completed")
+	}
+	if len(fake.maintenanceClearImages) == 0 {
+		t.Fatal("maintenance was never durably cleared")
+	}
+	for _, image := range fake.maintenanceClearImages {
+		if image != "sha256:old" {
+			t.Fatalf("maintenance cleared while image %q was selected; want previous image", image)
+		}
+	}
+}
+
 func readStateBytes(t *testing.T, path string) []byte {
 	t.Helper()
 	contents, err := os.ReadFile(path)