fix: harden Pi lifecycle recovery

2026-08-04 20:28:09 +02:00
parent 5b3ce93e31
commit 5ba2821a1b
24 changed files with 1764 additions and 384 deletions
@@ -7,12 +7,29 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"testing"

 	"github.com/aritmolab/thothii/tools/thothctl/internal/compose"
 )

+func TestActiveSessionsParsesAuthenticatedBackendBareArrayFixture(t *testing.T) {
+	contents, err := os.ReadFile(filepath.Join("..", "..", "..", "..", "backend", "test", "fixtures", "sessions-scope-all.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	fake := newFakeRunner()
+	fake.sessionsWire = string(contents)
+	active, err := activeSessions(context.Background(), fake)
+	if err != nil {
+		t.Fatalf("activeSessions() error = %v", err)
+	}
+	if !active {
+		t.Fatal("activeSessions() = false, want open session from backend wire fixture")
+	}
+}
+
 func TestUpdateBuildsPinnedVersionRecreatesOnlyCoreAndPersistsRecoveryState(t *testing.T) {
 	fake := newFakeRunner()
 	dir := t.TempDir()
@@ -28,8 +45,8 @@ func TestUpdateBuildsPinnedVersionRecreatesOnlyCoreAndPersistsRecoveryState(t *t
 	if result.Phase != PhaseVerified {
 		t.Fatalf("phase = %q, want %q", result.Phase, PhaseVerified)
 	}
-	assertCalled(t, fake.calls, "compose build --pull --build-arg PI_VERSION=0.81.0 core")
-	assertCalled(t, fake.calls, "compose up --detach --wait --wait-timeout 45 --no-deps --force-recreate core")
+	assertCalled(t, fake.calls, "build --pull --build-arg PI_VERSION=0.81.0 core")
+	assertCalled(t, fake.calls, "up --detach --wait --wait-timeout 45 --no-deps --force-recreate core")
 	assertNotCalled(t, fake.calls, "frontend")
 	if got := string(readStateBytes(t, result.StatePath)); strings.Contains(got, "secret") || !strings.Contains(got, `"phase": "verified"`) {
 		t.Fatalf("state = %q, want credential-free verified metadata", got)
@@ -39,15 +56,93 @@ func TestUpdateBuildsPinnedVersionRecreatesOnlyCoreAndPersistsRecoveryState(t *t
 	}
 }

+func TestUpdateUsesATransactionScopedComposeOverrideWithoutMutatingTheConfiguredImage(t *testing.T) {
+	fake := newFakeRunner()
+	statePath := filepath.Join(t.TempDir(), ".thothctl", "update-state.json")
+	if _, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true}); err != nil {
+		t.Fatal(err)
+	}
+	if fake.buildReference == "" || fake.buildReference == fake.configuredImage || !strings.Contains(fake.buildReference, "thothctl-") {
+		t.Fatalf("build reference = %q, configured = %q; want unique lifecycle tag", fake.buildReference, fake.configuredImage)
+	}
+	assertNotCalled(t, fake.calls, "image tag sha256:old "+fake.configuredImage)
+	if matches, err := filepath.Glob(filepath.Join(filepath.Dir(statePath), "pi-lifecycle-*.yaml")); err != nil || len(matches) != 0 {
+		t.Fatalf("terminal lifecycle overrides = %v, error = %v; want none", matches, err)
+	}
+}
+
+func TestTwoInstallationsSharingAConfiguredTagUseDifferentLifecycleTags(t *testing.T) {
+	first, second := newFakeRunner(), newFakeRunner()
+	for _, item := range []struct {
+		fake *fakeRunner
+		path string
+	}{
+		{first, filepath.Join(t.TempDir(), "one", "state.json")},
+		{second, filepath.Join(t.TempDir(), "two", "state.json")},
+	} {
+		if _, err := Update(context.Background(), item.fake, Request{StatePath: item.path, Version: "0.81.0", Source: BuildSource, Confirm: true}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if first.buildReference == second.buildReference {
+		t.Fatalf("installations reused lifecycle tag %q", first.buildReference)
+	}
+}
+
+func TestDigestPinnedConfiguredImageIsNeverUsedAsARollbackTagTarget(t *testing.T) {
+	fake := newFakeRunner()
+	fake.configuredImage = "registry.example.invalid/core@sha256:" + strings.Repeat("b", 64)
+	fake.tags = map[string]string{fake.configuredImage: "sha256:old"}
+	fake.fail = "health"
+	_, _ = Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true})
+	assertNotCalled(t, fake.calls, "image tag sha256:old "+fake.configuredImage)
+}
+
+func TestMaintenanceLostResponsesAreResolvedByStatusAndEveryRecreateStartsGated(t *testing.T) {
+	for _, lost := range []string{"activate", "deactivate"} {
+		t.Run(lost, func(t *testing.T) {
+			fake := newFakeRunner()
+			fake.lostMaintenanceResponse = lost
+			if _, err := Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true}); err != nil {
+				t.Fatal(err)
+			}
+			assertCalled(t, fake.calls, "/internal/maintenance/status")
+			for index, active := range fake.maintenanceAtRecreate {
+				if !active {
+					t.Fatalf("recreate %d started without durable maintenance", index+1)
+				}
+			}
+		})
+	}
+}
+
+func TestCompensationReactivatesMaintenanceAndRescansBeforeRollback(t *testing.T) {
+	fake := newFakeRunner()
+	fake.fail = "version"
+	fake.dropMaintenanceAfterCandidate = true
+	_, _ = Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true})
+	rollback := lastCallIndexBefore(fake.calls, "image tag sha256:old", len(fake.calls))
+	if rollback < 0 {
+		t.Fatalf("calls %v contain no rollback", fake.calls)
+	}
+	recreate := callIndex(fake.calls, "force-recreate core")
+	activate := lastCallIndexBefore(fake.calls, "/internal/maintenance/activate", rollback)
+	scan := lastCallIndexBefore(fake.calls, "/sessions?scope=all", rollback)
+	if activate <= recreate || scan <= recreate {
+		t.Fatalf("calls %v do not reactivate/confirm and rescan after candidate recreate before rollback", fake.calls)
+	}
+}
+
 func TestUpdatePullsOnlyDigestPinnedSource(t *testing.T) {
 	fake := newFakeRunner()
 	digest := "registry.example.invalid/thothii-core@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
 	_, err := Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: PullSource, Image: digest, Confirm: true})
-	if err == nil {
-		t.Fatal("Update() error = nil, want Pi version verification failure from unchanged fake image")
+	if err != nil {
+		t.Fatalf("Update() pull error = %v", err)
 	}
 	assertCalled(t, fake.calls, "pull "+digest)
-	assertCalled(t, fake.calls, "image tag "+digest+" thothii-core:local")
+	assertCalled(t, fake.calls, "image tag "+digest+" thothii-core:thothctl-")
+	assertNotCalled(t, fake.calls, "image tag "+digest+" thothii-core:local")

 	fake = newFakeRunner()
 	_, err = Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: PullSource, Image: "registry.example.invalid/thothii-core:latest", Confirm: true})
@@ -71,7 +166,7 @@ func TestUpdateIsNoOpWhenDesiredVersionAlreadyRuns(t *testing.T) {
 }

 func TestUpdateRollsBackAfterPostRecreateFailures(t *testing.T) {
-	for _, failure := range []string{"health", "version", "smoke"} {
+	for _, failure := range []string{"recreate", "health", "version", "smoke", "config-drift", "mount-drift"} {
 		t.Run(failure, func(t *testing.T) {
 			fake := newFakeRunner()
 			fake.fail = failure
@@ -83,11 +178,9 @@ func TestUpdateRollsBackAfterPostRecreateFailures(t *testing.T) {
 			if result.Phase != PhaseRolledBack {
 				t.Fatalf("phase = %q, want %q", result.Phase, PhaseRolledBack)
 			}
-			assertCalled(t, fake.calls, "image tag sha256:old thothii-core:local")
-			assertCalled(t, fake.calls, "compose up --detach --wait --wait-timeout 45 --no-deps --force-recreate core")
-			if strings.Join(fake.volumes, ",") != "settings,pi-state,sessions,workspace-registry" {
-				t.Fatalf("volumes changed: %v", fake.volumes)
-			}
+			assertCalled(t, fake.calls, "image tag sha256:old thothii-core:thothctl-")
+			assertNotCalled(t, fake.calls, "image tag sha256:old thothii-core:local")
+			assertCalled(t, fake.calls, "up --detach --wait --wait-timeout 45 --no-deps --force-recreate core")
 			if got := string(readStateBytes(t, statePath)); !strings.Contains(got, `"phase": "rolled_back"`) {
 				t.Fatalf("state = %q, want rollback metadata", got)
 			}
@@ -95,6 +188,143 @@ func TestUpdateRollsBackAfterPostRecreateFailures(t *testing.T) {
 	}
 }

+func TestEveryRecoveryStateWriteFailureIsHandledTransactionally(t *testing.T) {
+	for failAt := 1; failAt <= 4; failAt++ {
+		t.Run(fmt.Sprintf("write-%d", failAt), func(t *testing.T) {
+			fake := newFakeRunner()
+			writes := 0
+			hooks := defaultLifecycleHooks
+			hooks.writeState = func(path string, state State) error {
+				writes++
+				if writes == failAt {
+					return errors.New("injected state write failure")
+				}
+				return writeState(path, state)
+			}
+			result, err := updateWithHooks(context.Background(), fake, Request{
+				StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true,
+			}, hooks)
+			if err == nil {
+				t.Fatal("updateWithHooks() error = nil, want injected state failure")
+			}
+			if fake.currentImage != "sha256:old" {
+				t.Fatalf("current image = %q, want restored previous", fake.currentImage)
+			}
+			if failAt > 1 && result.Phase != PhaseRolledBack {
+				t.Fatalf("phase = %q, want rolled_back", result.Phase)
+			}
+			if fake.maintenance {
+				t.Fatal("maintenance remained active after proven stable recovery")
+			}
+		})
+	}
+}
+
+func TestCompensationWriteFailureKeepsMaintenanceActiveForExplicitRecovery(t *testing.T) {
+	fake := newFakeRunner()
+	fake.fail = "health"
+	writes := 0
+	hooks := defaultLifecycleHooks
+	hooks.writeState = func(path string, state State) error {
+		writes++
+		if writes == 4 {
+			return errors.New("injected compensation state write failure")
+		}
+		return writeState(path, state)
+	}
+	result, err := updateWithHooks(context.Background(), fake, Request{
+		StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true,
+	}, hooks)
+	if err == nil || result.Phase != PhaseFailed {
+		t.Fatalf("result=%+v error=%v, want failed recovery", result, err)
+	}
+	if !fake.maintenance {
+		t.Fatal("maintenance was cleared without durable rollback state")
+	}
+}
+
+func TestMaintenanceClearAndCompensationFailuresRemainGated(t *testing.T) {
+	for _, failure := range []string{"maintenance-clear", "compensation"} {
+		t.Run(failure, func(t *testing.T) {
+			fake := newFakeRunner()
+			fake.fail = failure
+			result, err := Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true})
+			if err == nil || result.Phase != PhaseFailed {
+				t.Fatalf("result=%+v error=%v", result, err)
+			}
+			if !fake.maintenance {
+				t.Fatal("maintenance was cleared after an unverified terminal failure")
+			}
+		})
+	}
+}
+
+func TestRecoverMaintenanceClearsOnlyAfterTerminalStateAndVerifiedSmoke(t *testing.T) {
+	fake := newFakeRunner()
+	fake.maintenance = true
+	statePath := filepath.Join(t.TempDir(), "state.json")
+	previous := stateImageForTest(t, fake)
+	state := State{Transaction: "recover-test", Phase: PhaseVerified, Previous: previous}
+	writeStateForTest(t, statePath, state)
+	overridePath := lifecycleOverridePath(statePath, state.Transaction)
+	if err := writeLifecycleOverride(overridePath, previous.Reference); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := RecoverMaintenance(context.Background(), fake, statePath, true); err != nil {
+		t.Fatalf("RecoverMaintenance() error = %v", err)
+	}
+	if fake.maintenance {
+		t.Fatal("maintenance remained active after verified terminal recovery")
+	}
+	if _, err := os.Stat(overridePath); !errors.Is(err, os.ErrNotExist) {
+		t.Fatalf("lifecycle override still exists: %v", err)
+	}
+	assertCalled(t, fake.calls, "/models")
+	assertCalled(t, fake.calls, "/settings")
+}
+
+func TestRecoverMaintenanceRefusesPendingTransaction(t *testing.T) {
+	fake := newFakeRunner()
+	fake.maintenance = true
+	statePath := filepath.Join(t.TempDir(), "state.json")
+	writeStateForTest(t, statePath, State{Phase: PhaseRecreated, Previous: stateImageForTest(t, fake)})
+
+	err := RecoverMaintenance(context.Background(), fake, statePath, true)
+	if !errors.Is(err, ErrInterruptedUpdate) {
+		t.Fatalf("RecoverMaintenance() error = %v, want ErrInterruptedUpdate", err)
+	}
+	if !fake.maintenance {
+		t.Fatal("pending transaction maintenance was cleared")
+	}
+}
+
+func TestRollbackFinalStateWriteFailureKeepsMaintenanceAndOverrideForRecovery(t *testing.T) {
+	fake := newFakeRunner()
+	statePath := filepath.Join(t.TempDir(), "state.json")
+	previous := stateImageForTest(t, fake)
+	previous.Reference = "thothii-core:thothctl-rollback-test-previous"
+	fake.tags[previous.Reference] = previous.ID
+	writeStateForTest(t, statePath, State{
+		Transaction: "rollback-test",
+		Phase:       PhaseRecreated,
+		Previous:    previous,
+	})
+	hooks := defaultLifecycleHooks
+	hooks.writeState = func(string, State) error { return errors.New("injected rollback state write failure") }
+
+	result, err := rollbackWithHooks(context.Background(), fake, statePath, true, hooks)
+	if err == nil || result.Phase != PhaseFailed {
+		t.Fatalf("rollbackWithHooks() = %+v, %v; want failed durable finalization", result, err)
+	}
+	if !fake.maintenance {
+		t.Fatal("maintenance was cleared without durable rollback finalization")
+	}
+	if _, err := os.Stat(lifecycleOverridePath(statePath, "rollback-test")); err != nil {
+		t.Fatalf("recovery override was not preserved: %v", err)
+	}
+}
+
 func TestUpdateDoesNotRecreateWhenPreflightOrBuildFails(t *testing.T) {
 	for _, failure := range []string{"preflight", "build"} {
 		t.Run(failure, func(t *testing.T) {
@@ -104,9 +334,15 @@ func TestUpdateDoesNotRecreateWhenPreflightOrBuildFails(t *testing.T) {
 			if err == nil {
 				t.Fatal("Update() error = nil, want failure")
 			}
-			if failure == "preflight" && result.Phase == PhaseRolledBack { t.Fatalf("preflight failure unexpectedly rolled back: %+v", result) }
-			if failure == "build" && result.Phase != PhaseRolledBack { t.Fatalf("candidate build failure must compensate: %+v", result) }
-			if failure == "preflight" { assertNotCalled(t, fake.calls, "force-recreate") }
+			if failure == "preflight" && result.Phase == PhaseRolledBack {
+				t.Fatalf("preflight failure unexpectedly rolled back: %+v", result)
+			}
+			if failure == "build" && result.Phase != PhaseRolledBack {
+				t.Fatalf("candidate build failure must compensate: %+v", result)
+			}
+			if failure == "preflight" {
+				assertNotCalled(t, fake.calls, "force-recreate")
+			}
 		})
 	}
 }
@@ -149,6 +385,8 @@ func TestRollbackRestoresInterruptedOrPreviouslyRecordedState(t *testing.T) {
 		t.Fatal(err)
 	}
 	previous.ConfigurationSHA = configured.ConfigurationSHA
+	previous.Reference = "thothii-core:thothctl-test-previous"
+	fake.tags[previous.Reference] = previous.ID
 	writeStateForTest(t, statePath, State{Version: 1, Phase: PhaseRecreated, Previous: previous})
 	result, err := Rollback(context.Background(), fake, statePath, true)
 	if err != nil {
@@ -157,14 +395,14 @@ func TestRollbackRestoresInterruptedOrPreviouslyRecordedState(t *testing.T) {
 	if result.Phase != PhaseRolledBack {
 		t.Fatalf("phase = %q, want %q", result.Phase, PhaseRolledBack)
 	}
-	assertCalled(t, fake.calls, "image tag sha256:old thothii-core:local")
-	assertCalled(t, fake.calls, "compose up --detach --wait --wait-timeout 45 --no-deps --force-recreate core")
+	assertCalled(t, fake.calls, "image tag sha256:old thothii-core:thothctl-test-previous")
+	assertCalled(t, fake.calls, "up --detach --wait --wait-timeout 45 --no-deps --force-recreate core")
 }

 func TestUpdateRefusesToOverwriteInterruptedRecoveryState(t *testing.T) {
 	fake := newFakeRunner()
 	statePath := filepath.Join(t.TempDir(), "state.json")
-	writeStateForTest(t, statePath, State{Phase: PhaseRecreated, Previous: Image{ID: "sha256:old", Reference: "thothii-core:local", Volumes: []string{"settings"}, MountFingerprint: mountFingerprint(nil)}})
+	writeStateForTest(t, statePath, State{Phase: PhaseRecreated, Previous: Image{ID: "sha256:old", Reference: "thothii-core:local", MountFingerprint: mountFingerprint(nil)}})
 	_, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true})
 	if !errors.Is(err, ErrInterruptedUpdate) {
 		t.Fatalf("Update() error = %v, want interrupted update error", err)
@@ -189,43 +427,63 @@ func TestRunningImageCapturesServerBindAndNamedMountIdentity(t *testing.T) {

 func TestCanonicalDigestReferenceRejectsCredentialsAndURLForms(t *testing.T) {
 	valid := "registry.example.invalid/thothii-core@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
-	if got, err := canonicalDigestReference(valid); err != nil || got != valid { t.Fatalf("canonicalDigestReference() = %q, %v", got, err) }
+	if got, err := canonicalDigestReference(valid); err != nil || got != valid {
+		t.Fatalf("canonicalDigestReference() = %q, %v", got, err)
+	}
 	for _, invalid := range []string{"https://registry.example.invalid/a@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "user:pass@registry.example/a@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "registry.example/a@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa?token=x"} {
-		if _, err := canonicalDigestReference(invalid); err == nil { t.Fatalf("accepted unsafe reference %q", invalid) }
+		if _, err := canonicalDigestReference(invalid); err == nil {
+			t.Fatalf("accepted unsafe reference %q", invalid)
+		}
 	}
 }

 type fakeRunner struct {
-	calls          []string
-	fail           string
-	version        string
-	activeSessions bool
-	built          bool
-	currentImage   string
-	volumes        []string
-	mountsJSON     string
+	calls                         []string
+	fail                          string
+	version                       string
+	activeSessions                bool
+	built                         bool
+	currentImage                  string
+	mountsJSON                    string
+	sessionsWire                  string
+	configuredImage               string
+	buildReference                string
+	tags                          map[string]string
+	imageVersions                 map[string]string
+	maintenance                   bool
+	maintenanceAtRecreate         []bool
+	lostMaintenanceResponse       string
+	dropMaintenanceAfterCandidate bool
+	modelsWire                    string
+	rollbackPrepared              bool
 }

 func newFakeRunner() *fakeRunner {
-	return &fakeRunner{version: "0.80.3", currentImage: "sha256:old", volumes: []string{"settings", "pi-state", "sessions", "workspace-registry"}}
+	return &fakeRunner{
+		version: "0.80.3", currentImage: "sha256:old", configuredImage: "thothii-core:local",
+		tags:          map[string]string{"thothii-core:local": "sha256:old"},
+		imageVersions: map[string]string{"sha256:old": "0.80.3"},
+	}
 }

 func (f *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
 	call := strings.Join(args, " ")
 	f.calls = append(f.calls, call)
-	if strings.Contains(call, "image tag sha256:old") {
+	if f.built && f.fail != "compensation" && strings.Contains(call, "image tag sha256:old") {
 		f.fail = ""
-		f.currentImage = "sha256:old"
 	}
 	if f.fail == "preflight" && strings.Contains(call, "config --format json") {
 		return compose.Result{ExitCode: 1}, errors.New("provider token=secret")
 	}
-	if f.fail == "build" && strings.Contains(call, "compose build") {
+	if f.fail == "build" && containsArg(args, "build") {
 		return compose.Result{ExitCode: 1}, errors.New("build token=secret")
 	}
 	if f.fail == "health" && f.built && strings.Contains(call, "curl -fsS http://127.0.0.1:8787/health") {
 		return compose.Result{ExitCode: 1}, errors.New("health token=secret")
 	}
+	if f.fail == "compensation" && f.built && !f.rollbackPrepared && strings.Contains(call, "curl -fsS http://127.0.0.1:8787/health") {
+		return compose.Result{ExitCode: 1}, errors.New("candidate health failure")
+	}
 	if f.fail == "version" && f.built && strings.Contains(call, "pi --version") && strings.Contains(call, "exec") {
 		return compose.Result{ExitCode: 1}, errors.New("version token=secret")
 	}
@@ -234,34 +492,98 @@ func (f *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose
 	}
 	switch {
 	case strings.Contains(call, "config --format json"):
-		return compose.Result{Stdout: `{"services":{"core":{"image":"thothii-core:local","environment":{"THT_LLM_URL":"https://llm.example.invalid"}}}}`}, nil
+		endpoint := "https://llm.example.invalid"
+		if f.fail == "config-drift" && f.currentImage == "sha256:candidate" {
+			endpoint = "https://drift.example.invalid"
+		}
+		return compose.Result{Stdout: `{"services":{"core":{"image":"` + selectedCoreReference(args, f.configuredImage) + `","environment":{"THT_LLM_URL":"` + endpoint + `"}}}}`}, nil
 	case strings.Contains(call, "ps -q core"):
 		return compose.Result{Stdout: "core-container\n"}, nil
 	case strings.Contains(call, "inspect --format {{.Image}}"):
 		return compose.Result{Stdout: f.currentImage + "\n"}, nil
 	case strings.Contains(call, "inspect --format {{json .Mounts}}"):
+		if f.fail == "mount-drift" && f.currentImage == "sha256:candidate" {
+			return compose.Result{Stdout: `[{"Type":"volume","Name":"wrong-settings","Source":"wrong-settings","Destination":"/data/settings","RW":true}]`}, nil
+		}
 		if f.mountsJSON != "" {
 			return compose.Result{Stdout: f.mountsJSON}, nil
 		}
 		return compose.Result{Stdout: `[{"Type":"volume","Name":"settings","Source":"settings","Destination":"/data/settings","RW":true},{"Type":"volume","Name":"pi-state","Source":"pi-state","Destination":"/home/thoth/.pi","RW":true},{"Type":"volume","Name":"sessions","Source":"sessions","Destination":"/data/sessions","RW":true},{"Type":"volume","Name":"workspace-registry","Source":"workspace-registry","Destination":"/data/workspace-registry","RW":true}]`}, nil
 	case strings.Contains(call, "/internal/maintenance/activate"):
+		f.maintenance = true
+		if f.lostMaintenanceResponse == "activate" {
+			f.lostMaintenanceResponse = ""
+			return compose.Result{ExitCode: 52}, errors.New("lost activation response")
+		}
 		return compose.Result{Stdout: `{"active":true,"admissions":0}`}, nil
 	case strings.Contains(call, "/internal/maintenance/deactivate"):
+		if f.fail == "maintenance-clear" {
+			return compose.Result{ExitCode: 53}, errors.New("maintenance clear failure")
+		}
+		f.maintenance = false
+		if f.lostMaintenanceResponse == "deactivate" {
+			f.lostMaintenanceResponse = ""
+			return compose.Result{ExitCode: 52}, errors.New("lost deactivation response")
+		}
 		return compose.Result{Stdout: `{"active":false,"admissions":0}`}, nil
+	case strings.Contains(call, "/internal/maintenance/status"):
+		return compose.Result{Stdout: fmt.Sprintf(`{"active":%t,"admissions":0}`, f.maintenance)}, nil
 	case strings.Contains(call, "/sessions?scope=all"):
+		if f.sessionsWire != "" {
+			return compose.Result{Stdout: f.sessionsWire}, nil
+		}
 		if f.activeSessions {
 			f.activeSessions = false
-			return compose.Result{Stdout: `{"sessions":[{"status":"open","archived":false}]}`}, nil
+			return compose.Result{Stdout: `[{"status":"open","archived":false}]`}, nil
 		}
-		return compose.Result{Stdout: `{"sessions":[]}`}, nil
-	case strings.Contains(call, "compose build"):
+		return compose.Result{Stdout: `[]`}, nil
+	case containsArg(args, "build"):
 		f.built = true
-		f.version = "0.81.0"
-		f.currentImage = "sha256:candidate"
+		f.buildReference = selectedCoreReference(args, f.configuredImage)
+		f.tags[f.buildReference] = "sha256:candidate"
+		f.imageVersions["sha256:candidate"] = "0.81.0"
+		return compose.Result{}, nil
+	case len(args) == 2 && args[0] == "pull":
+		f.tags[args[1]] = "sha256:candidate"
+		f.imageVersions["sha256:candidate"] = "0.81.0"
+		return compose.Result{}, nil
+	case len(args) >= 4 && args[0] == "image" && args[1] == "tag":
+		source, target := args[2], args[3]
+		id := source
+		if tagged, ok := f.tags[source]; ok {
+			id = tagged
+		}
+		f.tags[target] = id
+		if f.built && id == "sha256:old" {
+			f.rollbackPrepared = true
+		}
+		return compose.Result{}, nil
+	case containsArg(args, "up"):
+		f.maintenanceAtRecreate = append(f.maintenanceAtRecreate, f.maintenance)
+		reference := selectedCoreReference(args, f.configuredImage)
+		if id, ok := f.tags[reference]; ok {
+			f.currentImage = id
+		}
+		if version, ok := f.imageVersions[f.currentImage]; ok {
+			f.version = version
+		}
+		if f.dropMaintenanceAfterCandidate && f.currentImage == "sha256:candidate" {
+			f.maintenance = false
+			f.dropMaintenanceAfterCandidate = false
+		}
+		if f.fail == "recreate" && f.currentImage == "sha256:candidate" {
+			return compose.Result{ExitCode: 54}, errors.New("recreate failure")
+		}
+		if f.fail == "compensation" && f.rollbackPrepared {
+			return compose.Result{ExitCode: 55}, errors.New("rollback recreate failure")
+		}
 		return compose.Result{}, nil
 	case strings.Contains(call, "pi --version"):
 		return compose.Result{Stdout: f.version + "\n"}, nil
 	case strings.Contains(call, "/models"):
+		if f.modelsWire != "" {
+			return compose.Result{Stdout: f.modelsWire}, nil
+		}
 		return compose.Result{Stdout: `{"models":[{"id":"model","provider":"provider"}]}`}, nil
 	case strings.Contains(call, "/settings"):
 		return compose.Result{Stdout: `{"provider":"provider","model":"model","thinking":"medium"}`}, nil
@@ -271,6 +593,57 @@ func (f *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose
 	return compose.Result{}, nil
 }

+func containsArg(args []string, wanted string) bool {
+	for _, arg := range args {
+		if arg == wanted {
+			return true
+		}
+	}
+	return false
+}
+
+func selectedCoreReference(args []string, fallback string) string {
+	for index := 0; index+1 < len(args); index++ {
+		if args[index] != "-f" || !strings.Contains(filepath.Base(args[index+1]), "pi-lifecycle-") {
+			continue
+		}
+		contents, err := os.ReadFile(args[index+1])
+		if err != nil {
+			continue
+		}
+		for _, line := range strings.Split(string(contents), "\n") {
+			line = strings.TrimSpace(line)
+			if !strings.HasPrefix(line, "image:") {
+				continue
+			}
+			value := strings.TrimSpace(strings.TrimPrefix(line, "image:"))
+			if decoded, err := strconv.Unquote(value); err == nil {
+				return decoded
+			}
+			return value
+		}
+	}
+	return fallback
+}
+
+func callIndex(calls []string, contains string) int {
+	for index, call := range calls {
+		if strings.Contains(call, contains) {
+			return index
+		}
+	}
+	return -1
+}
+
+func lastCallIndexBefore(calls []string, contains string, before int) int {
+	for index := before - 1; index >= 0; index-- {
+		if strings.Contains(calls[index], contains) {
+			return index
+		}
+	}
+	return -1
+}
+
 func assertCalled(t *testing.T, calls []string, want string) {
 	t.Helper()
 	for _, call := range calls {
@@ -302,3 +675,18 @@ func writeStateForTest(t *testing.T, path string, state State) {
 		t.Fatal(err)
 	}
 }
+
+func stateImageForTest(t *testing.T, fake *fakeRunner) Image {
+	t.Helper()
+	configured, err := renderedCore(context.Background(), fake)
+	if err != nil {
+		t.Fatal(err)
+	}
+	image, err := runningImage(context.Background(), fake, fake.configuredImage)
+	if err != nil {
+		t.Fatal(err)
+	}
+	image.ConfigurationSHA = configured.ConfigurationSHA
+	image.Reference = "thothii-core:thothctl-test-previous"
+	return image
+}