ThothII/tools/tht/internal/pi/update_test.go

package pi

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"testing"

	"github.com/aritmolab/thothii/tools/tht/internal/compose"
)

func TestActiveSessionsParsesAuthenticatedBackendBareArrayFixture(t *testing.T) {
	contents, err := os.ReadFile(filepath.Join("..", "..", "..", "..", "backend", "test", "fixtures", "sessions-scope-all.json"))
	if err != nil {
		t.Fatal(err)
	}
	fake := newFakeRunner()
	fake.sessionsWire = string(contents)
	active, err := activeSessions(context.Background(), fake)
	if err != nil {
		t.Fatalf("activeSessions() error = %v", err)
	}
	if !active {
		t.Fatal("activeSessions() = false, want open session from backend wire fixture")
	}
}

type scopedSessionRunner struct {
	calls []string
	scope string
}

func (r *scopedSessionRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
	r.calls = append(r.calls, strings.Join(args, " "))
	return compose.Result{Stdout: "[]"}, nil
}

func (r *scopedSessionRunner) SessionInventoryScope() string { return r.scope }

func TestActiveSessionsUsesInstallationScopedInventory(t *testing.T) {
	for _, scope := range []string{"mine", "all"} {
		runner := &scopedSessionRunner{scope: scope}
		if active, err := activeSessions(context.Background(), runner); err != nil || active {
			t.Fatalf("activeSessions(%s) = %t, %v; want false, nil", scope, active, err)
		}
		if len(runner.calls) != 1 || !strings.Contains(runner.calls[0], "/sessions?scope="+scope) {
			t.Fatalf("activeSessions(%s) call = %v; want installation-scoped inventory", scope, runner.calls)
		}
	}
}

func TestUpdateBuildsRegistrySelectedVersionRecreatesOnlyCoreAndPersistsRecoveryState(t *testing.T) {
	fake := newFakeRunner()
	dir := t.TempDir()
	result, err := UpdateWithResolvedVersion(context.Background(), fake, Request{
		StatePath: filepath.Join(dir, ".tht", "update-state.json"),
		Source:    BuildSource,
		Confirm:   true,
	}, "@earendil-works/pi-coding-agent", registryFunc(func(context.Context, string) (string, error) {
		return "0.81.0", nil
	}))
	if err != nil {
		t.Fatalf("Update() error = %v", err)
	}
	if result.Phase != PhaseVerified {
		t.Fatalf("phase = %q, want %q", result.Phase, PhaseVerified)
	}
	if result.Version != "0.81.0" {
		t.Fatalf("resolved version = %q, want 0.81.0", result.Version)
	}
	assertCalled(t, fake.calls, "build --pull --build-arg PI_VERSION=0.81.0 --build-arg PI_RUNTIME_PACKAGE_VERSION=0.81.0 core")
	assertCalled(t, fake.calls, "up --detach --wait --wait-timeout 45 --no-deps --force-recreate core")
	assertNotCalled(t, fake.calls, "frontend")
	if got := string(readStateBytes(t, result.StatePath)); strings.Contains(got, "secret") || !strings.Contains(got, `"phase": "verified"`) {
		t.Fatalf("state = %q, want credential-free verified metadata", got)
	}
	if got := string(readStateBytes(t, result.StatePath)); strings.Contains(got, "llm.example.invalid") {
		t.Fatalf("state = %q, want an endpoint-free configuration digest", got)
	}
	if selected := readSelectorReference(t, currentImageOverridePath(result.StatePath)); selected != fake.buildReference {
		t.Fatalf("durable selector = %q, want verified candidate %q", selected, fake.buildReference)
	}
	state, err := readState(result.StatePath)
	if err != nil {
		t.Fatal(err)
	}
	if state.Target.Version != "0.81.0" {
		t.Fatalf("durable update state target = %q, want registry-selected version", state.Target.Version)
	}
}

func TestUpdateUsesATransactionScopedComposeOverrideWithoutMutatingTheConfiguredImage(t *testing.T) {
	fake := newFakeRunner()
	statePath := filepath.Join(t.TempDir(), ".tht", "update-state.json")
	if _, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true}); err != nil {
		t.Fatal(err)
	}
	if fake.buildReference == "" || fake.buildReference == fake.configuredImage || !strings.Contains(fake.buildReference, "tht-") {
		t.Fatalf("build reference = %q, configured = %q; want unique lifecycle tag", fake.buildReference, fake.configuredImage)
	}
	assertNotCalled(t, fake.calls, "image tag sha256:old "+fake.configuredImage)
	if matches, err := filepath.Glob(filepath.Join(filepath.Dir(statePath), "pi-lifecycle-*.yaml")); err != nil || len(matches) != 0 {
		t.Fatalf("terminal lifecycle overrides = %v, error = %v; want none", matches, err)
	}
	if _, err := os.Stat(currentImageOverridePath(statePath)); err != nil {
		t.Fatalf("durable current-image override missing: %v", err)
	}
}

func TestSuccessfulUpdateAndRollbackRemainSelectedOnFreshRecreate(t *testing.T) {
	fake := newFakeRunner()
	statePath := filepath.Join(t.TempDir(), ".tht", "update-state.json")
	if _, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true}); err != nil {
		t.Fatal(err)
	}
	fake.currentImage = "sha256:old"
	if err := recreateCore(context.Background(), composeOverrideRunner{Runner: fake, path: currentImageOverridePath(statePath)}); err != nil {
		t.Fatal(err)
	}
	if fake.currentImage != "sha256:candidate" {
		t.Fatalf("fresh recreate image = %q, want verified candidate", fake.currentImage)
	}
	if _, err := Rollback(context.Background(), fake, statePath, pairedRestartStatePath(statePath), true); err != nil {
		t.Fatal(err)
	}
	fake.currentImage = "sha256:candidate"
	if err := recreateCore(context.Background(), composeOverrideRunner{Runner: fake, path: currentImageOverridePath(statePath)}); err != nil {
		t.Fatal(err)
	}
	if fake.currentImage != "sha256:old" {
		t.Fatalf("fresh recreate after rollback image = %q, want previous image", fake.currentImage)
	}
}

func TestWriteLifecycleOverridePinsCoreAndWorkspaceMaintenanceToTheSameImage(t *testing.T) {
	path := filepath.Join(t.TempDir(), "current-image.yaml")
	if err := writeLifecycleOverride(path, "sha256:"+strings.Repeat("a", 64)); err != nil {
		t.Fatal(err)
	}
	contents, err := os.ReadFile(path)
	if err != nil {
		t.Fatal(err)
	}
	text := string(contents)
	wantedImage := `image: "sha256:` + strings.Repeat("a", 64) + `"`
	for _, expected := range []string{"services:", "core:", "workspace-maintenance:", wantedImage} {
		if !strings.Contains(text, expected) {
			t.Fatalf("override = %q, missing %q", text, expected)
		}
	}
}

func TestSelectorPromotionDoesNotMaskPostRenameDirectoryFsyncFailure(t *testing.T) {
	directory := t.TempDir()
	source := filepath.Join(directory, "candidate.yaml")
	destination := filepath.Join(directory, "current-image.yaml")
	if err := writeLifecycleOverride(source, "thothii-core:candidate"); err != nil {
		t.Fatal(err)
	}
	if err := writeLifecycleOverride(destination, "thothii-core:old"); err != nil {
		t.Fatal(err)
	}

	err := promoteLifecycleOverrideWith(
		source,
		destination,
		"thothii-core:candidate",
		func(source, destination, _ string) error {
			if err := os.Rename(source, destination); err != nil {
				return err
			}
			return errors.New("injected post-rename directory fsync failure")
		},
	)

	var recovery interface{ RecoveryRequired() bool }
	if err == nil || !errors.As(err, &recovery) || !recovery.RecoveryRequired() {
		t.Fatalf("promotion error = %v; want typed recovery-required result", err)
	}
	if selected := readSelectorReference(t, destination); selected != "thothii-core:candidate" {
		t.Fatalf("immediate selector = %q, want landed candidate bytes", selected)
	}
}

func TestTwoInstallationsSharingAConfiguredTagUseDifferentLifecycleTags(t *testing.T) {
	first, second := newFakeRunner(), newFakeRunner()
	firstPath := filepath.Join(t.TempDir(), "one", "state.json")
	secondPath := filepath.Join(t.TempDir(), "two", "state.json")
	for _, item := range []struct {
		fake *fakeRunner
		path string
	}{
		{first, firstPath},
		{second, secondPath},
	} {
		if _, err := Update(context.Background(), item.fake, Request{StatePath: item.path, Version: "0.81.0", Source: BuildSource, Confirm: true}); err != nil {
			t.Fatal(err)
		}
	}
	if first.buildReference == second.buildReference {
		t.Fatalf("installations reused lifecycle tag %q", first.buildReference)
	}
	firstSelector := readSelectorReference(t, currentImageOverridePath(firstPath))
	secondSelector := readSelectorReference(t, currentImageOverridePath(secondPath))
	if firstSelector == secondSelector || firstSelector != first.buildReference || secondSelector != second.buildReference {
		t.Fatalf("installation selectors = %q / %q, want isolated lifecycle references", firstSelector, secondSelector)
	}
}

func TestDigestPinnedConfiguredImageIsNeverUsedAsARollbackTagTarget(t *testing.T) {
	fake := newFakeRunner()
	fake.configuredImage = "registry.example.invalid/core@sha256:" + strings.Repeat("b", 64)
	fake.tags = map[string]string{fake.configuredImage: "sha256:old"}
	statePath := filepath.Join(t.TempDir(), ".tht", "state.json")
	if _, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true}); err != nil {
		t.Fatal(err)
	}
	if _, err := Rollback(context.Background(), fake, statePath, pairedRestartStatePath(statePath), true); err != nil {
		t.Fatal(err)
	}
	assertNotCalled(t, fake.calls, "image tag sha256:old "+fake.configuredImage)
	if selected := readSelectorReference(t, currentImageOverridePath(statePath)); !strings.Contains(selected, "-previous") {
		t.Fatalf("rollback selector = %q, want transaction previous tag for digest-pinned base", selected)
	}
}

func TestMaintenanceTransportLossAfterBackendRestartRequiresRecovery(t *testing.T) {
	for _, lost := range []string{"activate", "deactivate"} {
		t.Run(lost, func(t *testing.T) {
			fake := newFakeRunner()
			fake.lostMaintenanceResponse = lost
			fake.restartBackendOnLoss = true

			_, err := Update(context.Background(), fake, Request{
				StatePath: filepath.Join(t.TempDir(), "state.json"),
				Version:   "0.81.0",
				Source:    BuildSource,
				Confirm:   true,
			})

			var recovery *RecoveryRequiredError
			if err == nil || !errors.As(err, &recovery) || !recovery.RecoveryRequired() {
				t.Fatalf("Update() error = %v; want typed recovery-required result", err)
			}
			if fake.backendRestarts != 1 {
				t.Fatalf("backend restarts = %d, want 1", fake.backendRestarts)
			}
			assertCalled(t, fake.calls, "/internal/maintenance/status")
			for index, active := range fake.maintenanceAtRecreate {
				if !active {
					t.Fatalf("recreate %d started without durable maintenance", index+1)
				}
			}
		})
	}
}

func TestMaintenanceReconciliationDoesNotMaskExplicitDurabilityFailure(t *testing.T) {
	fake := newFakeRunner()
	fake.fail = "maintenance-activate-durability"

	err := setMaintenance(context.Background(), fake, true)

	var recovery interface{ RecoveryRequired() bool }
	if err == nil || !errors.As(err, &recovery) || !recovery.RecoveryRequired() {
		t.Fatalf("maintenance error = %v; want typed recovery-required result", err)
	}
	if !fake.maintenance {
		t.Fatal("safe marker state was not retained after activation durability failure")
	}
}

func TestMaintenanceReconciliationRequiresDurabilityProofAfterExplicitPOSTFailure(t *testing.T) {
	fake := newFakeRunner()
	fake.fail = "maintenance-activate-durability-without-status-flag"

	err := setMaintenance(context.Background(), fake, true)

	var recovery *RecoveryRequiredError
	if err == nil || !errors.As(err, &recovery) || !recovery.RecoveryRequired() {
		t.Fatalf("maintenance error = %v; want typed recovery-required result", err)
	}
	if !fake.maintenance {
		t.Fatal("safe marker state was not retained after activation durability failure")
	}
}

func TestSuccessfulLifecycleCommandsPreserveTypedRecoveryErrorFromMaintenanceCleanup(t *testing.T) {
	for _, operation := range []string{"update", "rollback"} {
		t.Run(operation, func(t *testing.T) {
			fake := newFakeRunner()
			statePath := filepath.Join(t.TempDir(), "state.json")
			if operation == "rollback" {
				if _, err := Update(context.Background(), fake, Request{
					StatePath: statePath,
					Version:   "0.81.0",
					Source:    BuildSource,
					Confirm:   true,
				}); err != nil {
					t.Fatal(err)
				}
			}
			fake.failDeactivationDurability = true

			var result Result
			var err error
			if operation == "update" {
				result, err = Update(context.Background(), fake, Request{
					StatePath: statePath,
					Version:   "0.81.0",
					Source:    BuildSource,
					Confirm:   true,
				})
			} else {
				result, err = Rollback(context.Background(), fake, statePath, pairedRestartStatePath(statePath), true)
			}

			if !fake.maintenance {
				t.Fatalf("%s did not retain the restored maintenance marker", operation)
			}
			var recovery *RecoveryRequiredError
			if err == nil || !errors.As(err, &recovery) || !recovery.RecoveryRequired() {
				t.Fatalf("%s error = %v; want typed recovery-required result", operation, err)
			}
			if result.Phase != PhaseFailed {
				t.Fatalf("%s phase = %q, want %q", operation, result.Phase, PhaseFailed)
			}
		})
	}
}

func TestCompensationReactivatesMaintenanceAndRescansBeforeRollback(t *testing.T) {
	fake := newFakeRunner()
	fake.fail = "version"
	fake.dropMaintenanceAfterCandidate = true
	_, _ = Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true})
	rollback := lastCallIndexBefore(fake.calls, "image tag sha256:old", len(fake.calls))
	if rollback < 0 {
		t.Fatalf("calls %v contain no rollback", fake.calls)
	}
	recreate := callIndex(fake.calls, "force-recreate core")
	activate := lastCallIndexBefore(fake.calls, "/internal/maintenance/activate", rollback)
	scan := lastCallIndexBefore(fake.calls, "/sessions?scope=all", rollback)
	if activate <= recreate || scan <= recreate {
		t.Fatalf("calls %v do not reactivate/confirm and rescan after candidate recreate before rollback", fake.calls)
	}
}

func TestAutomaticRollbackSurvivesADeadCandidateCore(t *testing.T) {
	fake := newFakeRunner()
	fake.fail = "dead-candidate"
	statePath := filepath.Join(t.TempDir(), ".tht", "update-state.json")

	result, err := Update(context.Background(), fake, Request{
		StatePath: statePath,
		Version:   "0.81.0",
		Source:    BuildSource,
		Confirm:   true,
	})

	if err == nil || result.Phase != PhaseRolledBack {
		t.Fatalf("Update() = %+v, %v; want automatic rollback after dead candidate", result, err)
	}
	if fake.currentImage != "sha256:old" || !fake.coreRunning {
		t.Fatalf("restored core = image:%q running:%t; want previous running image", fake.currentImage, fake.coreRunning)
	}
	if fake.execFailuresWhileStopped == 0 {
		t.Fatal("fake did not exercise candidate exec failure")
	}
	assertCalled(t, fake.calls, "run --rm --no-deps --entrypoint node")
	assertMaintenanceClearedAfterRestoredProof(t, fake)
}

func TestManualRollbackSurvivesADeadCandidateCore(t *testing.T) {
	fake := newFakeRunner()
	statePath := filepath.Join(t.TempDir(), ".tht", "update-state.json")
	previous := stateImageForTest(t, fake)
	previous.Reference = "thothii-core:tht-dead-candidate-previous"
	fake.tags[previous.Reference] = previous.ID
	writeStateForTest(t, statePath, State{
		Transaction:     "dead-candidate",
		Phase:           PhaseRecreated,
		MutationStarted: true,
		Previous:        previous,
	})
	fake.currentImage = "sha256:candidate"
	fake.version = "0.81.0"
	fake.coreRunning = false
	fake.maintenance = false

	result, err := Rollback(context.Background(), fake, statePath, pairedRestartStatePath(statePath), true)

	if err != nil || result.Phase != PhaseRolledBack {
		t.Fatalf("Rollback() = %+v, %v; want restored previous core", result, err)
	}
	if fake.currentImage != "sha256:old" || !fake.coreRunning {
		t.Fatalf("restored core = image:%q running:%t; want previous running image", fake.currentImage, fake.coreRunning)
	}
	if fake.execFailuresWhileStopped == 0 {
		t.Fatal("fake did not exercise candidate exec failure")
	}
	assertCalled(t, fake.calls, "run --rm --no-deps --entrypoint node")
	assertMaintenanceClearedAfterRestoredProof(t, fake)
}

func TestUpdatePullsOnlyDigestPinnedSource(t *testing.T) {
	fake := newFakeRunner()
	digest := "registry.example.invalid/thothii-core@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
	_, err := Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: PullSource, Image: digest, Confirm: true})
	if err != nil {
		t.Fatalf("Update() pull error = %v", err)
	}
	assertCalled(t, fake.calls, "pull "+digest)
	assertCalled(t, fake.calls, "image tag "+digest+" thothii-core:tht-")
	assertNotCalled(t, fake.calls, "image tag "+digest+" thothii-core:local")

	fake = newFakeRunner()
	_, err = Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: PullSource, Image: "registry.example.invalid/thothii-core:latest", Confirm: true})
	if err == nil || !strings.Contains(err.Error(), "immutable sha256 digest") {
		t.Fatalf("Update() error = %v, want digest-pinning rejection", err)
	}
	assertNotCalled(t, fake.calls, "compose")
}

func TestUpdateIsNoOpWhenDesiredVersionAlreadyRuns(t *testing.T) {
	fake := newFakeRunner()
	fake.version = "0.80.3"
	result, err := Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.80.3", Source: BuildSource, Confirm: true})
	if err != nil {
		t.Fatalf("Update() error = %v", err)
	}
	if result.Phase != PhaseNoop {
		t.Fatalf("phase = %q, want %q", result.Phase, PhaseNoop)
	}
	assertNotCalled(t, fake.calls, "compose build")
}

func TestUpdateRollsBackAfterPostRecreateFailures(t *testing.T) {
	for _, failure := range []string{"recreate", "health", "version", "smoke", "config-drift", "mount-drift"} {
		t.Run(failure, func(t *testing.T) {
			fake := newFakeRunner()
			fake.fail = failure
			statePath := filepath.Join(t.TempDir(), "state.json")
			result, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true})
			if err == nil {
				t.Fatal("Update() error = nil, want verification failure")
			}
			if result.Phase != PhaseRolledBack {
				t.Fatalf("phase = %q, want %q", result.Phase, PhaseRolledBack)
			}
			assertCalled(t, fake.calls, "image tag sha256:old thothii-core:tht-")
			assertNotCalled(t, fake.calls, "image tag sha256:old thothii-core:local")
			assertCalled(t, fake.calls, "up --detach --wait --wait-timeout 45 --no-deps --force-recreate core")
			if got := string(readStateBytes(t, statePath)); !strings.Contains(got, `"phase": "rolled_back"`) {
				t.Fatalf("state = %q, want rollback metadata", got)
			}
		})
	}
}

func TestCandidateVerificationRejectsEveryDeclaredVersionBoundaryMismatch(t *testing.T) {
	for _, boundary := range []string{"executable", "environment", "image-label"} {
		t.Run(boundary, func(t *testing.T) {
			fake := newFakeRunner()
			switch boundary {
			case "executable":
				fake.candidateVersion = "0.80.9"
			case "environment":
				fake.candidateExpectedVersion = "0.80.9"
			case "image-label":
				fake.candidateLabelVersion = "0.80.9"
			}

			result, err := Update(context.Background(), fake, Request{
				StatePath: filepath.Join(t.TempDir(), "state.json"),
				Version:   "0.81.0",
				Source:    BuildSource,
				Confirm:   true,
			})

			if err == nil || result.Phase != PhaseRolledBack {
				t.Fatalf("Update() = %+v, %v; want rollback for candidate %s mismatch", result, err, boundary)
			}
			if fake.currentImage != "sha256:old" {
				t.Fatalf("current image = %q, want restored previous image", fake.currentImage)
			}
		})
	}
}

func TestEveryRecoveryStateWriteFailureIsHandledTransactionally(t *testing.T) {
	for failAt := 1; failAt <= 6; failAt++ {
		t.Run(fmt.Sprintf("write-%d", failAt), func(t *testing.T) {
			fake := newFakeRunner()
			writes := 0
			hooks := defaultLifecycleHooks
			hooks.writeState = func(path string, state State) error {
				writes++
				if writes == failAt {
					return errors.New("injected state write failure")
				}
				return writeState(path, state)
			}
			result, err := updateWithHooks(context.Background(), fake, Request{
				StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true,
			}, hooks)
			if err == nil {
				t.Fatal("updateWithHooks() error = nil, want injected state failure")
			}
			if fake.currentImage != "sha256:old" {
				t.Fatalf("current image = %q, want restored previous", fake.currentImage)
			}
			wantPhase := Phase("")
			if failAt == 2 || failAt == 3 {
				wantPhase = PhaseFailed
			} else if failAt >= 4 {
				wantPhase = PhaseRolledBack
			}
			if result.Phase != wantPhase {
				t.Fatalf("phase = %q, want %q for write %d", result.Phase, wantPhase, failAt)
			}
			if fake.maintenance {
				t.Fatal("maintenance remained active after proven stable recovery")
			}
		})
	}
}

func TestCompensationWriteFailureKeepsMaintenanceActiveForExplicitRecovery(t *testing.T) {
	fake := newFakeRunner()
	fake.fail = "health"
	writes := 0
	hooks := defaultLifecycleHooks
	hooks.writeState = func(path string, state State) error {
		writes++
		if writes == 5 {
			return errors.New("injected compensation state write failure")
		}
		return writeState(path, state)
	}
	result, err := updateWithHooks(context.Background(), fake, Request{
		StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true,
	}, hooks)
	if err == nil || result.Phase != PhaseFailed {
		t.Fatalf("result=%+v error=%v, want failed recovery", result, err)
	}
	if !fake.maintenance {
		t.Fatal("maintenance was cleared without durable rollback state")
	}
}

func TestMaintenanceClearAndCompensationFailuresRemainGated(t *testing.T) {
	for _, failure := range []string{"maintenance-clear", "compensation"} {
		t.Run(failure, func(t *testing.T) {
			fake := newFakeRunner()
			fake.fail = failure
			result, err := Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true})
			if err == nil || result.Phase != PhaseFailed {
				t.Fatalf("result=%+v error=%v", result, err)
			}
			if !fake.maintenance {
				t.Fatal("maintenance was cleared after an unverified terminal failure")
			}
		})
	}
}

// Catches maintenance recovery clearing admission after the obsolete composite smoke rather than
// the same dedicated Pi Management smoke contract used for ordinary image verification.
func TestRecoverMaintenanceClearsOnlyAfterTerminalStateAndVerifiedSmoke(t *testing.T) {
	fake := newFakeRunner()
	fake.maintenance = true
	statePath := filepath.Join(t.TempDir(), "state.json")
	previous := stateImageForTest(t, fake)
	state := State{Transaction: "recover-test", Phase: PhaseRolledBack, MutationStarted: true, Previous: previous}
	writeStateForTest(t, statePath, state)
	overridePath := lifecycleOverridePath(statePath, state.Transaction)
	if err := writeLifecycleOverride(overridePath, previous.Reference); err != nil {
		t.Fatal(err)
	}
	if err := writeLifecycleOverride(currentImageOverridePath(statePath), previous.Reference); err != nil {
		t.Fatal(err)
	}

	if err := RecoverMaintenance(context.Background(), fake, statePath, true); err != nil {
		t.Fatalf("RecoverMaintenance() error = %v", err)
	}
	if fake.maintenance {
		t.Fatal("maintenance remained active after verified terminal recovery")
	}
	if _, err := os.Stat(overridePath); !errors.Is(err, os.ErrNotExist) {
		t.Fatalf("lifecycle override still exists: %v", err)
	}
	if selected := readSelectorReference(t, currentImageOverridePath(statePath)); selected != previous.Reference {
		t.Fatalf("maintenance cleanup changed durable selector to %q", selected)
	}
	assertCalled(t, fake.calls, "/pi-management/test")
}

func TestRecoverMaintenanceRefusesPendingTransaction(t *testing.T) {
	fake := newFakeRunner()
	fake.maintenance = true
	statePath := filepath.Join(t.TempDir(), "state.json")
	writeStateForTest(t, statePath, State{Phase: PhaseRecreated, MutationStarted: true, Previous: stateImageForTest(t, fake)})

	err := RecoverMaintenance(context.Background(), fake, statePath, true)
	if !errors.Is(err, ErrInterruptedUpdate) {
		t.Fatalf("RecoverMaintenance() error = %v, want ErrInterruptedUpdate", err)
	}
	if !fake.maintenance {
		t.Fatal("pending transaction maintenance was cleared")
	}
}

func TestRecoverMaintenanceCompletesAnInterruptedDurablePromotion(t *testing.T) {
	fake := newFakeRunner()
	fake.maintenance = true
	fake.currentImage = "sha256:candidate"
	fake.version = "0.81.0"
	fake.expectedVersion = "0.81.0"
	fake.labelVersion = "0.81.0"
	statePath := filepath.Join(t.TempDir(), ".tht", "update-state.json")
	previous := stateImageForTest(t, newFakeRunner())
	candidate := previous
	candidate.ID = "sha256:candidate"
	candidate.Reference = "thothii-core:tht-recover-candidate"
	fake.tags[candidate.Reference] = candidate.ID
	state := State{
		Transaction:     "promotion-recovery",
		Phase:           PhasePromoting,
		MutationStarted: true,
		Target:          Target{Version: "0.81.0", Source: string(BuildSource)},
		Previous:        previous,
		Candidate:       candidate,
	}
	writeStateForTest(t, statePath, state)
	if err := writeLifecycleOverride(lifecycleOverridePath(statePath, state.Transaction), candidate.Reference); err != nil {
		t.Fatal(err)
	}

	if err := RecoverMaintenance(context.Background(), fake, statePath, true); err != nil {
		t.Fatalf("RecoverMaintenance() promotion error = %v", err)
	}
	if selected := readSelectorReference(t, currentImageOverridePath(statePath)); selected != candidate.Reference {
		t.Fatalf("recovered selector = %q, want %q", selected, candidate.Reference)
	}
	if recovered, err := readState(statePath); err != nil || recovered.Phase != PhaseVerified {
		t.Fatalf("recovered state = %+v, %v; want verified", recovered, err)
	}
}

func TestRollbackFinalStateWriteFailureKeepsMaintenanceAndOverrideForRecovery(t *testing.T) {
	fake := newFakeRunner()
	statePath := filepath.Join(t.TempDir(), "state.json")
	previous := stateImageForTest(t, fake)
	previous.Reference = "thothii-core:tht-rollback-test-previous"
	fake.tags[previous.Reference] = previous.ID
	writeStateForTest(t, statePath, State{
		Transaction: "rollback-test",
		Phase:       PhaseRecreated,
		Previous:    previous,
	})
	hooks := defaultLifecycleHooks
	hooks.writeState = func(string, State) error { return errors.New("injected rollback state write failure") }

	result, err := rollbackWithHooks(context.Background(), fake, statePath, pairedRestartStatePath(statePath), true, hooks)
	if err == nil || result.Phase != PhaseFailed {
		t.Fatalf("rollbackWithHooks() = %+v, %v; want failed durable finalization", result, err)
	}
	if !fake.maintenance {
		t.Fatal("maintenance was cleared without durable rollback finalization")
	}
	if selected := readSelectorReference(t, currentImageOverridePath(statePath)); selected != previous.Reference {
		t.Fatalf("durable rollback selector = %q, want %q", selected, previous.Reference)
	}
}

func TestUpdateDoesNotRecreateWhenPreflightFails(t *testing.T) {
	for _, failure := range []string{"preflight"} {
		t.Run(failure, func(t *testing.T) {
			fake := newFakeRunner()
			fake.fail = failure
			result, err := Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true})
			if err == nil {
				t.Fatal("Update() error = nil, want failure")
			}
			if failure == "preflight" && result.Phase == PhaseRolledBack {
				t.Fatalf("preflight failure unexpectedly rolled back: %+v", result)
			}
			assertNotCalled(t, fake.calls, "force-recreate")
		})
	}
}

func TestCandidateBuildAndPullFailuresRemainPreMutationAndNeverRecreateCore(t *testing.T) {
	for _, testCase := range []struct {
		name    string
		source  Source
		image   string
		failure string
	}{
		{name: "build", source: BuildSource, failure: "build"},
		{name: "pull", source: PullSource, image: "registry.example.invalid/core@sha256:" + strings.Repeat("a", 64), failure: "pull"},
		{name: "candidate tag", source: PullSource, image: "registry.example.invalid/core@sha256:" + strings.Repeat("b", 64), failure: "tag"},
	} {
		t.Run(testCase.name, func(t *testing.T) {
			fake := newFakeRunner()
			fake.fail = testCase.failure
			statePath := filepath.Join(t.TempDir(), ".tht", "update-state.json")
			result, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: testCase.source, Image: testCase.image, Confirm: true})
			if err == nil {
				t.Fatal("Update() error = nil, want preparation failure")
			}
			if result.Phase != PhaseFailed {
				t.Fatalf("phase = %q, want safe failed preparation", result.Phase)
			}
			state, stateErr := readState(statePath)
			if stateErr != nil {
				t.Fatal(stateErr)
			}
			if state.MutationStarted {
				t.Fatal("preparation failure recorded mutationStarted")
			}
			assertNotCalled(t, fake.calls, "force-recreate")
			if fake.maintenance {
				t.Fatal("maintenance remained active after safe preparation failure")
			}
		})
	}
}

func TestSuccessfulCompensationPreservesTheOriginalTypedCause(t *testing.T) {
	for _, cause := range []error{ErrActiveSessions, ErrInterruptedUpdate} {
		fake := newFakeRunner()
		fake.maintenance = true
		statePath := filepath.Join(t.TempDir(), ".tht", "update-state.json")
		state := State{Transaction: "typed-cause", Phase: PhaseRecreated, MutationStarted: true, Previous: stateImageForTest(t, fake)}
		result, err, clear := compensate(context.Background(), fake, statePath, lifecycleOverridePath(statePath, state.Transaction), state, cause, defaultLifecycleHooks)
		if result.Phase != PhaseRolledBack || !clear {
			t.Fatalf("compensation = %+v, clear=%t; want successful rollback", result, clear)
		}
		if !errors.Is(err, cause) {
			t.Fatalf("compensation error = %v, want errors.Is(..., %v)", err, cause)
		}
		if !strings.Contains(err.Error(), "previous core image was restored") {
			t.Fatalf("compensation error = %v, want rollback-success report", err)
		}
	}
}

func TestUpdateRequiresConfirmationAndDrainsActiveSessions(t *testing.T) {
	fake := newFakeRunner()
	_, err := Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource})
	if !errors.Is(err, ErrConfirmationRequired) {
		t.Fatalf("Update() error = %v, want confirmation error", err)
	}
	assertNotCalled(t, fake.calls, "compose")

	fake = newFakeRunner()
	fake.activeSessions = true
	_, err = Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true})
	if !errors.Is(err, ErrActiveSessions) {
		t.Fatalf("Update() error = %v, want active-sessions error", err)
	}

	fake = newFakeRunner()
	fake.activeSessions = true
	_, err = Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true, Drain: true})
	if err != nil {
		t.Fatalf("Update() with drain error = %v", err)
	}
	assertCalled(t, fake.calls, "http://127.0.0.1:8787/sessions?scope=all")
	assertCalled(t, fake.calls, "/internal/maintenance/activate")
	assertCalled(t, fake.calls, "/internal/maintenance/deactivate")
}

func TestRollbackRestoresInterruptedOrPreviouslyRecordedState(t *testing.T) {
	fake := newFakeRunner()
	statePath := filepath.Join(t.TempDir(), "state.json")
	configured, err := renderedCore(context.Background(), fake)
	if err != nil {
		t.Fatal(err)
	}
	previous, err := runningImage(context.Background(), fake, "thothii-core:local")
	if err != nil {
		t.Fatal(err)
	}
	previous.ConfigurationSHA = configured.ConfigurationSHA
	previous.Reference = "thothii-core:tht-test-previous"
	fake.tags[previous.Reference] = previous.ID
	writeStateForTest(t, statePath, State{Version: 1, Phase: PhaseRecreated, Previous: previous})
	result, err := Rollback(context.Background(), fake, statePath, pairedRestartStatePath(statePath), true)
	if err != nil {
		t.Fatalf("Rollback() error = %v", err)
	}
	if result.Phase != PhaseRolledBack {
		t.Fatalf("phase = %q, want %q", result.Phase, PhaseRolledBack)
	}
	assertCalled(t, fake.calls, "image tag sha256:old thothii-core:tht-test-previous")
	assertCalled(t, fake.calls, "up --detach --wait --wait-timeout 45 --no-deps --force-recreate core")
}

func TestUpdateRefusesToOverwriteInterruptedRecoveryState(t *testing.T) {
	fake := newFakeRunner()
	statePath := filepath.Join(t.TempDir(), "state.json")
	writeStateForTest(t, statePath, State{Phase: PhaseRecreated, MutationStarted: true, Previous: Image{ID: "sha256:old", Reference: "thothii-core:local", MountFingerprint: mountFingerprint(nil)}})
	_, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true})
	if !errors.Is(err, ErrInterruptedUpdate) {
		t.Fatalf("Update() error = %v, want interrupted update error", err)
	}
	assertNotCalled(t, fake.calls, "compose")
}

func TestFailedRestartBlocksUpdateAndRollbackAndMalformedStateAlsoRejects(t *testing.T) {
	for _, operation := range []string{"update", "rollback"} {
		for _, restartState := range []string{"failed", "malformed"} {
			t.Run(operation+"_"+restartState, func(t *testing.T) {
				dir := t.TempDir()
				fake := newFakeRunner()
				updateStatePath := filepath.Join(dir, "update-state.json")
				restartStatePath := filepath.Join(dir, "restart-state.json")
				if restartState == "failed" {
					fake.fail = "health"
					_, restartErr := Restart(context.Background(), fake, RestartRequest{
						StatePath:       restartStatePath,
						UpdateStatePath: updateStatePath,
						Confirm:         true,
					})
					var recovery *RecoveryRequiredError
					if !errors.As(restartErr, &recovery) {
						t.Fatalf("Restart() error = %v, want failed restart recovery state", restartErr)
					}
					fake.fail = ""
				} else if err := os.WriteFile(restartStatePath, []byte("{malformed"), 0o600); err != nil {
					t.Fatal(err)
				}
				if operation == "rollback" {
					previous := stateImageForTest(t, fake)
					writeStateForTest(t, updateStatePath, State{
						Transaction:     "rollback-target",
						Phase:           PhaseRecreated,
						Target:          Target{Version: fake.version, Source: string(BuildSource)},
						Previous:        previous,
						MutationStarted: true,
					})
				}
				fake.calls = nil

				var err error
				if operation == "update" {
					_, err = Update(context.Background(), fake, Request{
						StatePath:        updateStatePath,
						RestartStatePath: restartStatePath,
						Version:          "0.81.0",
						Source:           BuildSource,
						Confirm:          true,
					})
				} else {
					_, err = Rollback(context.Background(), fake, updateStatePath, restartStatePath, true)
				}
				if err == nil {
					t.Fatalf("%s accepted %s restart recovery state", operation, restartState)
				}
				if restartState == "failed" && !errors.Is(err, ErrInterruptedRestart) {
					t.Fatalf("%s error = %v, want ErrInterruptedRestart", operation, err)
				}
				assertNotCalled(t, fake.calls, "compose")
			})
		}
	}
}

func TestUpdateCleansVerifiedRestartStateBeforeNormalLifecycleWork(t *testing.T) {
	dir := t.TempDir()
	fake := newFakeRunner()
	updateStatePath := filepath.Join(dir, "update-state.json")
	restartStatePath := filepath.Join(dir, "restart-state.json")
	state := State{
		Transaction:     "verified-restart",
		Phase:           PhaseVerified,
		Target:          Target{Version: fake.version, Source: "restart"},
		Previous:        stateImageForTest(t, fake),
		MutationStarted: true,
	}
	writeStateForTest(t, restartStatePath, state)
	overridePath := lifecycleOverridePath(restartStatePath, state.Transaction)
	if err := writeLifecycleOverride(overridePath, state.Previous.Reference); err != nil {
		t.Fatal(err)
	}

	result, err := Update(context.Background(), fake, Request{
		StatePath:        updateStatePath,
		RestartStatePath: restartStatePath,
		Version:          fake.version,
		Source:           BuildSource,
		Confirm:          true,
	})
	if err != nil || result.Phase != PhaseNoop {
		t.Fatalf("Update() = %+v, %v; want normal no-op after verified restart", result, err)
	}
	for _, path := range []string{restartStatePath, overridePath} {
		if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {
			t.Fatalf("terminal restart artifact %s still exists: %v", path, err)
		}
	}
}

func TestRunningImageCapturesServerBindAndNamedMountIdentity(t *testing.T) {
	fake := newFakeRunner()
	fake.mountsJSON = `[{"Type":"bind","Source":"/srv/thothii/data","Destination":"/data","RW":true},{"Type":"bind","Source":"/srv/thothii/pi","Destination":"/home/thoth/.pi","RW":true},{"Type":"volume","Name":"sessions","Source":"/var/lib/docker/volumes/sessions/_data","Destination":"/data/sessions","RW":true}]`
	image, err := runningImage(context.Background(), fake, "thothii-core:local")
	if err != nil {
		t.Fatalf("runningImage() error = %v", err)
	}
	if len(image.Mounts) != 3 || image.Mounts[0].Type != "bind" || image.Mounts[0].Destination != "/data" {
		t.Fatalf("mounts = %#v", image.Mounts)
	}
	if strings.Contains(fmt.Sprint(image), "/srv/thothii") || image.Mounts[0].SourceSHA256 == "" || image.MountFingerprint == "" {
		t.Fatalf("mount contract leaked a server source or lacks a safe fingerprint: %#v", image)
	}
}

func TestDockerDesktopBindAliasesPreserveLegacyStateOnlyOnDarwin(t *testing.T) {
	hostPath := "/private/var/folders/task/models.json"
	vmPath := "/host_mnt/private/var/folders/task/models.json"
	hostHash, vmHash := mountSourceHash(hostPath), mountSourceHash(vmPath)
	if hostHash == vmHash {
		t.Fatal("the persisted exact source hash changed instead of retaining state-v4 compatibility")
	}

	legacy := Mount{Type: "bind", SourceSHA256: vmHash, Destination: "/config/models.json"}
	darwinCurrent := Mount{Type: "bind", SourceSHA256: hostHash, SourceAliases: mountSourceAliases("bind", hostPath, "darwin"), Destination: "/config/models.json"}
	if !sameMounts([]Mount{legacy}, []Mount{darwinCurrent}) {
		t.Fatal("a legacy Docker Desktop source hash did not match its current Darwin alias")
	}

	linuxCurrent := Mount{Type: "bind", SourceSHA256: hostHash, SourceAliases: mountSourceAliases("bind", hostPath, "linux"), Destination: "/config/models.json"}
	if sameMounts([]Mount{legacy}, []Mount{linuxCurrent}) {
		t.Fatal("distinct Linux /host_mnt and host bind paths were collapsed")
	}
	if aliases := mountSourceAliases("volume", hostPath, "darwin"); len(aliases) != 0 {
		t.Fatalf("named-volume source received Docker Desktop bind aliases: %v", aliases)
	}
}

func TestCanonicalDigestReferenceRejectsCredentialsAndURLForms(t *testing.T) {
	valid := "registry.example.invalid/thothii-core@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
	if got, err := canonicalDigestReference(valid); err != nil || got != valid {
		t.Fatalf("canonicalDigestReference() = %q, %v", got, err)
	}
	for _, invalid := range []string{"https://registry.example.invalid/a@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "user:pass@registry.example/a@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "registry.example/a@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa?token=x"} {
		if _, err := canonicalDigestReference(invalid); err == nil {
			t.Fatalf("accepted unsafe reference %q", invalid)
		}
	}
}

type fakeRunner struct {
	calls                         []string
	fail                          string
	version                       string
	expectedVersion               string
	labelVersion                  string
	activeSessions                bool
	built                         bool
	recreated                     bool
	currentImage                  string
	mountsJSON                    string
	sessionsWire                  string
	configuredImage               string
	buildReference                string
	tags                          map[string]string
	imageVersions                 map[string]string
	maintenance                   bool
	maintenanceAtRecreate         []bool
	lostMaintenanceResponse       string
	restartBackendOnLoss          bool
	backendRestarts               int
	dropMaintenanceAfterCandidate bool
	modelsWire                    string
	piManagementOptionsWire       string
	piManagementTestWire          string
	rollbackPrepared              bool
	coreRunning                   bool
	execFailuresWhileStopped      int
	maintenanceHelperImages       []string
	maintenanceClearImages        []string
	failDeactivationDurability    bool
	deactivationFailed            bool
	restoredProofComplete         bool
	candidateVersion              string
	candidateExpectedVersion      string
	candidateLabelVersion         string
}

func newFakeRunner() *fakeRunner {
	return &fakeRunner{
		version: "0.80.3", expectedVersion: "0.80.3", labelVersion: "0.80.3", currentImage: "sha256:old", configuredImage: "thothii-core:local",
		tags:                     map[string]string{"thothii-core:local": "sha256:old"},
		imageVersions:            map[string]string{"sha256:old": "0.80.3"},
		coreRunning:              true,
		candidateVersion:         "0.81.0",
		candidateExpectedVersion: "0.81.0",
		candidateLabelVersion:    "0.81.0",
	}
}

func (f *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
	call := strings.Join(args, " ")
	f.calls = append(f.calls, call)
	if !f.coreRunning && containsArg(args, "exec") {
		f.execFailuresWhileStopped++
		return compose.Result{ExitCode: 1}, errors.New("core service is not running")
	}
	if (f.built || f.recreated) && f.fail != "compensation" && strings.Contains(call, "image tag sha256:old") {
		f.fail = ""
	}
	if f.fail == "preflight" && strings.Contains(call, "config --format json") {
		return compose.Result{ExitCode: 1}, errors.New("provider token=secret")
	}
	if f.fail == "build" && containsArg(args, "build") {
		return compose.Result{ExitCode: 1}, errors.New("build token=secret")
	}
	if f.fail == "pull" && len(args) > 0 && args[0] == "pull" {
		return compose.Result{ExitCode: 1}, errors.New("pull token=secret")
	}
	if f.fail == "tag" && len(args) >= 4 && args[0] == "image" && args[1] == "tag" && strings.Contains(args[3], "-candidate") {
		return compose.Result{ExitCode: 1}, errors.New("tag token=secret")
	}
	if f.fail == "health" && (f.built || f.recreated) && strings.Contains(call, "curl -fsS http://127.0.0.1:8787/health") {
		return compose.Result{ExitCode: 1}, errors.New("health token=secret")
	}
	if f.fail == "compensation" && (f.built || f.recreated) && !f.rollbackPrepared && strings.Contains(call, "curl -fsS http://127.0.0.1:8787/health") {
		return compose.Result{ExitCode: 1}, errors.New("candidate health failure")
	}
	if f.fail == "version" && (f.built || f.recreated) && strings.Contains(call, "pi --version") && strings.Contains(call, "exec") {
		return compose.Result{ExitCode: 1}, errors.New("version token=secret")
	}
	if f.fail == "smoke" && (f.built || f.recreated) && strings.Contains(call, "127.0.0.1:8787/pi-management/test") {
		return compose.Result{ExitCode: 1}, errors.New("smoke token=secret")
	}
	switch {
	case strings.Contains(call, "config --format json"):
		endpoint := "https://llm.example.invalid"
		if f.fail == "config-drift" && (f.built || f.recreated) {
			endpoint = "https://drift.example.invalid"
		}
		return compose.Result{Stdout: `{"services":{"core":{"image":"` + selectedCoreReference(args, f.configuredImage) + `","environment":{"THT_LLM_URL":"` + endpoint + `"}}}}`}, nil
	case strings.Contains(call, "ps --status running -q core"):
		if f.coreRunning {
			return compose.Result{Stdout: "core-container\n"}, nil
		}
		return compose.Result{}, nil
	case strings.Contains(call, "ps -q core"):
		return compose.Result{Stdout: "core-container\n"}, nil
	case strings.Contains(call, "inspect --format {{.Image}}"):
		if f.fail == "image-drift" && f.recreated {
			return compose.Result{Stdout: "sha256:unexpected\n"}, nil
		}
		return compose.Result{Stdout: f.currentImage + "\n"}, nil
	case strings.Contains(call, "io.thothii.pi.version"):
		return compose.Result{Stdout: f.labelVersion + "\n"}, nil
	case strings.Contains(call, "inspect --format {{json .Mounts}}"):
		if f.fail == "mount-drift" && (f.built || f.recreated) {
			return compose.Result{Stdout: `[{"Type":"volume","Name":"wrong-settings","Source":"wrong-settings","Destination":"/data/settings","RW":true}]`}, nil
		}
		if f.mountsJSON != "" {
			return compose.Result{Stdout: f.mountsJSON}, nil
		}
		return compose.Result{Stdout: `[{"Type":"volume","Name":"settings","Source":"settings","Destination":"/data/settings","RW":true},{"Type":"volume","Name":"pi-state","Source":"pi-state","Destination":"/home/thoth/.pi","RW":true},{"Type":"volume","Name":"sessions","Source":"sessions","Destination":"/data/sessions","RW":true},{"Type":"volume","Name":"workspace-registry","Source":"workspace-registry","Destination":"/data/workspace-registry","RW":true}]`}, nil
	case strings.Contains(call, "/internal/maintenance/activate"):
		f.maintenance = true
		if f.fail == "maintenance-activate-durability" || f.fail == "maintenance-activate-durability-without-status-flag" {
			return compose.Result{ExitCode: 22}, errors.New("maintenance activation durability was not acknowledged")
		}
		if f.lostMaintenanceResponse == "activate" {
			f.lostMaintenanceResponse = ""
			if f.restartBackendOnLoss {
				f.backendRestarts++
			}
			return compose.Result{ExitCode: 52}, errors.New("lost activation response")
		}
		return compose.Result{Stdout: `{"active":true,"admissions":0}`}, nil
	case strings.Contains(call, "/internal/maintenance/deactivate"):
		if f.fail == "maintenance-clear" {
			return compose.Result{ExitCode: 53}, errors.New("maintenance clear failure")
		}
		if f.failDeactivationDurability {
			f.deactivationFailed = true
			f.maintenance = true
			return compose.Result{ExitCode: 22}, errors.New("maintenance deactivation durability was not acknowledged")
		}
		f.maintenance = false
		f.maintenanceClearImages = append(f.maintenanceClearImages, f.currentImage)
		if f.lostMaintenanceResponse == "deactivate" {
			f.lostMaintenanceResponse = ""
			if f.restartBackendOnLoss {
				f.backendRestarts++
			}
			return compose.Result{ExitCode: 52}, errors.New("lost deactivation response")
		}
		return compose.Result{Stdout: `{"active":false,"admissions":0}`}, nil
	case strings.Contains(call, "/internal/maintenance/status"):
		if f.fail == "maintenance-proof" && f.recreated {
			return compose.Result{Stdout: fmt.Sprintf(`{"active":%t,"admissions":0,"recoveryRequired":true}`, f.maintenance)}, nil
		}
		if f.fail == "maintenance-activate-durability" {
			return compose.Result{Stdout: fmt.Sprintf(`{"active":%t,"admissions":0,"recoveryRequired":true}`, f.maintenance)}, nil
		}
		if f.deactivationFailed {
			return compose.Result{Stdout: fmt.Sprintf(`{"active":%t,"admissions":0,"recoveryRequired":true}`, f.maintenance)}, nil
		}
		return compose.Result{Stdout: fmt.Sprintf(`{"active":%t,"admissions":0}`, f.maintenance)}, nil
	case strings.Contains(call, "/sessions?scope=all"):
		if f.sessionsWire != "" {
			return compose.Result{Stdout: f.sessionsWire}, nil
		}
		if f.activeSessions {
			f.activeSessions = false
			return compose.Result{Stdout: `[{"status":"open","archived":false}]`}, nil
		}
		return compose.Result{Stdout: `[]`}, nil
	case containsArg(args, "run") && containsArg(args, "--entrypoint") && containsArg(args, "node"):
		reference := selectedCoreReference(args, f.configuredImage)
		f.maintenanceHelperImages = append(f.maintenanceHelperImages, reference)
		if reference == "" {
			return compose.Result{ExitCode: 1}, errors.New("maintenance helper has no selected image")
		}
		if f.tags[reference] != "sha256:old" {
			return compose.Result{ExitCode: 1}, errors.New("maintenance helper did not select the previous image")
		}
		f.maintenance = true
		return compose.Result{}, nil
	case containsArg(args, "build"):
		f.built = true
		f.buildReference = selectedCoreReference(args, f.configuredImage)
		f.tags[f.buildReference] = "sha256:candidate"
		f.imageVersions["sha256:candidate"] = "0.81.0"
		return compose.Result{}, nil
	case len(args) == 2 && args[0] == "pull":
		f.tags[args[1]] = "sha256:candidate"
		f.imageVersions["sha256:candidate"] = "0.81.0"
		return compose.Result{}, nil
	case len(args) >= 4 && args[0] == "image" && args[1] == "tag":
		source, target := args[2], args[3]
		id := source
		if tagged, ok := f.tags[source]; ok {
			id = tagged
		}
		f.tags[target] = id
		if f.built && id == "sha256:old" {
			f.rollbackPrepared = true
		}
		return compose.Result{}, nil
	case containsArg(args, "up"):
		f.recreated = true
		f.maintenanceAtRecreate = append(f.maintenanceAtRecreate, f.maintenance)
		reference := selectedCoreReference(args, f.configuredImage)
		if id, ok := f.tags[reference]; ok {
			f.currentImage = id
		}
		if version, ok := f.imageVersions[f.currentImage]; ok {
			f.version = version
		}
		if f.currentImage == "sha256:candidate" {
			f.version = f.candidateVersion
			f.expectedVersion = f.candidateExpectedVersion
			f.labelVersion = f.candidateLabelVersion
		} else if f.currentImage == "sha256:old" {
			f.expectedVersion = "0.80.3"
			f.labelVersion = "0.80.3"
		}
		f.coreRunning = true
		if f.currentImage == "sha256:candidate" {
			f.restoredProofComplete = false
		}
		if f.fail == "dead-candidate" && f.currentImage == "sha256:candidate" {
			f.coreRunning = false
		}
		if f.dropMaintenanceAfterCandidate && f.currentImage == "sha256:candidate" {
			f.maintenance = false
			f.dropMaintenanceAfterCandidate = false
		}
		if f.fail == "recreate" && (f.built || f.recreated) {
			return compose.Result{ExitCode: 54}, errors.New("recreate failure")
		}
		if f.fail == "compensation" && f.rollbackPrepared {
			return compose.Result{ExitCode: 55}, errors.New("rollback recreate failure")
		}
		return compose.Result{}, nil
	case strings.Contains(call, "pi --version"):
		return compose.Result{Stdout: f.version + "\n"}, nil
	case strings.Contains(call, "PI_VERSION"):
		return compose.Result{Stdout: f.expectedVersion + "\n"}, nil
	case strings.Contains(call, "/pi-management/options"):
		if f.piManagementOptionsWire != "" {
			return compose.Result{Stdout: f.piManagementOptionsWire}, nil
		}
		return compose.Result{Stdout: `{"providers":["provider"],"models":[{"id":"model","provider":"provider"}],"reasoning":["low","medium","high"]}`}, nil
	case strings.Contains(call, "/pi-management/test"):
		if f.currentImage == "sha256:old" {
			f.restoredProofComplete = true
		}
		if f.piManagementTestWire != "" {
			return compose.Result{Stdout: f.piManagementTestWire}, nil
		}
		return compose.Result{Stdout: `{"ready":true}`}, nil
	case strings.Contains(call, "/models"):
		if f.modelsWire != "" {
			return compose.Result{Stdout: f.modelsWire}, nil
		}
		return compose.Result{Stdout: `{"models":[{"id":"model","provider":"provider"}]}`}, nil
	case strings.Contains(call, "/settings"):
		if f.currentImage == "sha256:old" {
			f.restoredProofComplete = true
		}
		return compose.Result{Stdout: `{"provider":"provider","model":"model","thinking":"medium"}`}, nil
	case strings.Contains(call, "/health"):
		return compose.Result{Stdout: `{"status":"ok"}`}, nil
	}
	return compose.Result{}, nil
}

func containsArg(args []string, wanted string) bool {
	for _, arg := range args {
		if arg == wanted {
			return true
		}
	}
	return false
}

func selectedCoreReference(args []string, fallback string) string {
	for index := 0; index+1 < len(args); index++ {
		if args[index] != "-f" || (!strings.Contains(filepath.Base(args[index+1]), "pi-lifecycle-") && filepath.Base(args[index+1]) != "current-image.yaml") {
			continue
		}
		contents, err := os.ReadFile(args[index+1])
		if err != nil {
			continue
		}
		for _, line := range strings.Split(string(contents), "\n") {
			line = strings.TrimSpace(line)
			if !strings.HasPrefix(line, "image:") {
				continue
			}
			value := strings.TrimSpace(strings.TrimPrefix(line, "image:"))
			if decoded, err := strconv.Unquote(value); err == nil {
				return decoded
			}
			return value
		}
	}
	return fallback
}

func readSelectorReference(t *testing.T, path string) string {
	t.Helper()
	contents, err := os.ReadFile(path)
	if err != nil {
		t.Fatal(err)
	}
	for _, line := range strings.Split(string(contents), "\n") {
		line = strings.TrimSpace(line)
		if !strings.HasPrefix(line, "image:") {
			continue
		}
		value := strings.TrimSpace(strings.TrimPrefix(line, "image:"))
		if decoded, err := strconv.Unquote(value); err == nil {
			return decoded
		}
		return value
	}
	t.Fatalf("selector %s has no image", path)
	return ""
}

func callIndex(calls []string, contains string) int {
	for index, call := range calls {
		if strings.Contains(call, contains) {
			return index
		}
	}
	return -1
}

func lastCallIndexBefore(calls []string, contains string, before int) int {
	for index := before - 1; index >= 0; index-- {
		if strings.Contains(calls[index], contains) {
			return index
		}
	}
	return -1
}

func assertCalled(t *testing.T, calls []string, want string) {
	t.Helper()
	for _, call := range calls {
		if strings.Contains(call, want) {
			return
		}
	}
	t.Fatalf("calls %v did not include %q", calls, want)
}
func assertNotCalled(t *testing.T, calls []string, prohibited string) {
	t.Helper()
	for _, call := range calls {
		if strings.Contains(call, prohibited) {
			t.Fatalf("calls %v unexpectedly included %q", calls, prohibited)
		}
	}
}

func assertMaintenanceClearedAfterRestoredProof(t *testing.T, fake *fakeRunner) {
	t.Helper()
	if fake.maintenance {
		t.Fatal("maintenance remained active after restored-core proof")
	}
	if !fake.restoredProofComplete {
		t.Fatal("maintenance cleared before restored settings smoke completed")
	}
	if len(fake.maintenanceClearImages) == 0 {
		t.Fatal("maintenance was never durably cleared")
	}
	for _, image := range fake.maintenanceClearImages {
		if image != "sha256:old" {
			t.Fatalf("maintenance cleared while image %q was selected; want previous image", image)
		}
	}
}

func readStateBytes(t *testing.T, path string) []byte {
	t.Helper()
	contents, err := os.ReadFile(path)
	if err != nil {
		t.Fatal(err)
	}
	return contents
}
func writeStateForTest(t *testing.T, path string, state State) {
	t.Helper()
	if err := writeState(path, state); err != nil {
		t.Fatal(err)
	}
}

func stateImageForTest(t *testing.T, fake *fakeRunner) Image {
	t.Helper()
	configured, err := renderedCore(context.Background(), fake)
	if err != nil {
		t.Fatal(err)
	}
	image, err := runningImage(context.Background(), fake, fake.configuredImage)
	if err != nil {
		t.Fatal(err)
	}
	image.ConfigurationSHA = configured.ConfigurationSHA
	image.Reference = "thothii-core:tht-test-previous"
	return image
}