package pi import ( "context" "errors" "fmt" "io" "os" "path/filepath" "strconv" "strings" "testing" "github.com/aritmolab/thothii/tools/thothctl/internal/compose" ) func TestActiveSessionsParsesAuthenticatedBackendBareArrayFixture(t *testing.T) { contents, err := os.ReadFile(filepath.Join("..", "..", "..", "..", "backend", "test", "fixtures", "sessions-scope-all.json")) if err != nil { t.Fatal(err) } fake := newFakeRunner() fake.sessionsWire = string(contents) active, err := activeSessions(context.Background(), fake) if err != nil { t.Fatalf("activeSessions() error = %v", err) } if !active { t.Fatal("activeSessions() = false, want open session from backend wire fixture") } } type scopedSessionRunner struct { calls []string scope string } func (r *scopedSessionRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) { r.calls = append(r.calls, strings.Join(args, " ")) return compose.Result{Stdout: "[]"}, nil } func (r *scopedSessionRunner) SessionInventoryScope() string { return r.scope } func TestActiveSessionsUsesInstallationScopedInventory(t *testing.T) { for _, scope := range []string{"mine", "all"} { runner := &scopedSessionRunner{scope: scope} if active, err := activeSessions(context.Background(), runner); err != nil || active { t.Fatalf("activeSessions(%s) = %t, %v; want false, nil", scope, active, err) } if len(runner.calls) != 1 || !strings.Contains(runner.calls[0], "/sessions?scope="+scope) { t.Fatalf("activeSessions(%s) call = %v; want installation-scoped inventory", scope, runner.calls) } } } func TestUpdateBuildsPinnedVersionRecreatesOnlyCoreAndPersistsRecoveryState(t *testing.T) { fake := newFakeRunner() dir := t.TempDir() result, err := Update(context.Background(), fake, Request{ StatePath: filepath.Join(dir, ".thothctl", "update-state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true, }) if err != nil { t.Fatalf("Update() error = %v", err) } if result.Phase != PhaseVerified { t.Fatalf("phase = %q, want %q", result.Phase, PhaseVerified) } assertCalled(t, fake.calls, "build --pull --build-arg PI_VERSION=0.81.0 core") assertCalled(t, fake.calls, "up --detach --wait --wait-timeout 45 --no-deps --force-recreate core") assertNotCalled(t, fake.calls, "frontend") if got := string(readStateBytes(t, result.StatePath)); strings.Contains(got, "secret") || !strings.Contains(got, `"phase": "verified"`) { t.Fatalf("state = %q, want credential-free verified metadata", got) } if got := string(readStateBytes(t, result.StatePath)); strings.Contains(got, "llm.example.invalid") { t.Fatalf("state = %q, want an endpoint-free configuration digest", got) } if selected := readSelectorReference(t, currentImageOverridePath(result.StatePath)); selected != fake.buildReference { t.Fatalf("durable selector = %q, want verified candidate %q", selected, fake.buildReference) } } func TestUpdateUsesATransactionScopedComposeOverrideWithoutMutatingTheConfiguredImage(t *testing.T) { fake := newFakeRunner() statePath := filepath.Join(t.TempDir(), ".thothctl", "update-state.json") if _, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true}); err != nil { t.Fatal(err) } if fake.buildReference == "" || fake.buildReference == fake.configuredImage || !strings.Contains(fake.buildReference, "thothctl-") { t.Fatalf("build reference = %q, configured = %q; want unique lifecycle tag", fake.buildReference, fake.configuredImage) } assertNotCalled(t, fake.calls, "image tag sha256:old "+fake.configuredImage) if matches, err := filepath.Glob(filepath.Join(filepath.Dir(statePath), "pi-lifecycle-*.yaml")); err != nil || len(matches) != 0 { t.Fatalf("terminal lifecycle overrides = %v, error = %v; want none", matches, err) } if _, err := os.Stat(currentImageOverridePath(statePath)); err != nil { t.Fatalf("durable current-image override missing: %v", err) } } func TestSuccessfulUpdateAndRollbackRemainSelectedOnFreshRecreate(t *testing.T) { fake := newFakeRunner() statePath := filepath.Join(t.TempDir(), ".thothctl", "update-state.json") if _, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true}); err != nil { t.Fatal(err) } fake.currentImage = "sha256:old" if err := recreateCore(context.Background(), composeOverrideRunner{Runner: fake, path: currentImageOverridePath(statePath)}); err != nil { t.Fatal(err) } if fake.currentImage != "sha256:candidate" { t.Fatalf("fresh recreate image = %q, want verified candidate", fake.currentImage) } if _, err := Rollback(context.Background(), fake, statePath, pairedRestartStatePath(statePath), true); err != nil { t.Fatal(err) } fake.currentImage = "sha256:candidate" if err := recreateCore(context.Background(), composeOverrideRunner{Runner: fake, path: currentImageOverridePath(statePath)}); err != nil { t.Fatal(err) } if fake.currentImage != "sha256:old" { t.Fatalf("fresh recreate after rollback image = %q, want previous image", fake.currentImage) } } func TestWriteLifecycleOverridePinsCoreAndWorkspaceMaintenanceToTheSameImage(t *testing.T) { path := filepath.Join(t.TempDir(), "current-image.yaml") if err := writeLifecycleOverride(path, "sha256:"+strings.Repeat("a", 64)); err != nil { t.Fatal(err) } contents, err := os.ReadFile(path) if err != nil { t.Fatal(err) } text := string(contents) wantedImage := `image: "sha256:` + strings.Repeat("a", 64) + `"` for _, expected := range []string{"services:", "core:", "workspace-maintenance:", wantedImage} { if !strings.Contains(text, expected) { t.Fatalf("override = %q, missing %q", text, expected) } } } func TestSelectorPromotionDoesNotMaskPostRenameDirectoryFsyncFailure(t *testing.T) { directory := t.TempDir() source := filepath.Join(directory, "candidate.yaml") destination := filepath.Join(directory, "current-image.yaml") if err := writeLifecycleOverride(source, "thothii-core:candidate"); err != nil { t.Fatal(err) } if err := writeLifecycleOverride(destination, "thothii-core:old"); err != nil { t.Fatal(err) } err := promoteLifecycleOverrideWith( source, destination, "thothii-core:candidate", func(source, destination, _ string) error { if err := os.Rename(source, destination); err != nil { return err } return errors.New("injected post-rename directory fsync failure") }, ) var recovery interface{ RecoveryRequired() bool } if err == nil || !errors.As(err, &recovery) || !recovery.RecoveryRequired() { t.Fatalf("promotion error = %v; want typed recovery-required result", err) } if selected := readSelectorReference(t, destination); selected != "thothii-core:candidate" { t.Fatalf("immediate selector = %q, want landed candidate bytes", selected) } } func TestTwoInstallationsSharingAConfiguredTagUseDifferentLifecycleTags(t *testing.T) { first, second := newFakeRunner(), newFakeRunner() firstPath := filepath.Join(t.TempDir(), "one", "state.json") secondPath := filepath.Join(t.TempDir(), "two", "state.json") for _, item := range []struct { fake *fakeRunner path string }{ {first, firstPath}, {second, secondPath}, } { if _, err := Update(context.Background(), item.fake, Request{StatePath: item.path, Version: "0.81.0", Source: BuildSource, Confirm: true}); err != nil { t.Fatal(err) } } if first.buildReference == second.buildReference { t.Fatalf("installations reused lifecycle tag %q", first.buildReference) } firstSelector := readSelectorReference(t, currentImageOverridePath(firstPath)) secondSelector := readSelectorReference(t, currentImageOverridePath(secondPath)) if firstSelector == secondSelector || firstSelector != first.buildReference || secondSelector != second.buildReference { t.Fatalf("installation selectors = %q / %q, want isolated lifecycle references", firstSelector, secondSelector) } } func TestDigestPinnedConfiguredImageIsNeverUsedAsARollbackTagTarget(t *testing.T) { fake := newFakeRunner() fake.configuredImage = "registry.example.invalid/core@sha256:" + strings.Repeat("b", 64) fake.tags = map[string]string{fake.configuredImage: "sha256:old"} statePath := filepath.Join(t.TempDir(), ".thothctl", "state.json") if _, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true}); err != nil { t.Fatal(err) } if _, err := Rollback(context.Background(), fake, statePath, pairedRestartStatePath(statePath), true); err != nil { t.Fatal(err) } assertNotCalled(t, fake.calls, "image tag sha256:old "+fake.configuredImage) if selected := readSelectorReference(t, currentImageOverridePath(statePath)); !strings.Contains(selected, "-previous") { t.Fatalf("rollback selector = %q, want transaction previous tag for digest-pinned base", selected) } } func TestMaintenanceTransportLossAfterBackendRestartRequiresRecovery(t *testing.T) { for _, lost := range []string{"activate", "deactivate"} { t.Run(lost, func(t *testing.T) { fake := newFakeRunner() fake.lostMaintenanceResponse = lost fake.restartBackendOnLoss = true _, err := Update(context.Background(), fake, Request{ StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true, }) var recovery *RecoveryRequiredError if err == nil || !errors.As(err, &recovery) || !recovery.RecoveryRequired() { t.Fatalf("Update() error = %v; want typed recovery-required result", err) } if fake.backendRestarts != 1 { t.Fatalf("backend restarts = %d, want 1", fake.backendRestarts) } assertCalled(t, fake.calls, "/internal/maintenance/status") for index, active := range fake.maintenanceAtRecreate { if !active { t.Fatalf("recreate %d started without durable maintenance", index+1) } } }) } } func TestMaintenanceReconciliationDoesNotMaskExplicitDurabilityFailure(t *testing.T) { fake := newFakeRunner() fake.fail = "maintenance-activate-durability" err := setMaintenance(context.Background(), fake, true) var recovery interface{ RecoveryRequired() bool } if err == nil || !errors.As(err, &recovery) || !recovery.RecoveryRequired() { t.Fatalf("maintenance error = %v; want typed recovery-required result", err) } if !fake.maintenance { t.Fatal("safe marker state was not retained after activation durability failure") } } func TestMaintenanceReconciliationRequiresDurabilityProofAfterExplicitPOSTFailure(t *testing.T) { fake := newFakeRunner() fake.fail = "maintenance-activate-durability-without-status-flag" err := setMaintenance(context.Background(), fake, true) var recovery *RecoveryRequiredError if err == nil || !errors.As(err, &recovery) || !recovery.RecoveryRequired() { t.Fatalf("maintenance error = %v; want typed recovery-required result", err) } if !fake.maintenance { t.Fatal("safe marker state was not retained after activation durability failure") } } func TestSuccessfulLifecycleCommandsPreserveTypedRecoveryErrorFromMaintenanceCleanup(t *testing.T) { for _, operation := range []string{"update", "rollback"} { t.Run(operation, func(t *testing.T) { fake := newFakeRunner() statePath := filepath.Join(t.TempDir(), "state.json") if operation == "rollback" { if _, err := Update(context.Background(), fake, Request{ StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true, }); err != nil { t.Fatal(err) } } fake.failDeactivationDurability = true var result Result var err error if operation == "update" { result, err = Update(context.Background(), fake, Request{ StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true, }) } else { result, err = Rollback(context.Background(), fake, statePath, pairedRestartStatePath(statePath), true) } if !fake.maintenance { t.Fatalf("%s did not retain the restored maintenance marker", operation) } var recovery *RecoveryRequiredError if err == nil || !errors.As(err, &recovery) || !recovery.RecoveryRequired() { t.Fatalf("%s error = %v; want typed recovery-required result", operation, err) } if result.Phase != PhaseFailed { t.Fatalf("%s phase = %q, want %q", operation, result.Phase, PhaseFailed) } }) } } func TestCompensationReactivatesMaintenanceAndRescansBeforeRollback(t *testing.T) { fake := newFakeRunner() fake.fail = "version" fake.dropMaintenanceAfterCandidate = true _, _ = Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true}) rollback := lastCallIndexBefore(fake.calls, "image tag sha256:old", len(fake.calls)) if rollback < 0 { t.Fatalf("calls %v contain no rollback", fake.calls) } recreate := callIndex(fake.calls, "force-recreate core") activate := lastCallIndexBefore(fake.calls, "/internal/maintenance/activate", rollback) scan := lastCallIndexBefore(fake.calls, "/sessions?scope=all", rollback) if activate <= recreate || scan <= recreate { t.Fatalf("calls %v do not reactivate/confirm and rescan after candidate recreate before rollback", fake.calls) } } func TestAutomaticRollbackSurvivesADeadCandidateCore(t *testing.T) { fake := newFakeRunner() fake.fail = "dead-candidate" statePath := filepath.Join(t.TempDir(), ".thothctl", "update-state.json") result, err := Update(context.Background(), fake, Request{ StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true, }) if err == nil || result.Phase != PhaseRolledBack { t.Fatalf("Update() = %+v, %v; want automatic rollback after dead candidate", result, err) } if fake.currentImage != "sha256:old" || !fake.coreRunning { t.Fatalf("restored core = image:%q running:%t; want previous running image", fake.currentImage, fake.coreRunning) } if fake.execFailuresWhileStopped == 0 { t.Fatal("fake did not exercise candidate exec failure") } assertCalled(t, fake.calls, "run --rm --no-deps --entrypoint node") assertMaintenanceClearedAfterRestoredProof(t, fake) } func TestManualRollbackSurvivesADeadCandidateCore(t *testing.T) { fake := newFakeRunner() statePath := filepath.Join(t.TempDir(), ".thothctl", "update-state.json") previous := stateImageForTest(t, fake) previous.Reference = "thothii-core:thothctl-dead-candidate-previous" fake.tags[previous.Reference] = previous.ID writeStateForTest(t, statePath, State{ Transaction: "dead-candidate", Phase: PhaseRecreated, MutationStarted: true, Previous: previous, }) fake.currentImage = "sha256:candidate" fake.version = "0.81.0" fake.coreRunning = false fake.maintenance = false result, err := Rollback(context.Background(), fake, statePath, pairedRestartStatePath(statePath), true) if err != nil || result.Phase != PhaseRolledBack { t.Fatalf("Rollback() = %+v, %v; want restored previous core", result, err) } if fake.currentImage != "sha256:old" || !fake.coreRunning { t.Fatalf("restored core = image:%q running:%t; want previous running image", fake.currentImage, fake.coreRunning) } if fake.execFailuresWhileStopped == 0 { t.Fatal("fake did not exercise candidate exec failure") } assertCalled(t, fake.calls, "run --rm --no-deps --entrypoint node") assertMaintenanceClearedAfterRestoredProof(t, fake) } func TestUpdatePullsOnlyDigestPinnedSource(t *testing.T) { fake := newFakeRunner() digest := "registry.example.invalid/thothii-core@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" _, err := Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: PullSource, Image: digest, Confirm: true}) if err != nil { t.Fatalf("Update() pull error = %v", err) } assertCalled(t, fake.calls, "pull "+digest) assertCalled(t, fake.calls, "image tag "+digest+" thothii-core:thothctl-") assertNotCalled(t, fake.calls, "image tag "+digest+" thothii-core:local") fake = newFakeRunner() _, err = Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: PullSource, Image: "registry.example.invalid/thothii-core:latest", Confirm: true}) if err == nil || !strings.Contains(err.Error(), "immutable sha256 digest") { t.Fatalf("Update() error = %v, want digest-pinning rejection", err) } assertNotCalled(t, fake.calls, "compose") } func TestUpdateIsNoOpWhenDesiredVersionAlreadyRuns(t *testing.T) { fake := newFakeRunner() fake.version = "0.80.3" result, err := Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.80.3", Source: BuildSource, Confirm: true}) if err != nil { t.Fatalf("Update() error = %v", err) } if result.Phase != PhaseNoop { t.Fatalf("phase = %q, want %q", result.Phase, PhaseNoop) } assertNotCalled(t, fake.calls, "compose build") } func TestUpdateRollsBackAfterPostRecreateFailures(t *testing.T) { for _, failure := range []string{"recreate", "health", "version", "smoke", "config-drift", "mount-drift"} { t.Run(failure, func(t *testing.T) { fake := newFakeRunner() fake.fail = failure statePath := filepath.Join(t.TempDir(), "state.json") result, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true}) if err == nil { t.Fatal("Update() error = nil, want verification failure") } if result.Phase != PhaseRolledBack { t.Fatalf("phase = %q, want %q", result.Phase, PhaseRolledBack) } assertCalled(t, fake.calls, "image tag sha256:old thothii-core:thothctl-") assertNotCalled(t, fake.calls, "image tag sha256:old thothii-core:local") assertCalled(t, fake.calls, "up --detach --wait --wait-timeout 45 --no-deps --force-recreate core") if got := string(readStateBytes(t, statePath)); !strings.Contains(got, `"phase": "rolled_back"`) { t.Fatalf("state = %q, want rollback metadata", got) } }) } } func TestCandidateVerificationRejectsEveryDeclaredVersionBoundaryMismatch(t *testing.T) { for _, boundary := range []string{"executable", "environment", "image-label"} { t.Run(boundary, func(t *testing.T) { fake := newFakeRunner() switch boundary { case "executable": fake.candidateVersion = "0.80.9" case "environment": fake.candidateExpectedVersion = "0.80.9" case "image-label": fake.candidateLabelVersion = "0.80.9" } result, err := Update(context.Background(), fake, Request{ StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true, }) if err == nil || result.Phase != PhaseRolledBack { t.Fatalf("Update() = %+v, %v; want rollback for candidate %s mismatch", result, err, boundary) } if fake.currentImage != "sha256:old" { t.Fatalf("current image = %q, want restored previous image", fake.currentImage) } }) } } func TestEveryRecoveryStateWriteFailureIsHandledTransactionally(t *testing.T) { for failAt := 1; failAt <= 6; failAt++ { t.Run(fmt.Sprintf("write-%d", failAt), func(t *testing.T) { fake := newFakeRunner() writes := 0 hooks := defaultLifecycleHooks hooks.writeState = func(path string, state State) error { writes++ if writes == failAt { return errors.New("injected state write failure") } return writeState(path, state) } result, err := updateWithHooks(context.Background(), fake, Request{ StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true, }, hooks) if err == nil { t.Fatal("updateWithHooks() error = nil, want injected state failure") } if fake.currentImage != "sha256:old" { t.Fatalf("current image = %q, want restored previous", fake.currentImage) } wantPhase := Phase("") if failAt == 2 || failAt == 3 { wantPhase = PhaseFailed } else if failAt >= 4 { wantPhase = PhaseRolledBack } if result.Phase != wantPhase { t.Fatalf("phase = %q, want %q for write %d", result.Phase, wantPhase, failAt) } if fake.maintenance { t.Fatal("maintenance remained active after proven stable recovery") } }) } } func TestCompensationWriteFailureKeepsMaintenanceActiveForExplicitRecovery(t *testing.T) { fake := newFakeRunner() fake.fail = "health" writes := 0 hooks := defaultLifecycleHooks hooks.writeState = func(path string, state State) error { writes++ if writes == 5 { return errors.New("injected compensation state write failure") } return writeState(path, state) } result, err := updateWithHooks(context.Background(), fake, Request{ StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true, }, hooks) if err == nil || result.Phase != PhaseFailed { t.Fatalf("result=%+v error=%v, want failed recovery", result, err) } if !fake.maintenance { t.Fatal("maintenance was cleared without durable rollback state") } } func TestMaintenanceClearAndCompensationFailuresRemainGated(t *testing.T) { for _, failure := range []string{"maintenance-clear", "compensation"} { t.Run(failure, func(t *testing.T) { fake := newFakeRunner() fake.fail = failure result, err := Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true}) if err == nil || result.Phase != PhaseFailed { t.Fatalf("result=%+v error=%v", result, err) } if !fake.maintenance { t.Fatal("maintenance was cleared after an unverified terminal failure") } }) } } // Catches maintenance recovery clearing admission after the obsolete composite smoke rather than // the same dedicated Pi Management smoke contract used for ordinary image verification. func TestRecoverMaintenanceClearsOnlyAfterTerminalStateAndVerifiedSmoke(t *testing.T) { fake := newFakeRunner() fake.maintenance = true statePath := filepath.Join(t.TempDir(), "state.json") previous := stateImageForTest(t, fake) state := State{Transaction: "recover-test", Phase: PhaseRolledBack, MutationStarted: true, Previous: previous} writeStateForTest(t, statePath, state) overridePath := lifecycleOverridePath(statePath, state.Transaction) if err := writeLifecycleOverride(overridePath, previous.Reference); err != nil { t.Fatal(err) } if err := writeLifecycleOverride(currentImageOverridePath(statePath), previous.Reference); err != nil { t.Fatal(err) } if err := RecoverMaintenance(context.Background(), fake, statePath, true); err != nil { t.Fatalf("RecoverMaintenance() error = %v", err) } if fake.maintenance { t.Fatal("maintenance remained active after verified terminal recovery") } if _, err := os.Stat(overridePath); !errors.Is(err, os.ErrNotExist) { t.Fatalf("lifecycle override still exists: %v", err) } if selected := readSelectorReference(t, currentImageOverridePath(statePath)); selected != previous.Reference { t.Fatalf("maintenance cleanup changed durable selector to %q", selected) } assertCalled(t, fake.calls, "/pi-management/test") } func TestRecoverMaintenanceRefusesPendingTransaction(t *testing.T) { fake := newFakeRunner() fake.maintenance = true statePath := filepath.Join(t.TempDir(), "state.json") writeStateForTest(t, statePath, State{Phase: PhaseRecreated, MutationStarted: true, Previous: stateImageForTest(t, fake)}) err := RecoverMaintenance(context.Background(), fake, statePath, true) if !errors.Is(err, ErrInterruptedUpdate) { t.Fatalf("RecoverMaintenance() error = %v, want ErrInterruptedUpdate", err) } if !fake.maintenance { t.Fatal("pending transaction maintenance was cleared") } } func TestRecoverMaintenanceCompletesAnInterruptedDurablePromotion(t *testing.T) { fake := newFakeRunner() fake.maintenance = true fake.currentImage = "sha256:candidate" fake.version = "0.81.0" fake.expectedVersion = "0.81.0" fake.labelVersion = "0.81.0" statePath := filepath.Join(t.TempDir(), ".thothctl", "update-state.json") previous := stateImageForTest(t, newFakeRunner()) candidate := previous candidate.ID = "sha256:candidate" candidate.Reference = "thothii-core:thothctl-recover-candidate" fake.tags[candidate.Reference] = candidate.ID state := State{ Transaction: "promotion-recovery", Phase: PhasePromoting, MutationStarted: true, Target: Target{Version: "0.81.0", Source: string(BuildSource)}, Previous: previous, Candidate: candidate, } writeStateForTest(t, statePath, state) if err := writeLifecycleOverride(lifecycleOverridePath(statePath, state.Transaction), candidate.Reference); err != nil { t.Fatal(err) } if err := RecoverMaintenance(context.Background(), fake, statePath, true); err != nil { t.Fatalf("RecoverMaintenance() promotion error = %v", err) } if selected := readSelectorReference(t, currentImageOverridePath(statePath)); selected != candidate.Reference { t.Fatalf("recovered selector = %q, want %q", selected, candidate.Reference) } if recovered, err := readState(statePath); err != nil || recovered.Phase != PhaseVerified { t.Fatalf("recovered state = %+v, %v; want verified", recovered, err) } } func TestRollbackFinalStateWriteFailureKeepsMaintenanceAndOverrideForRecovery(t *testing.T) { fake := newFakeRunner() statePath := filepath.Join(t.TempDir(), "state.json") previous := stateImageForTest(t, fake) previous.Reference = "thothii-core:thothctl-rollback-test-previous" fake.tags[previous.Reference] = previous.ID writeStateForTest(t, statePath, State{ Transaction: "rollback-test", Phase: PhaseRecreated, Previous: previous, }) hooks := defaultLifecycleHooks hooks.writeState = func(string, State) error { return errors.New("injected rollback state write failure") } result, err := rollbackWithHooks(context.Background(), fake, statePath, pairedRestartStatePath(statePath), true, hooks) if err == nil || result.Phase != PhaseFailed { t.Fatalf("rollbackWithHooks() = %+v, %v; want failed durable finalization", result, err) } if !fake.maintenance { t.Fatal("maintenance was cleared without durable rollback finalization") } if selected := readSelectorReference(t, currentImageOverridePath(statePath)); selected != previous.Reference { t.Fatalf("durable rollback selector = %q, want %q", selected, previous.Reference) } } func TestUpdateDoesNotRecreateWhenPreflightFails(t *testing.T) { for _, failure := range []string{"preflight"} { t.Run(failure, func(t *testing.T) { fake := newFakeRunner() fake.fail = failure result, err := Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true}) if err == nil { t.Fatal("Update() error = nil, want failure") } if failure == "preflight" && result.Phase == PhaseRolledBack { t.Fatalf("preflight failure unexpectedly rolled back: %+v", result) } assertNotCalled(t, fake.calls, "force-recreate") }) } } func TestCandidateBuildAndPullFailuresRemainPreMutationAndNeverRecreateCore(t *testing.T) { for _, testCase := range []struct { name string source Source image string failure string }{ {name: "build", source: BuildSource, failure: "build"}, {name: "pull", source: PullSource, image: "registry.example.invalid/core@sha256:" + strings.Repeat("a", 64), failure: "pull"}, {name: "candidate tag", source: PullSource, image: "registry.example.invalid/core@sha256:" + strings.Repeat("b", 64), failure: "tag"}, } { t.Run(testCase.name, func(t *testing.T) { fake := newFakeRunner() fake.fail = testCase.failure statePath := filepath.Join(t.TempDir(), ".thothctl", "update-state.json") result, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: testCase.source, Image: testCase.image, Confirm: true}) if err == nil { t.Fatal("Update() error = nil, want preparation failure") } if result.Phase != PhaseFailed { t.Fatalf("phase = %q, want safe failed preparation", result.Phase) } state, stateErr := readState(statePath) if stateErr != nil { t.Fatal(stateErr) } if state.MutationStarted { t.Fatal("preparation failure recorded mutationStarted") } assertNotCalled(t, fake.calls, "force-recreate") if fake.maintenance { t.Fatal("maintenance remained active after safe preparation failure") } }) } } func TestSuccessfulCompensationPreservesTheOriginalTypedCause(t *testing.T) { for _, cause := range []error{ErrActiveSessions, ErrInterruptedUpdate} { fake := newFakeRunner() fake.maintenance = true statePath := filepath.Join(t.TempDir(), ".thothctl", "update-state.json") state := State{Transaction: "typed-cause", Phase: PhaseRecreated, MutationStarted: true, Previous: stateImageForTest(t, fake)} result, err, clear := compensate(context.Background(), fake, statePath, lifecycleOverridePath(statePath, state.Transaction), state, cause, defaultLifecycleHooks) if result.Phase != PhaseRolledBack || !clear { t.Fatalf("compensation = %+v, clear=%t; want successful rollback", result, clear) } if !errors.Is(err, cause) { t.Fatalf("compensation error = %v, want errors.Is(..., %v)", err, cause) } if !strings.Contains(err.Error(), "previous core image was restored") { t.Fatalf("compensation error = %v, want rollback-success report", err) } } } func TestUpdateRequiresConfirmationAndDrainsActiveSessions(t *testing.T) { fake := newFakeRunner() _, err := Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource}) if !errors.Is(err, ErrConfirmationRequired) { t.Fatalf("Update() error = %v, want confirmation error", err) } assertNotCalled(t, fake.calls, "compose") fake = newFakeRunner() fake.activeSessions = true _, err = Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true}) if !errors.Is(err, ErrActiveSessions) { t.Fatalf("Update() error = %v, want active-sessions error", err) } fake = newFakeRunner() fake.activeSessions = true _, err = Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true, Drain: true}) if err != nil { t.Fatalf("Update() with drain error = %v", err) } assertCalled(t, fake.calls, "http://127.0.0.1:8787/sessions?scope=all") assertCalled(t, fake.calls, "/internal/maintenance/activate") assertCalled(t, fake.calls, "/internal/maintenance/deactivate") } func TestRollbackRestoresInterruptedOrPreviouslyRecordedState(t *testing.T) { fake := newFakeRunner() statePath := filepath.Join(t.TempDir(), "state.json") configured, err := renderedCore(context.Background(), fake) if err != nil { t.Fatal(err) } previous, err := runningImage(context.Background(), fake, "thothii-core:local") if err != nil { t.Fatal(err) } previous.ConfigurationSHA = configured.ConfigurationSHA previous.Reference = "thothii-core:thothctl-test-previous" fake.tags[previous.Reference] = previous.ID writeStateForTest(t, statePath, State{Version: 1, Phase: PhaseRecreated, Previous: previous}) result, err := Rollback(context.Background(), fake, statePath, pairedRestartStatePath(statePath), true) if err != nil { t.Fatalf("Rollback() error = %v", err) } if result.Phase != PhaseRolledBack { t.Fatalf("phase = %q, want %q", result.Phase, PhaseRolledBack) } assertCalled(t, fake.calls, "image tag sha256:old thothii-core:thothctl-test-previous") assertCalled(t, fake.calls, "up --detach --wait --wait-timeout 45 --no-deps --force-recreate core") } func TestUpdateRefusesToOverwriteInterruptedRecoveryState(t *testing.T) { fake := newFakeRunner() statePath := filepath.Join(t.TempDir(), "state.json") writeStateForTest(t, statePath, State{Phase: PhaseRecreated, MutationStarted: true, Previous: Image{ID: "sha256:old", Reference: "thothii-core:local", MountFingerprint: mountFingerprint(nil)}}) _, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true}) if !errors.Is(err, ErrInterruptedUpdate) { t.Fatalf("Update() error = %v, want interrupted update error", err) } assertNotCalled(t, fake.calls, "compose") } func TestFailedRestartBlocksUpdateAndRollbackAndMalformedStateAlsoRejects(t *testing.T) { for _, operation := range []string{"update", "rollback"} { for _, restartState := range []string{"failed", "malformed"} { t.Run(operation+"_"+restartState, func(t *testing.T) { dir := t.TempDir() fake := newFakeRunner() updateStatePath := filepath.Join(dir, "update-state.json") restartStatePath := filepath.Join(dir, "restart-state.json") if restartState == "failed" { fake.fail = "health" _, restartErr := Restart(context.Background(), fake, RestartRequest{ StatePath: restartStatePath, UpdateStatePath: updateStatePath, Confirm: true, }) var recovery *RecoveryRequiredError if !errors.As(restartErr, &recovery) { t.Fatalf("Restart() error = %v, want failed restart recovery state", restartErr) } fake.fail = "" } else if err := os.WriteFile(restartStatePath, []byte("{malformed"), 0o600); err != nil { t.Fatal(err) } if operation == "rollback" { previous := stateImageForTest(t, fake) writeStateForTest(t, updateStatePath, State{ Transaction: "rollback-target", Phase: PhaseRecreated, Target: Target{Version: fake.version, Source: string(BuildSource)}, Previous: previous, MutationStarted: true, }) } fake.calls = nil var err error if operation == "update" { _, err = Update(context.Background(), fake, Request{ StatePath: updateStatePath, RestartStatePath: restartStatePath, Version: "0.81.0", Source: BuildSource, Confirm: true, }) } else { _, err = Rollback(context.Background(), fake, updateStatePath, restartStatePath, true) } if err == nil { t.Fatalf("%s accepted %s restart recovery state", operation, restartState) } if restartState == "failed" && !errors.Is(err, ErrInterruptedRestart) { t.Fatalf("%s error = %v, want ErrInterruptedRestart", operation, err) } assertNotCalled(t, fake.calls, "compose") }) } } } func TestUpdateCleansVerifiedRestartStateBeforeNormalLifecycleWork(t *testing.T) { dir := t.TempDir() fake := newFakeRunner() updateStatePath := filepath.Join(dir, "update-state.json") restartStatePath := filepath.Join(dir, "restart-state.json") state := State{ Transaction: "verified-restart", Phase: PhaseVerified, Target: Target{Version: fake.version, Source: "restart"}, Previous: stateImageForTest(t, fake), MutationStarted: true, } writeStateForTest(t, restartStatePath, state) overridePath := lifecycleOverridePath(restartStatePath, state.Transaction) if err := writeLifecycleOverride(overridePath, state.Previous.Reference); err != nil { t.Fatal(err) } result, err := Update(context.Background(), fake, Request{ StatePath: updateStatePath, RestartStatePath: restartStatePath, Version: fake.version, Source: BuildSource, Confirm: true, }) if err != nil || result.Phase != PhaseNoop { t.Fatalf("Update() = %+v, %v; want normal no-op after verified restart", result, err) } for _, path := range []string{restartStatePath, overridePath} { if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { t.Fatalf("terminal restart artifact %s still exists: %v", path, err) } } } func TestRunningImageCapturesServerBindAndNamedMountIdentity(t *testing.T) { fake := newFakeRunner() fake.mountsJSON = `[{"Type":"bind","Source":"/srv/thothii/data","Destination":"/data","RW":true},{"Type":"bind","Source":"/srv/thothii/pi","Destination":"/home/thoth/.pi","RW":true},{"Type":"volume","Name":"sessions","Source":"/var/lib/docker/volumes/sessions/_data","Destination":"/data/sessions","RW":true}]` image, err := runningImage(context.Background(), fake, "thothii-core:local") if err != nil { t.Fatalf("runningImage() error = %v", err) } if len(image.Mounts) != 3 || image.Mounts[0].Type != "bind" || image.Mounts[0].Destination != "/data" { t.Fatalf("mounts = %#v", image.Mounts) } if strings.Contains(fmt.Sprint(image), "/srv/thothii") || image.Mounts[0].SourceSHA256 == "" || image.MountFingerprint == "" { t.Fatalf("mount contract leaked a server source or lacks a safe fingerprint: %#v", image) } } func TestDockerDesktopBindAliasesPreserveLegacyStateOnlyOnDarwin(t *testing.T) { hostPath := "/private/var/folders/task/models.json" vmPath := "/host_mnt/private/var/folders/task/models.json" hostHash, vmHash := mountSourceHash(hostPath), mountSourceHash(vmPath) if hostHash == vmHash { t.Fatal("the persisted exact source hash changed instead of retaining state-v4 compatibility") } legacy := Mount{Type: "bind", SourceSHA256: vmHash, Destination: "/config/models.json"} darwinCurrent := Mount{Type: "bind", SourceSHA256: hostHash, SourceAliases: mountSourceAliases("bind", hostPath, "darwin"), Destination: "/config/models.json"} if !sameMounts([]Mount{legacy}, []Mount{darwinCurrent}) { t.Fatal("a legacy Docker Desktop source hash did not match its current Darwin alias") } linuxCurrent := Mount{Type: "bind", SourceSHA256: hostHash, SourceAliases: mountSourceAliases("bind", hostPath, "linux"), Destination: "/config/models.json"} if sameMounts([]Mount{legacy}, []Mount{linuxCurrent}) { t.Fatal("distinct Linux /host_mnt and host bind paths were collapsed") } if aliases := mountSourceAliases("volume", hostPath, "darwin"); len(aliases) != 0 { t.Fatalf("named-volume source received Docker Desktop bind aliases: %v", aliases) } } func TestCanonicalDigestReferenceRejectsCredentialsAndURLForms(t *testing.T) { valid := "registry.example.invalid/thothii-core@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" if got, err := canonicalDigestReference(valid); err != nil || got != valid { t.Fatalf("canonicalDigestReference() = %q, %v", got, err) } for _, invalid := range []string{"https://registry.example.invalid/a@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "user:pass@registry.example/a@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "registry.example/a@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa?token=x"} { if _, err := canonicalDigestReference(invalid); err == nil { t.Fatalf("accepted unsafe reference %q", invalid) } } } type fakeRunner struct { calls []string fail string version string expectedVersion string labelVersion string activeSessions bool built bool recreated bool currentImage string mountsJSON string sessionsWire string configuredImage string buildReference string tags map[string]string imageVersions map[string]string maintenance bool maintenanceAtRecreate []bool lostMaintenanceResponse string restartBackendOnLoss bool backendRestarts int dropMaintenanceAfterCandidate bool modelsWire string piManagementOptionsWire string piManagementTestWire string rollbackPrepared bool coreRunning bool execFailuresWhileStopped int maintenanceHelperImages []string maintenanceClearImages []string failDeactivationDurability bool deactivationFailed bool restoredProofComplete bool candidateVersion string candidateExpectedVersion string candidateLabelVersion string } func newFakeRunner() *fakeRunner { return &fakeRunner{ version: "0.80.3", expectedVersion: "0.80.3", labelVersion: "0.80.3", currentImage: "sha256:old", configuredImage: "thothii-core:local", tags: map[string]string{"thothii-core:local": "sha256:old"}, imageVersions: map[string]string{"sha256:old": "0.80.3"}, coreRunning: true, candidateVersion: "0.81.0", candidateExpectedVersion: "0.81.0", candidateLabelVersion: "0.81.0", } } func (f *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) { call := strings.Join(args, " ") f.calls = append(f.calls, call) if !f.coreRunning && containsArg(args, "exec") { f.execFailuresWhileStopped++ return compose.Result{ExitCode: 1}, errors.New("core service is not running") } if (f.built || f.recreated) && f.fail != "compensation" && strings.Contains(call, "image tag sha256:old") { f.fail = "" } if f.fail == "preflight" && strings.Contains(call, "config --format json") { return compose.Result{ExitCode: 1}, errors.New("provider token=secret") } if f.fail == "build" && containsArg(args, "build") { return compose.Result{ExitCode: 1}, errors.New("build token=secret") } if f.fail == "pull" && len(args) > 0 && args[0] == "pull" { return compose.Result{ExitCode: 1}, errors.New("pull token=secret") } if f.fail == "tag" && len(args) >= 4 && args[0] == "image" && args[1] == "tag" && strings.Contains(args[3], "-candidate") { return compose.Result{ExitCode: 1}, errors.New("tag token=secret") } if f.fail == "health" && (f.built || f.recreated) && strings.Contains(call, "curl -fsS http://127.0.0.1:8787/health") { return compose.Result{ExitCode: 1}, errors.New("health token=secret") } if f.fail == "compensation" && (f.built || f.recreated) && !f.rollbackPrepared && strings.Contains(call, "curl -fsS http://127.0.0.1:8787/health") { return compose.Result{ExitCode: 1}, errors.New("candidate health failure") } if f.fail == "version" && (f.built || f.recreated) && strings.Contains(call, "pi --version") && strings.Contains(call, "exec") { return compose.Result{ExitCode: 1}, errors.New("version token=secret") } if f.fail == "smoke" && (f.built || f.recreated) && strings.Contains(call, "127.0.0.1:8787/pi-management/test") { return compose.Result{ExitCode: 1}, errors.New("smoke token=secret") } switch { case strings.Contains(call, "config --format json"): endpoint := "https://llm.example.invalid" if f.fail == "config-drift" && (f.built || f.recreated) { endpoint = "https://drift.example.invalid" } return compose.Result{Stdout: `{"services":{"core":{"image":"` + selectedCoreReference(args, f.configuredImage) + `","environment":{"THT_LLM_URL":"` + endpoint + `"}}}}`}, nil case strings.Contains(call, "ps --status running -q core"): if f.coreRunning { return compose.Result{Stdout: "core-container\n"}, nil } return compose.Result{}, nil case strings.Contains(call, "ps -q core"): return compose.Result{Stdout: "core-container\n"}, nil case strings.Contains(call, "inspect --format {{.Image}}"): if f.fail == "image-drift" && f.recreated { return compose.Result{Stdout: "sha256:unexpected\n"}, nil } return compose.Result{Stdout: f.currentImage + "\n"}, nil case strings.Contains(call, "io.thothii.pi.version"): return compose.Result{Stdout: f.labelVersion + "\n"}, nil case strings.Contains(call, "inspect --format {{json .Mounts}}"): if f.fail == "mount-drift" && (f.built || f.recreated) { return compose.Result{Stdout: `[{"Type":"volume","Name":"wrong-settings","Source":"wrong-settings","Destination":"/data/settings","RW":true}]`}, nil } if f.mountsJSON != "" { return compose.Result{Stdout: f.mountsJSON}, nil } return compose.Result{Stdout: `[{"Type":"volume","Name":"settings","Source":"settings","Destination":"/data/settings","RW":true},{"Type":"volume","Name":"pi-state","Source":"pi-state","Destination":"/home/thoth/.pi","RW":true},{"Type":"volume","Name":"sessions","Source":"sessions","Destination":"/data/sessions","RW":true},{"Type":"volume","Name":"workspace-registry","Source":"workspace-registry","Destination":"/data/workspace-registry","RW":true}]`}, nil case strings.Contains(call, "/internal/maintenance/activate"): f.maintenance = true if f.fail == "maintenance-activate-durability" || f.fail == "maintenance-activate-durability-without-status-flag" { return compose.Result{ExitCode: 22}, errors.New("maintenance activation durability was not acknowledged") } if f.lostMaintenanceResponse == "activate" { f.lostMaintenanceResponse = "" if f.restartBackendOnLoss { f.backendRestarts++ } return compose.Result{ExitCode: 52}, errors.New("lost activation response") } return compose.Result{Stdout: `{"active":true,"admissions":0}`}, nil case strings.Contains(call, "/internal/maintenance/deactivate"): if f.fail == "maintenance-clear" { return compose.Result{ExitCode: 53}, errors.New("maintenance clear failure") } if f.failDeactivationDurability { f.deactivationFailed = true f.maintenance = true return compose.Result{ExitCode: 22}, errors.New("maintenance deactivation durability was not acknowledged") } f.maintenance = false f.maintenanceClearImages = append(f.maintenanceClearImages, f.currentImage) if f.lostMaintenanceResponse == "deactivate" { f.lostMaintenanceResponse = "" if f.restartBackendOnLoss { f.backendRestarts++ } return compose.Result{ExitCode: 52}, errors.New("lost deactivation response") } return compose.Result{Stdout: `{"active":false,"admissions":0}`}, nil case strings.Contains(call, "/internal/maintenance/status"): if f.fail == "maintenance-proof" && f.recreated { return compose.Result{Stdout: fmt.Sprintf(`{"active":%t,"admissions":0,"recoveryRequired":true}`, f.maintenance)}, nil } if f.fail == "maintenance-activate-durability" { return compose.Result{Stdout: fmt.Sprintf(`{"active":%t,"admissions":0,"recoveryRequired":true}`, f.maintenance)}, nil } if f.deactivationFailed { return compose.Result{Stdout: fmt.Sprintf(`{"active":%t,"admissions":0,"recoveryRequired":true}`, f.maintenance)}, nil } return compose.Result{Stdout: fmt.Sprintf(`{"active":%t,"admissions":0}`, f.maintenance)}, nil case strings.Contains(call, "/sessions?scope=all"): if f.sessionsWire != "" { return compose.Result{Stdout: f.sessionsWire}, nil } if f.activeSessions { f.activeSessions = false return compose.Result{Stdout: `[{"status":"open","archived":false}]`}, nil } return compose.Result{Stdout: `[]`}, nil case containsArg(args, "run") && containsArg(args, "--entrypoint") && containsArg(args, "node"): reference := selectedCoreReference(args, f.configuredImage) f.maintenanceHelperImages = append(f.maintenanceHelperImages, reference) if reference == "" { return compose.Result{ExitCode: 1}, errors.New("maintenance helper has no selected image") } if f.tags[reference] != "sha256:old" { return compose.Result{ExitCode: 1}, errors.New("maintenance helper did not select the previous image") } f.maintenance = true return compose.Result{}, nil case containsArg(args, "build"): f.built = true f.buildReference = selectedCoreReference(args, f.configuredImage) f.tags[f.buildReference] = "sha256:candidate" f.imageVersions["sha256:candidate"] = "0.81.0" return compose.Result{}, nil case len(args) == 2 && args[0] == "pull": f.tags[args[1]] = "sha256:candidate" f.imageVersions["sha256:candidate"] = "0.81.0" return compose.Result{}, nil case len(args) >= 4 && args[0] == "image" && args[1] == "tag": source, target := args[2], args[3] id := source if tagged, ok := f.tags[source]; ok { id = tagged } f.tags[target] = id if f.built && id == "sha256:old" { f.rollbackPrepared = true } return compose.Result{}, nil case containsArg(args, "up"): f.recreated = true f.maintenanceAtRecreate = append(f.maintenanceAtRecreate, f.maintenance) reference := selectedCoreReference(args, f.configuredImage) if id, ok := f.tags[reference]; ok { f.currentImage = id } if version, ok := f.imageVersions[f.currentImage]; ok { f.version = version } if f.currentImage == "sha256:candidate" { f.version = f.candidateVersion f.expectedVersion = f.candidateExpectedVersion f.labelVersion = f.candidateLabelVersion } else if f.currentImage == "sha256:old" { f.expectedVersion = "0.80.3" f.labelVersion = "0.80.3" } f.coreRunning = true if f.currentImage == "sha256:candidate" { f.restoredProofComplete = false } if f.fail == "dead-candidate" && f.currentImage == "sha256:candidate" { f.coreRunning = false } if f.dropMaintenanceAfterCandidate && f.currentImage == "sha256:candidate" { f.maintenance = false f.dropMaintenanceAfterCandidate = false } if f.fail == "recreate" && (f.built || f.recreated) { return compose.Result{ExitCode: 54}, errors.New("recreate failure") } if f.fail == "compensation" && f.rollbackPrepared { return compose.Result{ExitCode: 55}, errors.New("rollback recreate failure") } return compose.Result{}, nil case strings.Contains(call, "pi --version"): return compose.Result{Stdout: f.version + "\n"}, nil case strings.Contains(call, "PI_VERSION"): return compose.Result{Stdout: f.expectedVersion + "\n"}, nil case strings.Contains(call, "/pi-management/options"): if f.piManagementOptionsWire != "" { return compose.Result{Stdout: f.piManagementOptionsWire}, nil } return compose.Result{Stdout: `{"providers":["provider"],"models":[{"id":"model","provider":"provider"}],"reasoning":["low","medium","high"]}`}, nil case strings.Contains(call, "/pi-management/test"): if f.currentImage == "sha256:old" { f.restoredProofComplete = true } if f.piManagementTestWire != "" { return compose.Result{Stdout: f.piManagementTestWire}, nil } return compose.Result{Stdout: `{"ready":true}`}, nil case strings.Contains(call, "/models"): if f.modelsWire != "" { return compose.Result{Stdout: f.modelsWire}, nil } return compose.Result{Stdout: `{"models":[{"id":"model","provider":"provider"}]}`}, nil case strings.Contains(call, "/settings"): if f.currentImage == "sha256:old" { f.restoredProofComplete = true } return compose.Result{Stdout: `{"provider":"provider","model":"model","thinking":"medium"}`}, nil case strings.Contains(call, "/health"): return compose.Result{Stdout: `{"status":"ok"}`}, nil } return compose.Result{}, nil } func containsArg(args []string, wanted string) bool { for _, arg := range args { if arg == wanted { return true } } return false } func selectedCoreReference(args []string, fallback string) string { for index := 0; index+1 < len(args); index++ { if args[index] != "-f" || (!strings.Contains(filepath.Base(args[index+1]), "pi-lifecycle-") && filepath.Base(args[index+1]) != "current-image.yaml") { continue } contents, err := os.ReadFile(args[index+1]) if err != nil { continue } for _, line := range strings.Split(string(contents), "\n") { line = strings.TrimSpace(line) if !strings.HasPrefix(line, "image:") { continue } value := strings.TrimSpace(strings.TrimPrefix(line, "image:")) if decoded, err := strconv.Unquote(value); err == nil { return decoded } return value } } return fallback } func readSelectorReference(t *testing.T, path string) string { t.Helper() contents, err := os.ReadFile(path) if err != nil { t.Fatal(err) } for _, line := range strings.Split(string(contents), "\n") { line = strings.TrimSpace(line) if !strings.HasPrefix(line, "image:") { continue } value := strings.TrimSpace(strings.TrimPrefix(line, "image:")) if decoded, err := strconv.Unquote(value); err == nil { return decoded } return value } t.Fatalf("selector %s has no image", path) return "" } func callIndex(calls []string, contains string) int { for index, call := range calls { if strings.Contains(call, contains) { return index } } return -1 } func lastCallIndexBefore(calls []string, contains string, before int) int { for index := before - 1; index >= 0; index-- { if strings.Contains(calls[index], contains) { return index } } return -1 } func assertCalled(t *testing.T, calls []string, want string) { t.Helper() for _, call := range calls { if strings.Contains(call, want) { return } } t.Fatalf("calls %v did not include %q", calls, want) } func assertNotCalled(t *testing.T, calls []string, prohibited string) { t.Helper() for _, call := range calls { if strings.Contains(call, prohibited) { t.Fatalf("calls %v unexpectedly included %q", calls, prohibited) } } } func assertMaintenanceClearedAfterRestoredProof(t *testing.T, fake *fakeRunner) { t.Helper() if fake.maintenance { t.Fatal("maintenance remained active after restored-core proof") } if !fake.restoredProofComplete { t.Fatal("maintenance cleared before restored settings smoke completed") } if len(fake.maintenanceClearImages) == 0 { t.Fatal("maintenance was never durably cleared") } for _, image := range fake.maintenanceClearImages { if image != "sha256:old" { t.Fatalf("maintenance cleared while image %q was selected; want previous image", image) } } } func readStateBytes(t *testing.T, path string) []byte { t.Helper() contents, err := os.ReadFile(path) if err != nil { t.Fatal(err) } return contents } func writeStateForTest(t *testing.T, path string, state State) { t.Helper() if err := writeState(path, state); err != nil { t.Fatal(err) } } func stateImageForTest(t *testing.T, fake *fakeRunner) Image { t.Helper() configured, err := renderedCore(context.Background(), fake) if err != nil { t.Fatal(err) } image, err := runningImage(context.Background(), fake, fake.configuredImage) if err != nil { t.Fatal(err) } image.ConfigurationSHA = configured.ConfigurationSHA image.Reference = "thothii-core:thothctl-test-previous" return image }