mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix: finalize durable Pi lifecycle

Browse Source
This commit is contained in:
marcopan
2026-08-04 21:34:08 +02:00
parent 5ba2821a1b
commit a368889838
20 changed files with 1016 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/thothctl/cmd/thothctl/main.go
+3 -3
View File
@@ -189,7 +189,7 @@ func piCommand(ctx context.Context, installation config.Installation, runner com
fmt.Fprintf(stdout, "Pi defaults applied and read back. Provider credentials remain only in the host file %s (mode 0600). Never pass credentials to thothctl.\n", authFile)
return 0
case "update":
request, err := parsePiUpdateArgs(args[1:], filepath.Join(installation.ProjectDirectory, ".thothctl", "update-state.json"))
request, err := parsePiUpdateArgs(args[1:], installation.UpdateStatePath())
if err != nil {
return commandUsageError(stderr, err.Error())
}
@@ -207,7 +207,7 @@ func piCommand(ctx context.Context, installation config.Installation, runner com
if len(args) != 2 || args[1] != "--yes" {
return commandUsageError(stderr, "pi rollback requires --yes")
}
result, err := pi.Rollback(ctx, controlled, filepath.Join(installation.ProjectDirectory, ".thothctl", "update-state.json"), true)
result, err := pi.Rollback(ctx, controlled, installation.UpdateStatePath(), true)
if err != nil {
return piFailure(stderr, err, secretValues)
}
@@ -223,7 +223,7 @@ func piCommand(ctx context.Context, installation config.Installation, runner com
return 0
}
if len(args) == 3 && args[1] == "recover" && args[2] == "--yes" {
statePath := filepath.Join(installation.ProjectDirectory, ".thothctl", "update-state.json")
statePath := installation.UpdateStatePath()
if err := pi.RecoverMaintenance(ctx, controlled, statePath, true); err != nil {
return piFailure(stderr, err, secretValues)
}
tools/thothctl/cmd/thothctl/main_test.go
+28 -1
View File
@@ -12,6 +12,7 @@ import (
"testing"
"github.com/aritmolab/thothii/tools/thothctl/internal/compose"
"github.com/aritmolab/thothii/tools/thothctl/internal/config"
"github.com/aritmolab/thothii/tools/thothctl/internal/pi"
"github.com/aritmolab/thothii/tools/thothctl/internal/testsupport"
)
@@ -50,7 +51,8 @@ func TestResolvePiConfigureRequiresExplicitFlagsWithoutTTY(t *testing.T) {
func TestPiLifecycleContractErrorsExitTwo(t *testing.T) {
for _, lifecycleErr := range []error{pi.ErrActiveSessions, pi.ErrInterruptedUpdate} {
var stderr bytes.Buffer
if code := piFailure(&stderr, lifecycleErr, nil); code != 2 {
wrapped := fmt.Errorf("automatic rollback succeeded: %w", lifecycleErr)
if code := piFailure(&stderr, wrapped, nil); code != 2 {
t.Errorf("piFailure(%v) = %d, want 2", lifecycleErr, code)
}
}
@@ -373,6 +375,28 @@ func TestRunStatusUsesStableComposeArguments(t *testing.T) {
}
}
func TestRunStartAutomaticallyUsesTheDurableCurrentImageOverride(t *testing.T) {
fixture := newCLIFixture(t, "SAFE_VALUE=1\n")
fixture.setEnvironment(t)
installation, err := config.Load(fixture.installationPath)
if err != nil {
t.Fatal(err)
}
currentImage := installation.CurrentImageOverridePath()
if err := os.MkdirAll(filepath.Dir(currentImage), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(currentImage, []byte("services:\n core:\n image: thothii-core:verified\n"), 0o600); err != nil {
t.Fatal(err)
}
var stdout, stderr bytes.Buffer
if code := run(context.Background(), []string{"--installation", fixture.installationPath, "start"}, &stdout, &stderr); code != 0 {
t.Fatalf("start exit = %d, stderr = %s", code, stderr.String())
}
assertInvocationContains(t, fixture.invocations(t), "-f", currentImage, "up", "--detach", "--remove-orphans")
}
func TestRunExplainsWhenDockerIsNotAvailable(t *testing.T) {
fixture := newCLIFixture(t, "SAFE_VALUE=1\n")
fixture.setEnvironment(t)
@@ -591,8 +615,11 @@ printf '%s\n' -- >> "$THOTHCTL_FAKE_ARGS"
case " $* " in
*" config --format json "*) printf '%s\n' '{"volumes":{"settings":{}},"services":{"core":{"image":"thothii-core:local","environment":{"THT_LLM_URL":"https://llm.example.invalid"}}}}' ;;
*" ps --format json "*) printf '%s\n' '[{"Service":"core","State":"running","Health":"healthy"},{"Service":"frontend","State":"running","Health":"healthy"}]' ;;
*"io.thothii.pi.version"*) printf '%s\n' '0.80.3' ;;
*"PI_VERSION"*) printf '%s\n' '0.80.3' ;;
*" pi --version "*) printf '%s\n' '0.80.3' ;;
*"/models "*) printf '%s\n' '{"models":[{"provider":"provider","id":"model"}]}' ;;
*"settings-cli.js --snapshot"*) printf '%s\n' '{"exists":false,"rawBase64":""}' ;;
*"/settings "*) printf '%s\n' '{"provider":"provider","model":"model","thinking":"medium"}' ;;
*"/internal/maintenance/status "*) printf '%s\n' '{"active":true,"admissions":0}' ;;
*" logs "*) printf '%s\n' "$THOTHCTL_FAKE_LOG" ;;
tools/thothctl/go.mod
+4 -2
View File
@@ -1,6 +1,8 @@
module github.com/aritmolab/thothii/tools/thothctl
go 1.24
go 1.24.0
toolchain go1.24.13
require gopkg.in/yaml.v3 v3.0.1
@@ -9,7 +11,7 @@ require (
github.com/distribution/reference v0.6.0
github.com/gofrs/flock v0.12.1
github.com/sirupsen/logrus v1.9.0
golang.org/x/sys v0.22.0
golang.org/x/sys v0.41.0
)
require github.com/opencontainers/go-digest v1.0.0 // indirect
tools/thothctl/go.sum
+2
View File
@@ -24,6 +24,8 @@ golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
tools/thothctl/internal/config/installation.go
+33 -3
View File
@@ -101,6 +101,13 @@ func Load(path string) (Installation, error) {
return Installation{}, err
}
}
if info, err := os.Lstat(installation.CurrentImageOverridePath()); err == nil {
if !info.Mode().IsRegular() {
return Installation{}, errors.New("installation current-image override must be a regular file")
}
} else if !errors.Is(err, os.ErrNotExist) {
return Installation{}, errors.New("installation current-image override could not be inspected")
}
return installation, nil
}
@@ -111,7 +118,26 @@ func (i Installation) ComposeFiles() []string {
filepath.Join(i.ProjectDirectory, "compose.yaml"),
filepath.Join(i.ProjectDirectory, "deploy", "compose."+i.Profile+".yaml"),
}
return append(files, i.Overrides...)
files = append(files, i.Overrides...)
currentImage := i.CurrentImageOverridePath()
if info, err := os.Lstat(currentImage); err == nil && info.Mode().IsRegular() {
files = append(files, currentImage)
}
return files
}
// ControlDirectory contains state that is private to one installation descriptor, even when
// multiple installations intentionally share one source checkout.
func (i Installation) ControlDirectory() string {
return filepath.Join(i.ProjectDirectory, ".thothctl", i.ProjectName())
}
func (i Installation) CurrentImageOverridePath() string {
return filepath.Join(i.ControlDirectory(), "current-image.yaml")
}
func (i Installation) UpdateStatePath() string {
return filepath.Join(i.ControlDirectory(), "update-state.json")
}
// ProjectName is stable for one installation and avoids collisions between different checkouts.
@@ -168,9 +194,13 @@ func (i Installation) SecretFiles() ([]string, error) {
// callers. It is used only for operator-visible file locations, never for secret content.
func (i Installation) EnvironmentValue(name string) (string, error) {
contents, err := safeio.ReadCanonicalRegular(i.EnvFile, maxEnvironmentFileBytes)
if err != nil { return "", errors.New("installation environment could not be read") }
if err != nil {
return "", errors.New("installation environment could not be read")
}
values, err := parseComposeDotenv(contents)
if err != nil { return "", errors.New("installation environment could not be read") }
if err != nil {
return "", errors.New("installation environment could not be read")
}
return values[name], nil
}
tools/thothctl/internal/config/installation_test.go
+64
View File
@@ -51,6 +51,51 @@ func TestLoadSelectsServerComposeFiles(t *testing.T) {
assertStringsEqual(t, installation.ComposeFiles(), want)
}
func TestComposeArgsAutomaticallyIncludeTheInstallationCurrentImageOverride(t *testing.T) {
t.Parallel()
installationPath, _, _, _ := writeInstallation(t, "local")
seed, err := Load(installationPath)
if err != nil {
t.Fatal(err)
}
currentImage := seed.CurrentImageOverridePath()
if err := os.MkdirAll(filepath.Dir(currentImage), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(currentImage, []byte("services:\n core:\n image: candidate\n"), 0o600); err != nil {
t.Fatal(err)
}
installation, err := Load(installationPath)
if err != nil {
t.Fatal(err)
}
args := installation.ComposeArgs("up", "--detach")
want := []string{"-f", currentImage, "up", "--detach"}
if !containsSequence(args, want) {
t.Fatalf("ComposeArgs() = %#v, want durable override immediately before command", args)
}
}
func TestInstallationControlPathsAreIsolatedForDescriptorsSharingOneCheckout(t *testing.T) {
projectDirectory := t.TempDir()
first := Installation{Path: filepath.Join(t.TempDir(), installationFileName), ProjectDirectory: projectDirectory}
second := Installation{Path: filepath.Join(t.TempDir(), installationFileName), ProjectDirectory: projectDirectory}
if first.CurrentImageOverridePath() == second.CurrentImageOverridePath() {
t.Fatalf("shared-checkout installations reused %q", first.CurrentImageOverridePath())
}
for _, installation := range []Installation{first, second} {
if filepath.Dir(filepath.Dir(installation.CurrentImageOverridePath())) != filepath.Join(projectDirectory, ".thothctl") {
t.Fatalf("current-image path %q is not installation-specific under .thothctl", installation.CurrentImageOverridePath())
}
if filepath.Dir(installation.UpdateStatePath()) != filepath.Dir(installation.CurrentImageOverridePath()) {
t.Fatalf("state %q and selector %q do not share one installation control directory", installation.UpdateStatePath(), installation.CurrentImageOverridePath())
}
}
}
func TestLoadRejectsRelativeInstallationPaths(t *testing.T) {
t.Parallel()
@@ -100,3 +145,22 @@ func assertStringsEqual(t *testing.T, got, want []string) {
}
}
}
func containsSequence(values, wanted []string) bool {
for start := range values {
if len(values)-start < len(wanted) {
continue
}
matched := true
for offset := range wanted {
if values[start+offset] != wanted[offset] {
matched = false
break
}
}
if matched {
return true
}
}
return false
}
tools/thothctl/internal/pi/commands.go
+94 -26
View File
@@ -1,8 +1,10 @@
package pi
import (
"bytes"
"context"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
@@ -26,6 +28,11 @@ type ModelOption struct {
ID string `json:"id"`
}
type settingsFileSnapshot struct {
Exists bool `json:"exists"`
RawBase64 string `json:"rawBase64"`
}
var internalIdentityHeaders = []string{
"-H", "x-thoth-principal-issuer: thothctl",
"-H", "x-thoth-principal-subject: thothctl-maintenance",
@@ -59,40 +66,34 @@ func Configure(ctx context.Context, runner Runner, value Defaults) error {
if !found {
return errors.New("provider/model is not in Pi options")
}
settingsArgs := append([]string{"exec", "-T", "core", "curl", "-fsS"}, internalIdentityHeaders...)
settingsArgs = append(settingsArgs, "http://127.0.0.1:8787/settings")
oldResult, err := runCompose(ctx, runner, settingsArgs...)
old, err := captureSettingsFile(ctx, runner)
if err != nil {
return commandError("Pi installation settings capture", oldResult, err)
return err
}
var old Defaults
if json.Unmarshal([]byte(oldResult.Stdout), &old) != nil || old.Provider == "" || old.Model == "" || old.Thinking == "" {
return errors.New("Pi installation settings capture is invalid")
oldEffective, err := readEffectiveSettings(ctx, runner)
if err != nil {
return err
}
restore := func(cause error) error {
result, restoreErr := writeDefaults(context.Background(), runner, old)
if restoreErr != nil {
return fmt.Errorf("%w; previous Pi settings could not be restored: recovery required", cause)
}
if result.ExitCode != 0 {
return fmt.Errorf("%w; previous Pi settings could not be restored: recovery required", cause)
}
verified, readErr := readDefaults(context.Background(), runner, settingsArgs)
if readErr != nil || verified != old {
if restoreErr := restoreSettingsFile(context.Background(), runner, old); restoreErr != nil {
return fmt.Errorf("%w; previous Pi settings restoration could not be verified: recovery required", cause)
}
restoredEffective, restoreErr := readEffectiveSettings(context.Background(), runner)
if restoreErr != nil || !bytes.Equal(restoredEffective, oldEffective) {
return fmt.Errorf("%w; previous effective Pi settings could not be verified: recovery required", cause)
}
return cause
}
result, err := writeDefaults(ctx, runner, value)
if err != nil {
return restore(commandError("Pi installation settings write", result, err))
}
settings, err := runCompose(ctx, runner, settingsArgs...)
settings, err := readEffectiveSettings(ctx, runner)
if err != nil {
return restore(commandError("Pi installation settings read-back", settings, err))
return restore(err)
}
var saved Defaults
if json.Unmarshal([]byte(settings.Stdout), &saved) != nil || saved.Provider != value.Provider || saved.Model != value.Model || saved.Thinking != value.Thinking {
if json.Unmarshal(settings, &saved) != nil || saved.Provider != value.Provider || saved.Model != value.Model || saved.Thinking != value.Thinking {
return restore(errors.New("Pi installation settings read-back did not match requested provider, model, and thinking"))
}
after, err := renderedCore(ctx, runner)
@@ -130,16 +131,55 @@ func writeDefaults(ctx context.Context, runner Runner, value Defaults) (compose.
return runCompose(ctx, runner, "exec", "-T", "core", "node", "/app/backend/dist/settings/settings-cli.js", "--provider", value.Provider, "--model", value.Model, "--thinking", value.Thinking)
}
func readDefaults(ctx context.Context, runner Runner, args []string) (Defaults, error) {
func captureSettingsFile(ctx context.Context, runner Runner) (settingsFileSnapshot, error) {
result, err := runCompose(ctx, runner, "exec", "-T", "core", "node", "/app/backend/dist/settings/settings-cli.js", "--snapshot")
if err != nil {
return settingsFileSnapshot{}, commandError("Pi installation settings snapshot", result, err)
}
var snapshot settingsFileSnapshot
if json.Unmarshal([]byte(result.Stdout), &snapshot) != nil {
return settingsFileSnapshot{}, errors.New("Pi installation settings snapshot is invalid")
}
raw, decodeErr := base64.StdEncoding.DecodeString(snapshot.RawBase64)
if decodeErr != nil || base64.StdEncoding.EncodeToString(raw) != snapshot.RawBase64 || (!snapshot.Exists && len(raw) != 0) {
return settingsFileSnapshot{}, errors.New("Pi installation settings snapshot is invalid")
}
return snapshot, nil
}
func restoreSettingsFile(ctx context.Context, runner Runner, snapshot settingsFileSnapshot) error {
payload, err := json.Marshal(snapshot)
if err != nil {
return errors.New("Pi installation settings snapshot could not be encoded")
}
args := []string{"compose", "exec", "-T", "core", "node", "/app/backend/dist/settings/settings-cli.js", "--restore"}
result, restoreErr := runner.Run(ctx, args, bytes.NewReader(payload))
verified, verifyErr := captureSettingsFile(ctx, runner)
if verifyErr == nil && verified == snapshot {
return nil
}
if restoreErr != nil {
return commandError("Pi installation settings restore", result, restoreErr)
}
return errors.New("Pi installation settings restore did not reproduce the exact prior file state")
}
func readEffectiveSettings(ctx context.Context, runner Runner) ([]byte, error) {
args := append([]string{"exec", "-T", "core", "curl", "-fsS"}, internalIdentityHeaders...)
args = append(args, "http://127.0.0.1:8787/settings")
result, err := runCompose(ctx, runner, args...)
if err != nil {
return Defaults{}, commandError("Pi installation settings restoration read-back", result, err)
return nil, commandError("Pi installation settings read-back", result, err)
}
var value Defaults
if json.Unmarshal([]byte(result.Stdout), &value) != nil {
return Defaults{}, errors.New("Pi installation settings restoration read-back is invalid")
var settings map[string]json.RawMessage
if json.Unmarshal([]byte(result.Stdout), &settings) != nil || settings == nil {
return nil, errors.New("Pi installation settings read-back is invalid")
}
return value, nil
canonical, err := json.Marshal(settings)
if err != nil {
return nil, errors.New("Pi installation settings read-back could not be normalized")
}
return canonical, nil
}
// Runner is the narrow, shell-free command boundary shared with thothctl.
@@ -165,9 +205,17 @@ func Doctor(ctx context.Context, runner Runner) error {
if _, err := renderedCore(ctx, runner); err != nil {
return err
}
if _, err := Status(ctx, runner); err != nil {
actual, err := Status(ctx, runner)
if err != nil {
return err
}
expected, label, err := expectedVersions(ctx, runner)
if err != nil {
return err
}
if actual != expected || actual != label {
return errors.New("Pi version does not match the image PI_VERSION and io.thothii.pi.version contract")
}
for _, check := range [][]string{
{"exec", "-T", "core", "sh", "-ceu", "test -w /home/thoth/.pi"},
{"exec", "-T", "core", "sh", "-ceu", "test -r /home/thoth/.pi/agent/auth.json"},
@@ -181,6 +229,26 @@ func Doctor(ctx context.Context, runner Runner) error {
return Test(ctx, runner)
}
func expectedVersions(ctx context.Context, runner Runner) (string, string, error) {
environment, err := runCompose(ctx, runner, "exec", "-T", "core", "sh", "-ceu", `printf '%s\n' "${PI_VERSION:-}"`)
if err != nil {
return "", "", commandError("Pi expected-version check", environment, err)
}
container, err := runCompose(ctx, runner, "ps", "-q", "core")
if err != nil || strings.TrimSpace(container.Stdout) == "" {
return "", "", commandError("Pi image-label check", container, err)
}
label, err := runner.Run(ctx, []string{"inspect", "--format", `{{ index .Config.Labels "io.thothii.pi.version" }}`, strings.TrimSpace(container.Stdout)}, nil)
if err != nil {
return "", "", commandError("Pi image-label check", label, err)
}
expectedValue, labelValue := strings.TrimSpace(environment.Stdout), strings.TrimSpace(label.Stdout)
if expectedValue == "" || labelValue == "" {
return "", "", errors.New("Pi image expected-version contract is empty")
}
return expectedValue, labelValue, nil
}
// Test performs the pre-Task-8 composite smoke through core's private loopback endpoint.
func Test(ctx context.Context, runner Runner) error {
if _, err := Status(ctx, runner); err != nil {
tools/thothctl/internal/pi/commands_test.go
+99 -10
View File
@@ -2,6 +2,7 @@ package pi
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"io"
@@ -16,15 +17,36 @@ func TestDoctorRequiresExternalEndpointAuthPiStateAndHealth(t *testing.T) {
if err := Doctor(context.Background(), fake); err != nil {
t.Fatalf("Doctor() error = %v", err)
}
for _, command := range []string{"pi --version", "test -w /home/thoth/.pi", "test -r /home/thoth/.pi/agent/auth.json", "/health"} {
for _, command := range []string{"pi --version", "PI_VERSION", "io.thothii.pi.version", "test -w /home/thoth/.pi", "test -r /home/thoth/.pi/agent/auth.json", "/health"} {
assertCalled(t, fake.calls, command)
}
}
func TestDoctorRejectsActualEnvironmentAndImageLabelVersionMismatches(t *testing.T) {
for _, mismatch := range []string{"actual", "environment", "label"} {
t.Run(mismatch, func(t *testing.T) {
fake := newFakeRunner()
switch mismatch {
case "actual":
fake.version = "0.80.2"
case "environment":
fake.expectedVersion = "0.80.2"
case "label":
fake.labelVersion = "0.80.2"
}
if err := Doctor(context.Background(), fake); err == nil || !strings.Contains(err.Error(), "version") {
t.Fatalf("Doctor() error = %v, want expected-version mismatch", err)
}
})
}
}
func TestConfigureRestoresAndVerifiesOldSettingsAfterEveryPostSnapshotFailure(t *testing.T) {
for _, failure := range []string{"helper", "readback", "digest"} {
t.Run(failure, func(t *testing.T) {
fake := &configureRunner{failure: failure, settings: Defaults{Provider: "old", Model: "old-model", Thinking: "low"}}
old := Defaults{Provider: "old", Model: "old-model", Thinking: "low"}
raw, _ := json.Marshal(old)
fake := &configureRunner{failure: failure, settings: old, settingsExist: true, settingsRaw: raw}
err := Configure(context.Background(), fake, Defaults{Provider: "new", Model: "new-model", Thinking: "high"})
if err == nil {
t.Fatal("Configure() error = nil, want injected failure")
@@ -32,12 +54,8 @@ func TestConfigureRestoresAndVerifiesOldSettingsAfterEveryPostSnapshotFailure(t
if fake.settings != (Defaults{Provider: "old", Model: "old-model", Thinking: "low"}) {
t.Fatalf("settings after failure = %#v, want old snapshot", fake.settings)
}
minimumReads := 3
if failure == "helper" {
minimumReads = 2
}
if fake.settingsReads < minimumReads {
t.Fatalf("settings read count = %d, want capture/failure reads plus verified restore", fake.settingsReads)
if !fake.settingsExist || string(fake.settingsRaw) != string(raw) {
t.Fatalf("settings raw snapshot after failure = exists:%t raw:%q, want %q", fake.settingsExist, fake.settingsRaw, raw)
}
})
}
@@ -46,11 +64,14 @@ func TestConfigureRestoresAndVerifiesOldSettingsAfterEveryPostSnapshotFailure(t
type configureRunner struct {
failure string
settings Defaults
settingsExist bool
settingsRaw []byte
settingsReads int
configReads int
writes int
}
func (f *configureRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
func (f *configureRunner) Run(_ context.Context, args []string, stdin io.Reader) (compose.Result, error) {
call := strings.Join(args, " ")
switch {
case strings.Contains(call, "config --format json"):
@@ -62,21 +83,50 @@ func (f *configureRunner) Run(_ context.Context, args []string, _ io.Reader) (co
return compose.Result{Stdout: `{"services":{"core":{"image":"thothii-core:local","environment":{"THT_LLM_URL":"` + endpoint + `"}}}}`}, nil
case strings.Contains(call, "/models"):
return compose.Result{Stdout: `{"models":[{"provider":"old","id":"old-model"},{"provider":"new","id":"new-model"}]}`}, nil
case strings.Contains(call, "settings-cli.js --snapshot"):
raw := f.settingsRaw
payload := map[string]any{"exists": f.settingsExist, "rawBase64": base64.StdEncoding.EncodeToString(raw)}
contents, _ := json.Marshal(payload)
return compose.Result{Stdout: string(contents)}, nil
case strings.Contains(call, "settings-cli.js --restore"):
var payload struct {
Exists bool `json:"exists"`
RawBase64 string `json:"rawBase64"`
}
contents, _ := io.ReadAll(stdin)
if json.Unmarshal(contents, &payload) != nil {
return compose.Result{ExitCode: 2}, errors.New("invalid restore payload")
}
f.settingsExist = payload.Exists
f.settingsRaw, _ = base64.StdEncoding.DecodeString(payload.RawBase64)
f.settings = Defaults{Provider: "old", Model: "old-model", Thinking: "low"}
if payload.Exists {
_ = json.Unmarshal(f.settingsRaw, &f.settings)
}
return compose.Result{}, nil
case strings.Contains(call, "settings-cli.js"):
if strings.Contains(call, "--provider new") {
f.settings = Defaults{Provider: "new", Model: "new-model", Thinking: "high"}
f.settingsExist = true
f.settingsRaw, _ = json.MarshalIndent(f.settings, "", " ")
f.writes++
if f.failure == "helper" {
return compose.Result{ExitCode: 17}, errors.New("injected helper failure")
}
} else {
f.settings = Defaults{Provider: "old", Model: "old-model", Thinking: "low"}
f.settingsExist = true
f.settingsRaw, _ = json.Marshal(f.settings)
}
return compose.Result{}, nil
case strings.Contains(call, "/settings"):
f.settingsReads++
if f.failure == "readback" && f.settingsReads == 2 {
if f.failure == "readback" && f.settings.Provider == "new" {
return compose.Result{Stdout: `{}`}, nil
}
if !f.settingsExist {
return compose.Result{Stdout: `{"provider":"old","model":"old-model","thinking":"low"}`}, nil
}
contents, _ := json.Marshal(f.settings)
return compose.Result{Stdout: string(contents)}, nil
default:
@@ -84,6 +134,45 @@ func (f *configureRunner) Run(_ context.Context, args []string, _ io.Reader) (co
}
}
func TestConfigureAllowsAFirstRunWithoutAnExistingSettingsFile(t *testing.T) {
fake := &configureRunner{}
if err := Configure(context.Background(), fake, Defaults{Provider: "new", Model: "new-model", Thinking: "high"}); err != nil {
t.Fatalf("Configure() clean install error = %v", err)
}
if !fake.settingsExist || fake.writes != 1 || fake.settings.Provider != "new" {
t.Fatalf("clean settings = exists:%t writes:%d value:%#v", fake.settingsExist, fake.writes, fake.settings)
}
}
func TestConfigureCompensationRestoresAbsentAndExactEmptyPriorFiles(t *testing.T) {
for _, prior := range []struct {
name string
exists bool
raw []byte
}{
{name: "absent"},
{name: "empty", exists: true, raw: []byte{}},
{name: "exact raw", exists: true, raw: []byte("{\n \"workspace\": \"kept\",\n \"provider\": \"old\",\n \"model\": \"old-model\",\n \"thinking\": \"low\"\n}\n")},
} {
t.Run(prior.name, func(t *testing.T) {
fake := &configureRunner{failure: "digest", settingsExist: prior.exists, settingsRaw: append([]byte{}, prior.raw...), settings: Defaults{Provider: "old", Model: "old-model", Thinking: "low"}}
err := Configure(context.Background(), fake, Defaults{Provider: "new", Model: "new-model", Thinking: "high"})
if err == nil {
t.Fatal("Configure() error = nil, want compensated digest failure")
}
if fake.writes != 1 {
t.Fatalf("settings writes = %d, want selected values written before compensation", fake.writes)
}
if fake.settingsExist != prior.exists || string(fake.settingsRaw) != string(prior.raw) {
t.Fatalf("restored exists/raw = %t/%q, want %t/%q", fake.settingsExist, fake.settingsRaw, prior.exists, prior.raw)
}
if fake.settingsReads < 3 {
t.Fatalf("settings reads = %d, want prior effective state, requested readback, and restored default verification", fake.settingsReads)
}
})
}
}
func TestConfigureValidatesBackendModelOptionsWritesRealCoreSettingsAndUsesUpstreamIdentity(t *testing.T) {
fake := newFakeRunner()
if err := Configure(context.Background(), fake, Defaults{Provider: "provider", Model: "model", Thinking: "medium"}); err != nil {
tools/thothctl/internal/pi/state.go
+11 -9
View File
@@ -15,7 +15,7 @@ import (
"github.com/gofrs/flock"
)
const stateFileVersion = 3
const stateFileVersion = 4
// Phase describes the durable point reached by a Pi update.
type Phase string
@@ -24,6 +24,7 @@ const (
PhasePreflight Phase = "preflight"
PhaseBuilding Phase = "building"
PhaseRecreated Phase = "recreated"
PhasePromoting Phase = "promoting"
PhaseVerified Phase = "verified"
PhaseRolledBack Phase = "rolled_back"
PhaseFailed Phase = "failed"
@@ -59,14 +60,15 @@ type Target struct {
// State is recovery metadata stored below the installation project. It never stores environment
// values, secret paths, credentials, or command output.
type State struct {
Version int `json:"version"`
Transaction string `json:"transaction"`
Phase Phase `json:"phase"`
UpdatedAt time.Time `json:"updated_at"`
Target Target `json:"target,omitempty"`
Previous Image `json:"previous"`
Candidate Image `json:"candidate,omitempty"`
Error string `json:"error,omitempty"`
Version int `json:"version"`
Transaction string `json:"transaction"`
Phase Phase `json:"phase"`
UpdatedAt time.Time `json:"updated_at"`
Target Target `json:"target,omitempty"`
Previous Image `json:"previous"`
Candidate Image `json:"candidate,omitempty"`
MutationStarted bool `json:"mutation_started,omitempty"`
Error string `json:"error,omitempty"`
}
func readState(path string) (State, error) {
tools/thothctl/internal/pi/update.go
+158 -20
View File
@@ -95,10 +95,14 @@ func updateWithHooks(ctx context.Context, runner Runner, request Request, hooks
}
request.Image = canonical
}
if old, err := readState(request.StatePath); err == nil && old.Phase != PhaseVerified && old.Phase != PhaseRolledBack && old.Phase != PhaseNoop {
if old, err := readState(request.StatePath); err == nil && stateNeedsRecovery(old) {
return Result{StatePath: request.StatePath}, ErrInterruptedUpdate
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
return Result{StatePath: request.StatePath}, err
} else if err == nil && !old.MutationStarted {
if cleanupErr := hooks.removeFile(lifecycleOverridePath(request.StatePath, old.Transaction)); cleanupErr != nil {
return Result{StatePath: request.StatePath}, errors.New("safe prior preparation state could not be cleaned up")
}
}
if err := setMaintenance(ctx, runner, true); err != nil {
return Result{StatePath: request.StatePath}, err
@@ -183,24 +187,30 @@ func updateWithHooks(ctx context.Context, runner Runner, request Request, hooks
lifecycle := composeOverrideRunner{Runner: runner, path: overridePath}
state.Phase = PhaseBuilding
clearMaintenance = false
if err := hooks.writeState(request.StatePath, state); err != nil {
result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
result, retErr, clearMaintenance = failPreparation(request.StatePath, overridePath, state, err, hooks)
return result, retErr
}
if err := prepareCandidate(ctx, lifecycle, request, candidateReference); err != nil {
result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
result, retErr, clearMaintenance = failPreparation(request.StatePath, overridePath, state, err, hooks)
return result, retErr
}
running, err = activeSessions(ctx, runner)
if err != nil {
result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
result, retErr, clearMaintenance = failPreparation(request.StatePath, overridePath, state, err, hooks)
return result, retErr
}
if running {
result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, ErrActiveSessions, hooks)
result, retErr, clearMaintenance = failPreparation(request.StatePath, overridePath, state, ErrActiveSessions, hooks)
return result, retErr
}
state.MutationStarted = true
if err := hooks.writeState(request.StatePath, state); err != nil {
state.MutationStarted = false
result, retErr, clearMaintenance = failPreparation(request.StatePath, overridePath, state, err, hooks)
return result, retErr
}
clearMaintenance = false
if err := recreateCore(ctx, lifecycle); err != nil {
result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
return result, retErr
@@ -223,18 +233,48 @@ func updateWithHooks(ctx context.Context, runner Runner, request Request, hooks
result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
return result, retErr
}
state.Phase, state.Error = PhaseVerified, ""
state.Phase, state.Error = PhasePromoting, ""
if err := hooks.writeState(request.StatePath, state); err != nil {
result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
return result, retErr
}
if err := hooks.removeFile(overridePath); err != nil {
return Result{Phase: PhaseFailed, StatePath: request.StatePath}, errors.New("verified update override cleanup failed: maintenance recovery required")
if err := promoteLifecycleOverride(overridePath, currentImageOverridePath(request.StatePath), candidateReference); err != nil {
result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
return result, retErr
}
state.Phase = PhaseVerified
if err := hooks.writeState(request.StatePath, state); err != nil {
result, retErr, clearMaintenance = compensate(ctx, runner, request.StatePath, overridePath, state, err, hooks)
return result, retErr
}
clearMaintenance = true
return Result{Phase: PhaseVerified, StatePath: request.StatePath}, nil
}
func stateNeedsRecovery(state State) bool {
switch state.Phase {
case PhaseVerified, PhaseRolledBack, PhaseNoop:
return false
case PhaseFailed:
return state.MutationStarted
default:
return state.MutationStarted
}
}
func failPreparation(statePath, overridePath string, state State, cause error, hooks lifecycleHooks) (Result, error, bool) {
state.Phase = PhaseFailed
state.MutationStarted = false
state.Error = "candidate preparation failed before core mutation"
writeErr := hooks.writeState(statePath, state)
removeErr := hooks.removeFile(overridePath)
message := "candidate preparation failed before core mutation"
if writeErr != nil || removeErr != nil {
message += "; safe preparation cleanup was incomplete"
}
return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("%s: %w", message, cause), true
}
// Rollback restores the image recorded in durable update state. It is safe for interrupted runs.
func Rollback(ctx context.Context, runner Runner, statePath string, confirm bool) (result Result, retErr error) {
return rollbackWithHooks(ctx, runner, statePath, confirm, defaultLifecycleHooks)
@@ -290,13 +330,13 @@ func rollbackWithHooks(ctx context.Context, runner Runner, statePath string, con
}
return Result{Phase: PhaseFailed, StatePath: statePath}, err
}
if err := promoteLifecycleOverride(overridePath, currentImageOverridePath(statePath), state.Previous.Reference); err != nil {
return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("rollback restored the core but durable current-image promotion failed: recovery required")
}
state.Phase, state.Error = PhaseRolledBack, ""
if err := hooks.writeState(statePath, state); err != nil {
return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("rollback restored the core but recovery state could not be persisted")
}
if err := hooks.removeFile(overridePath); err != nil {
return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("rollback override cleanup failed: maintenance recovery required")
}
clearMaintenance = true
return Result{Phase: PhaseRolledBack, StatePath: statePath}, nil
}
@@ -323,14 +363,14 @@ func compensate(ctx context.Context, runner Runner, statePath, overridePath stri
}
return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("update failed; automatic rollback also failed: recovery required"), false
}
if err := promoteLifecycleOverride(overridePath, currentImageOverridePath(statePath), state.Previous.Reference); err != nil {
return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("previous core image was restored but durable selector promotion failed: recovery required"), false
}
state.Phase, state.Error = PhaseRolledBack, ""
if writeErr := hooks.writeState(statePath, state); writeErr != nil {
return Result{Phase: PhaseFailed, StatePath: statePath}, fmt.Errorf("previous core image was restored but recovery state write failed: recovery required"), false
}
if err := hooks.removeFile(overridePath); err != nil {
return Result{Phase: PhaseFailed, StatePath: statePath}, errors.New("previous core image was restored but override cleanup failed: recovery required"), false
}
return Result{Phase: PhaseRolledBack, StatePath: statePath}, fmt.Errorf("update failed; previous core image was restored"), true
return Result{Phase: PhaseRolledBack, StatePath: statePath}, fmt.Errorf("update failed; previous core image was restored: %w", cause), true
}
func sourceValue(request Request) string {
@@ -600,6 +640,10 @@ func lifecycleOverridePath(statePath, transaction string) string {
return filepath.Join(filepath.Dir(statePath), "pi-lifecycle-"+transaction+".yaml")
}
func currentImageOverridePath(statePath string) string {
return filepath.Join(filepath.Dir(statePath), "current-image.yaml")
}
func writeLifecycleOverride(path, image string) error {
quoted, err := json.Marshal(image)
if err != nil {
@@ -612,6 +656,41 @@ func writeLifecycleOverride(path, image string) error {
return nil
}
func promoteLifecycleOverride(source, destination, expectedImage string) error {
if err := durableReplace(source, destination, filepath.Dir(destination)); err != nil {
selected, readErr := readLifecycleOverride(destination)
if readErr == nil && selected == expectedImage {
return nil
}
return errors.New("lifecycle image override could not be promoted durably")
}
selected, err := readLifecycleOverride(destination)
if err != nil || selected != expectedImage {
return errors.New("promoted lifecycle image override could not be verified")
}
return nil
}
func readLifecycleOverride(path string) (string, error) {
contents, err := os.ReadFile(path)
if err != nil {
return "", err
}
for _, line := range strings.Split(string(contents), "\n") {
line = strings.TrimSpace(line)
if !strings.HasPrefix(line, "image:") {
continue
}
encoded := strings.TrimSpace(strings.TrimPrefix(line, "image:"))
var image string
if json.Unmarshal([]byte(encoded), &image) != nil || image == "" || strings.ContainsAny(image, "\r\n") {
return "", errors.New("lifecycle image override is invalid")
}
return image, nil
}
return "", errors.New("lifecycle image override has no core image")
}
// RecoverMaintenance clears a stale durable gate only after the running core and terminal
// recovery metadata prove that no rollback is still required.
func RecoverMaintenance(ctx context.Context, runner Runner, statePath string, confirm bool) error {
@@ -625,11 +704,26 @@ func RecoverMaintenance(ctx context.Context, runner Runner, statePath string, co
defer lock.Release()
state, stateErr := readState(statePath)
if stateErr == nil {
if state.Phase != PhaseVerified && state.Phase != PhaseRolledBack && state.Phase != PhaseNoop {
transactionOverride := lifecycleOverridePath(statePath, state.Transaction)
switch {
case state.Phase == PhasePromoting:
if err := recoverPromotion(ctx, runner, statePath, transactionOverride, &state); err != nil {
return err
}
case !state.MutationStarted && state.Phase != PhaseVerified && state.Phase != PhaseRolledBack && state.Phase != PhaseNoop:
state.Phase, state.Error = PhaseFailed, "candidate preparation interrupted before core mutation"
if err := writeState(statePath, state); err != nil {
return errors.New("maintenance recovery could not finalize safe preparation state")
}
if err := durableRemove(transactionOverride); err != nil {
return errors.New("maintenance recovery could not remove the safe preparation override")
}
case stateNeedsRecovery(state):
return ErrInterruptedUpdate
}
if err := durableRemove(lifecycleOverridePath(statePath, state.Transaction)); err != nil {
return errors.New("maintenance recovery could not remove the lifecycle override")
default:
if err := durableRemove(transactionOverride); err != nil {
return errors.New("maintenance recovery could not remove the lifecycle override")
}
}
} else if !errors.Is(stateErr, os.ErrNotExist) {
return stateErr
@@ -647,6 +741,50 @@ func RecoverMaintenance(ctx context.Context, runner Runner, statePath string, co
return setMaintenance(ctx, runner, false)
}
func recoverPromotion(ctx context.Context, runner Runner, statePath, transactionOverride string, state *State) error {
currentOverride := currentImageOverridePath(statePath)
selected, currentErr := readLifecycleOverride(currentOverride)
if currentErr != nil || selected != state.Candidate.Reference {
pending, pendingErr := readLifecycleOverride(transactionOverride)
if pendingErr != nil || pending != state.Candidate.Reference {
if currentErr == nil && selected == state.Previous.Reference {
if err := verifyRestoredCurrent(ctx, runner, state.Previous); err != nil {
return ErrInterruptedUpdate
}
state.Phase, state.Error = PhaseRolledBack, ""
return writeState(statePath, *state)
}
return ErrInterruptedUpdate
}
if err := promoteLifecycleOverride(transactionOverride, currentOverride, state.Candidate.Reference); err != nil {
return err
}
}
if err := verifyCandidate(ctx, runner, state.Target.Version, state.Previous); err != nil {
return err
}
state.Phase, state.Error = PhaseVerified, ""
return writeState(statePath, *state)
}
func verifyRestoredCurrent(ctx context.Context, runner Runner, previous Image) error {
configured, err := renderedCore(ctx, runner)
if err != nil {
return err
}
after, err := runningImage(ctx, runner, configured.Reference)
if err != nil {
return err
}
if after.ID != previous.ID || configured.ConfigurationSHA != previous.ConfigurationSHA || !sameMounts(previous.Mounts, after.Mounts) {
return errors.New("running core does not match the durable previous-image selector")
}
if err := Doctor(ctx, runner); err != nil {
return err
}
return Test(ctx, runner)
}
func sameStrings(left, right []string) bool {
left, right = append([]string(nil), left...), append([]string(nil), right...)
sort.Strings(left)
tools/thothctl/internal/pi/update_test.go
+200 -21
View File
@@ -54,6 +54,9 @@ func TestUpdateBuildsPinnedVersionRecreatesOnlyCoreAndPersistsRecoveryState(t *t
if got := string(readStateBytes(t, result.StatePath)); strings.Contains(got, "llm.example.invalid") {
t.Fatalf("state = %q, want an endpoint-free configuration digest", got)
}
if selected := readSelectorReference(t, currentImageOverridePath(result.StatePath)); selected != fake.buildReference {
t.Fatalf("durable selector = %q, want verified candidate %q", selected, fake.buildReference)
}
}
func TestUpdateUsesATransactionScopedComposeOverrideWithoutMutatingTheConfiguredImage(t *testing.T) {
@@ -69,16 +72,46 @@ func TestUpdateUsesATransactionScopedComposeOverrideWithoutMutatingTheConfigured
if matches, err := filepath.Glob(filepath.Join(filepath.Dir(statePath), "pi-lifecycle-*.yaml")); err != nil || len(matches) != 0 {
t.Fatalf("terminal lifecycle overrides = %v, error = %v; want none", matches, err)
}
if _, err := os.Stat(currentImageOverridePath(statePath)); err != nil {
t.Fatalf("durable current-image override missing: %v", err)
}
}
func TestSuccessfulUpdateAndRollbackRemainSelectedOnFreshRecreate(t *testing.T) {
fake := newFakeRunner()
statePath := filepath.Join(t.TempDir(), ".thothctl", "update-state.json")
if _, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true}); err != nil {
t.Fatal(err)
}
fake.currentImage = "sha256:old"
if err := recreateCore(context.Background(), composeOverrideRunner{Runner: fake, path: currentImageOverridePath(statePath)}); err != nil {
t.Fatal(err)
}
if fake.currentImage != "sha256:candidate" {
t.Fatalf("fresh recreate image = %q, want verified candidate", fake.currentImage)
}
if _, err := Rollback(context.Background(), fake, statePath, true); err != nil {
t.Fatal(err)
}
fake.currentImage = "sha256:candidate"
if err := recreateCore(context.Background(), composeOverrideRunner{Runner: fake, path: currentImageOverridePath(statePath)}); err != nil {
t.Fatal(err)
}
if fake.currentImage != "sha256:old" {
t.Fatalf("fresh recreate after rollback image = %q, want previous image", fake.currentImage)
}
}
func TestTwoInstallationsSharingAConfiguredTagUseDifferentLifecycleTags(t *testing.T) {
first, second := newFakeRunner(), newFakeRunner()
firstPath := filepath.Join(t.TempDir(), "one", "state.json")
secondPath := filepath.Join(t.TempDir(), "two", "state.json")
for _, item := range []struct {
fake *fakeRunner
path string
}{
{first, filepath.Join(t.TempDir(), "one", "state.json")},
{second, filepath.Join(t.TempDir(), "two", "state.json")},
{first, firstPath},
{second, secondPath},
} {
if _, err := Update(context.Background(), item.fake, Request{StatePath: item.path, Version: "0.81.0", Source: BuildSource, Confirm: true}); err != nil {
t.Fatal(err)
@@ -87,15 +120,28 @@ func TestTwoInstallationsSharingAConfiguredTagUseDifferentLifecycleTags(t *testi
if first.buildReference == second.buildReference {
t.Fatalf("installations reused lifecycle tag %q", first.buildReference)
}
firstSelector := readSelectorReference(t, currentImageOverridePath(firstPath))
secondSelector := readSelectorReference(t, currentImageOverridePath(secondPath))
if firstSelector == secondSelector || firstSelector != first.buildReference || secondSelector != second.buildReference {
t.Fatalf("installation selectors = %q / %q, want isolated lifecycle references", firstSelector, secondSelector)
}
}
func TestDigestPinnedConfiguredImageIsNeverUsedAsARollbackTagTarget(t *testing.T) {
fake := newFakeRunner()
fake.configuredImage = "registry.example.invalid/core@sha256:" + strings.Repeat("b", 64)
fake.tags = map[string]string{fake.configuredImage: "sha256:old"}
fake.fail = "health"
_, _ = Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource, Confirm: true})
statePath := filepath.Join(t.TempDir(), ".thothctl", "state.json")
if _, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true}); err != nil {
t.Fatal(err)
}
if _, err := Rollback(context.Background(), fake, statePath, true); err != nil {
t.Fatal(err)
}
assertNotCalled(t, fake.calls, "image tag sha256:old "+fake.configuredImage)
if selected := readSelectorReference(t, currentImageOverridePath(statePath)); !strings.Contains(selected, "-previous") {
t.Fatalf("rollback selector = %q, want transaction previous tag for digest-pinned base", selected)
}
}
func TestMaintenanceLostResponsesAreResolvedByStatusAndEveryRecreateStartsGated(t *testing.T) {
@@ -189,7 +235,7 @@ func TestUpdateRollsBackAfterPostRecreateFailures(t *testing.T) {
}
func TestEveryRecoveryStateWriteFailureIsHandledTransactionally(t *testing.T) {
for failAt := 1; failAt <= 4; failAt++ {
for failAt := 1; failAt <= 6; failAt++ {
t.Run(fmt.Sprintf("write-%d", failAt), func(t *testing.T) {
fake := newFakeRunner()
writes := 0
@@ -210,8 +256,14 @@ func TestEveryRecoveryStateWriteFailureIsHandledTransactionally(t *testing.T) {
if fake.currentImage != "sha256:old" {
t.Fatalf("current image = %q, want restored previous", fake.currentImage)
}
if failAt > 1 && result.Phase != PhaseRolledBack {
t.Fatalf("phase = %q, want rolled_back", result.Phase)
wantPhase := Phase("")
if failAt == 2 || failAt == 3 {
wantPhase = PhaseFailed
} else if failAt >= 4 {
wantPhase = PhaseRolledBack
}
if result.Phase != wantPhase {
t.Fatalf("phase = %q, want %q for write %d", result.Phase, wantPhase, failAt)
}
if fake.maintenance {
t.Fatal("maintenance remained active after proven stable recovery")
@@ -227,7 +279,7 @@ func TestCompensationWriteFailureKeepsMaintenanceActiveForExplicitRecovery(t *te
hooks := defaultLifecycleHooks
hooks.writeState = func(path string, state State) error {
writes++
if writes == 4 {
if writes == 5 {
return errors.New("injected compensation state write failure")
}
return writeState(path, state)
@@ -264,12 +316,15 @@ func TestRecoverMaintenanceClearsOnlyAfterTerminalStateAndVerifiedSmoke(t *testi
fake.maintenance = true
statePath := filepath.Join(t.TempDir(), "state.json")
previous := stateImageForTest(t, fake)
state := State{Transaction: "recover-test", Phase: PhaseVerified, Previous: previous}
state := State{Transaction: "recover-test", Phase: PhaseRolledBack, MutationStarted: true, Previous: previous}
writeStateForTest(t, statePath, state)
overridePath := lifecycleOverridePath(statePath, state.Transaction)
if err := writeLifecycleOverride(overridePath, previous.Reference); err != nil {
t.Fatal(err)
}
if err := writeLifecycleOverride(currentImageOverridePath(statePath), previous.Reference); err != nil {
t.Fatal(err)
}
if err := RecoverMaintenance(context.Background(), fake, statePath, true); err != nil {
t.Fatalf("RecoverMaintenance() error = %v", err)
@@ -280,6 +335,9 @@ func TestRecoverMaintenanceClearsOnlyAfterTerminalStateAndVerifiedSmoke(t *testi
if _, err := os.Stat(overridePath); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("lifecycle override still exists: %v", err)
}
if selected := readSelectorReference(t, currentImageOverridePath(statePath)); selected != previous.Reference {
t.Fatalf("maintenance cleanup changed durable selector to %q", selected)
}
assertCalled(t, fake.calls, "/models")
assertCalled(t, fake.calls, "/settings")
}
@@ -288,7 +346,7 @@ func TestRecoverMaintenanceRefusesPendingTransaction(t *testing.T) {
fake := newFakeRunner()
fake.maintenance = true
statePath := filepath.Join(t.TempDir(), "state.json")
writeStateForTest(t, statePath, State{Phase: PhaseRecreated, Previous: stateImageForTest(t, fake)})
writeStateForTest(t, statePath, State{Phase: PhaseRecreated, MutationStarted: true, Previous: stateImageForTest(t, fake)})
err := RecoverMaintenance(context.Background(), fake, statePath, true)
if !errors.Is(err, ErrInterruptedUpdate) {
@@ -299,6 +357,43 @@ func TestRecoverMaintenanceRefusesPendingTransaction(t *testing.T) {
}
}
func TestRecoverMaintenanceCompletesAnInterruptedDurablePromotion(t *testing.T) {
fake := newFakeRunner()
fake.maintenance = true
fake.currentImage = "sha256:candidate"
fake.version = "0.81.0"
fake.expectedVersion = "0.81.0"
fake.labelVersion = "0.81.0"
statePath := filepath.Join(t.TempDir(), ".thothctl", "update-state.json")
previous := stateImageForTest(t, newFakeRunner())
candidate := previous
candidate.ID = "sha256:candidate"
candidate.Reference = "thothii-core:thothctl-recover-candidate"
fake.tags[candidate.Reference] = candidate.ID
state := State{
Transaction: "promotion-recovery",
Phase: PhasePromoting,
MutationStarted: true,
Target: Target{Version: "0.81.0", Source: string(BuildSource)},
Previous: previous,
Candidate: candidate,
}
writeStateForTest(t, statePath, state)
if err := writeLifecycleOverride(lifecycleOverridePath(statePath, state.Transaction), candidate.Reference); err != nil {
t.Fatal(err)
}
if err := RecoverMaintenance(context.Background(), fake, statePath, true); err != nil {
t.Fatalf("RecoverMaintenance() promotion error = %v", err)
}
if selected := readSelectorReference(t, currentImageOverridePath(statePath)); selected != candidate.Reference {
t.Fatalf("recovered selector = %q, want %q", selected, candidate.Reference)
}
if recovered, err := readState(statePath); err != nil || recovered.Phase != PhaseVerified {
t.Fatalf("recovered state = %+v, %v; want verified", recovered, err)
}
}
func TestRollbackFinalStateWriteFailureKeepsMaintenanceAndOverrideForRecovery(t *testing.T) {
fake := newFakeRunner()
statePath := filepath.Join(t.TempDir(), "state.json")
@@ -320,13 +415,13 @@ func TestRollbackFinalStateWriteFailureKeepsMaintenanceAndOverrideForRecovery(t
if !fake.maintenance {
t.Fatal("maintenance was cleared without durable rollback finalization")
}
if _, err := os.Stat(lifecycleOverridePath(statePath, "rollback-test")); err != nil {
t.Fatalf("recovery override was not preserved: %v", err)
if selected := readSelectorReference(t, currentImageOverridePath(statePath)); selected != previous.Reference {
t.Fatalf("durable rollback selector = %q, want %q", selected, previous.Reference)
}
}
func TestUpdateDoesNotRecreateWhenPreflightOrBuildFails(t *testing.T) {
for _, failure := range []string{"preflight", "build"} {
func TestUpdateDoesNotRecreateWhenPreflightFails(t *testing.T) {
for _, failure := range []string{"preflight"} {
t.Run(failure, func(t *testing.T) {
fake := newFakeRunner()
fake.fail = failure
@@ -337,16 +432,67 @@ func TestUpdateDoesNotRecreateWhenPreflightOrBuildFails(t *testing.T) {
if failure == "preflight" && result.Phase == PhaseRolledBack {
t.Fatalf("preflight failure unexpectedly rolled back: %+v", result)
}
if failure == "build" && result.Phase != PhaseRolledBack {
t.Fatalf("candidate build failure must compensate: %+v", result)
assertNotCalled(t, fake.calls, "force-recreate")
})
}
}
func TestCandidateBuildAndPullFailuresRemainPreMutationAndNeverRecreateCore(t *testing.T) {
for _, testCase := range []struct {
name string
source Source
image string
failure string
}{
{name: "build", source: BuildSource, failure: "build"},
{name: "pull", source: PullSource, image: "registry.example.invalid/core@sha256:" + strings.Repeat("a", 64), failure: "pull"},
{name: "candidate tag", source: PullSource, image: "registry.example.invalid/core@sha256:" + strings.Repeat("b", 64), failure: "tag"},
} {
t.Run(testCase.name, func(t *testing.T) {
fake := newFakeRunner()
fake.fail = testCase.failure
statePath := filepath.Join(t.TempDir(), ".thothctl", "update-state.json")
result, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: testCase.source, Image: testCase.image, Confirm: true})
if err == nil {
t.Fatal("Update() error = nil, want preparation failure")
}
if failure == "preflight" {
assertNotCalled(t, fake.calls, "force-recreate")
if result.Phase != PhaseFailed {
t.Fatalf("phase = %q, want safe failed preparation", result.Phase)
}
state, stateErr := readState(statePath)
if stateErr != nil {
t.Fatal(stateErr)
}
if state.MutationStarted {
t.Fatal("preparation failure recorded mutationStarted")
}
assertNotCalled(t, fake.calls, "force-recreate")
if fake.maintenance {
t.Fatal("maintenance remained active after safe preparation failure")
}
})
}
}
func TestSuccessfulCompensationPreservesTheOriginalTypedCause(t *testing.T) {
for _, cause := range []error{ErrActiveSessions, ErrInterruptedUpdate} {
fake := newFakeRunner()
fake.maintenance = true
statePath := filepath.Join(t.TempDir(), ".thothctl", "update-state.json")
state := State{Transaction: "typed-cause", Phase: PhaseRecreated, MutationStarted: true, Previous: stateImageForTest(t, fake)}
result, err, clear := compensate(context.Background(), fake, statePath, lifecycleOverridePath(statePath, state.Transaction), state, cause, defaultLifecycleHooks)
if result.Phase != PhaseRolledBack || !clear {
t.Fatalf("compensation = %+v, clear=%t; want successful rollback", result, clear)
}
if !errors.Is(err, cause) {
t.Fatalf("compensation error = %v, want errors.Is(..., %v)", err, cause)
}
if !strings.Contains(err.Error(), "previous core image was restored") {
t.Fatalf("compensation error = %v, want rollback-success report", err)
}
}
}
func TestUpdateRequiresConfirmationAndDrainsActiveSessions(t *testing.T) {
fake := newFakeRunner()
_, err := Update(context.Background(), fake, Request{StatePath: filepath.Join(t.TempDir(), "state.json"), Version: "0.81.0", Source: BuildSource})
@@ -402,7 +548,7 @@ func TestRollbackRestoresInterruptedOrPreviouslyRecordedState(t *testing.T) {
func TestUpdateRefusesToOverwriteInterruptedRecoveryState(t *testing.T) {
fake := newFakeRunner()
statePath := filepath.Join(t.TempDir(), "state.json")
writeStateForTest(t, statePath, State{Phase: PhaseRecreated, Previous: Image{ID: "sha256:old", Reference: "thothii-core:local", MountFingerprint: mountFingerprint(nil)}})
writeStateForTest(t, statePath, State{Phase: PhaseRecreated, MutationStarted: true, Previous: Image{ID: "sha256:old", Reference: "thothii-core:local", MountFingerprint: mountFingerprint(nil)}})
_, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true})
if !errors.Is(err, ErrInterruptedUpdate) {
t.Fatalf("Update() error = %v, want interrupted update error", err)
@@ -441,6 +587,8 @@ type fakeRunner struct {
calls []string
fail string
version string
expectedVersion string
labelVersion string
activeSessions bool
built bool
currentImage string
@@ -460,7 +608,7 @@ type fakeRunner struct {
func newFakeRunner() *fakeRunner {
return &fakeRunner{
version: "0.80.3", currentImage: "sha256:old", configuredImage: "thothii-core:local",
version: "0.80.3", expectedVersion: "0.80.3", labelVersion: "0.80.3", currentImage: "sha256:old", configuredImage: "thothii-core:local",
tags: map[string]string{"thothii-core:local": "sha256:old"},
imageVersions: map[string]string{"sha256:old": "0.80.3"},
}
@@ -478,6 +626,12 @@ func (f *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose
if f.fail == "build" && containsArg(args, "build") {
return compose.Result{ExitCode: 1}, errors.New("build token=secret")
}
if f.fail == "pull" && len(args) > 0 && args[0] == "pull" {
return compose.Result{ExitCode: 1}, errors.New("pull token=secret")
}
if f.fail == "tag" && len(args) >= 4 && args[0] == "image" && args[1] == "tag" && strings.Contains(args[3], "-candidate") {
return compose.Result{ExitCode: 1}, errors.New("tag token=secret")
}
if f.fail == "health" && f.built && strings.Contains(call, "curl -fsS http://127.0.0.1:8787/health") {
return compose.Result{ExitCode: 1}, errors.New("health token=secret")
}
@@ -501,6 +655,8 @@ func (f *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose
return compose.Result{Stdout: "core-container\n"}, nil
case strings.Contains(call, "inspect --format {{.Image}}"):
return compose.Result{Stdout: f.currentImage + "\n"}, nil
case strings.Contains(call, "io.thothii.pi.version"):
return compose.Result{Stdout: f.labelVersion + "\n"}, nil
case strings.Contains(call, "inspect --format {{json .Mounts}}"):
if f.fail == "mount-drift" && f.currentImage == "sha256:candidate" {
return compose.Result{Stdout: `[{"Type":"volume","Name":"wrong-settings","Source":"wrong-settings","Destination":"/data/settings","RW":true}]`}, nil
@@ -580,6 +736,8 @@ func (f *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose
return compose.Result{}, nil
case strings.Contains(call, "pi --version"):
return compose.Result{Stdout: f.version + "\n"}, nil
case strings.Contains(call, "PI_VERSION"):
return compose.Result{Stdout: f.expectedVersion + "\n"}, nil
case strings.Contains(call, "/models"):
if f.modelsWire != "" {
return compose.Result{Stdout: f.modelsWire}, nil
@@ -604,7 +762,7 @@ func containsArg(args []string, wanted string) bool {
func selectedCoreReference(args []string, fallback string) string {
for index := 0; index+1 < len(args); index++ {
if args[index] != "-f" || !strings.Contains(filepath.Base(args[index+1]), "pi-lifecycle-") {
if args[index] != "-f" || (!strings.Contains(filepath.Base(args[index+1]), "pi-lifecycle-") && filepath.Base(args[index+1]) != "current-image.yaml") {
continue
}
contents, err := os.ReadFile(args[index+1])
@@ -626,6 +784,27 @@ func selectedCoreReference(args []string, fallback string) string {
return fallback
}
func readSelectorReference(t *testing.T, path string) string {
t.Helper()
contents, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
for _, line := range strings.Split(string(contents), "\n") {
line = strings.TrimSpace(line)
if !strings.HasPrefix(line, "image:") {
continue
}
value := strings.TrimSpace(strings.TrimPrefix(line, "image:"))
if decoded, err := strconv.Unquote(value); err == nil {
return decoded
}
return value
}
t.Fatalf("selector %s has no image", path)
return ""
}
func callIndex(calls []string, contains string) int {
for index, call := range calls {
if strings.Contains(call, contains) {