mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix: acknowledge pi maintenance barrier

Browse Source
This commit is contained in:
marcopan
2026-08-04 19:32:05 +02:00
parent 0b9ad7f53f
commit 5b3ce93e31
11 changed files with 168 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/thothctl/internal/pi/commands.go
+15 -3
View File
@@ -30,7 +30,7 @@ var internalIdentityHeaders = []string{
// Configure changes the backend's real installation settings through a core-side helper. It
// deliberately has no secret or endpoint input: external endpoints remain Compose-owned.
func Configure(ctx context.Context, runner Runner, value Defaults) error {
func Configure(ctx context.Context, runner Runner, value Defaults) (retErr error) {
if !choicePattern.MatchString(value.Provider) || !choicePattern.MatchString(value.Model) {
return errors.New("provider and model must be supported identifiers")
}
@@ -63,10 +63,22 @@ func Configure(ctx context.Context, runner Runner, value Defaults) error {
if !found {
return errors.New("provider/model is not in Pi options")
}
result, err := runCompose(ctx, runner, "exec", "-T", "core", "node", "/app/backend/dist/settings/settings-cli.js", "--provider", value.Provider, "--model", value.Model, "--thinking", value.Thinking)
if err != nil { return commandError("Pi installation settings write", result, err) }
settingsArgs := append([]string{"exec", "-T", "core", "curl", "-fsS"}, internalIdentityHeaders...)
settingsArgs = append(settingsArgs, "http://127.0.0.1:8787/settings")
oldResult, err := runCompose(ctx, runner, settingsArgs...)
if err != nil { return commandError("Pi installation settings capture", oldResult, err) }
var old Defaults
if json.Unmarshal([]byte(oldResult.Stdout), &old) != nil || old.Provider == "" || old.Model == "" || old.Thinking == "" { return errors.New("Pi installation settings capture is invalid") }
wrote := false
defer func() {
if retErr != nil && wrote {
result, restoreErr := runCompose(context.Background(), runner, "exec", "-T", "core", "node", "/app/backend/dist/settings/settings-cli.js", "--provider", old.Provider, "--model", old.Model, "--thinking", old.Thinking)
if restoreErr != nil || result.ExitCode != 0 { retErr = fmt.Errorf("%w; previous Pi settings could not be restored: recovery required", retErr) }
}
}()
result, err := runCompose(ctx, runner, "exec", "-T", "core", "node", "/app/backend/dist/settings/settings-cli.js", "--provider", value.Provider, "--model", value.Model, "--thinking", value.Thinking)
if err != nil { return commandError("Pi installation settings write", result, err) }
wrote = true
settings, err := runCompose(ctx, runner, settingsArgs...)
if err != nil { return commandError("Pi installation settings read-back", settings, err) }
var saved Defaults
tools/thothctl/internal/pi/state.go
+23 -11
View File
@@ -79,6 +79,9 @@ func readState(path string) (State, error) {
if state.Version != stateFileVersion || state.Previous.ID == "" || state.Previous.Reference == "" || state.Previous.MountFingerprint == "" {
return State{}, errors.New("update recovery state is incomplete")
}
if mountFingerprint(state.Previous.Mounts) != state.Previous.MountFingerprint || (state.Candidate.ID != "" && mountFingerprint(state.Candidate.Mounts) != state.Candidate.MountFingerprint) {
return State{}, errors.New("update recovery state mount fingerprint is invalid")
}
return state, nil
}
@@ -144,9 +147,10 @@ func acquireLock(statePath string) (*updateLock, error) {
return nil, errors.New("could not create Pi update recovery directory")
}
path := statePath + ".lock"
if err := os.Mkdir(path, 0o700); err != nil {
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
if err != nil {
if errors.Is(err, os.ErrExist) {
if reclaimDeadLocalLock(path) {
if reclaimDeadLocalLock(path, statePath) {
return acquireLock(statePath)
}
return nil, ErrLockHeld
@@ -154,28 +158,36 @@ func acquireLock(statePath string) (*updateLock, error) {
return nil, errors.New("could not acquire Pi update lock")
}
host, err := os.Hostname()
if err != nil { _ = os.Remove(path); return nil, errors.New("could not identify Pi update lock owner") }
if err != nil { _ = file.Close(); _ = os.Remove(path); return nil, errors.New("could not identify Pi update lock owner") }
owner := lockOwner{PID: os.Getpid(), Host: host, StartedAt: time.Now().UTC(), Transaction: fmt.Sprintf("%d-%d", os.Getpid(), time.Now().UnixNano())}
contents, err := json.Marshal(owner)
if err != nil { _ = os.Remove(path); return nil, errors.New("could not record Pi update lock owner") }
if err := writeFileDurably(filepath.Join(path, "owner.json"), ".owner-", append(contents, '\n')); err != nil {
_ = os.Remove(path)
if err != nil { _ = file.Close(); _ = os.Remove(path); return nil, errors.New("could not record Pi update lock owner") }
if _, err := file.Write(append(contents, '\n')); err != nil || file.Sync() != nil || file.Close() != nil {
_ = file.Close(); _ = os.Remove(path)
return nil, errors.New("could not record Pi update lock owner")
}
return &updateLock{path: path}, nil
}
func (l *updateLock) Release() { _ = os.Remove(filepath.Join(l.path, "owner.json")); _ = os.Remove(l.path) }
func (l *updateLock) Release() { _ = os.Remove(l.path) }
// reclaimDeadLocalLock is deliberately conservative: a malformed, remote, or merely old lock
// is recovery-required. Only a process we can prove is gone on this machine is reclaimed.
func reclaimDeadLocalLock(path string) bool {
contents, err := os.ReadFile(filepath.Join(path, "owner.json"))
if err != nil { return false }
func reclaimDeadLocalLock(path, statePath string) bool {
info, err := os.Stat(path)
if err != nil || time.Since(info.ModTime()) < 5*time.Minute || !hasPendingRecoveryState(statePath) { return false }
contents, err := os.ReadFile(path)
if err != nil { return os.Remove(path) == nil }
var owner lockOwner
if json.Unmarshal(contents, &owner) != nil || owner.PID <= 0 || owner.Host == "" { return false }
host, err := os.Hostname()
if err != nil || owner.Host != host { return false }
if processAlive(owner.PID) { return false }
if err := os.Remove(filepath.Join(path, "owner.json")); err != nil { return false }
return os.Remove(path) == nil
}
func hasPendingRecoveryState(path string) bool {
contents, err := os.ReadFile(path); if err != nil { return false }
var state State
if json.Unmarshal(contents, &state) != nil { return false }
return state.Phase != PhaseVerified && state.Phase != PhaseRolledBack && state.Phase != PhaseNoop
}
tools/thothctl/internal/pi/update.go
+26 -12
View File
@@ -166,7 +166,7 @@ func Update(ctx context.Context, runner Runner, request Request) (result Result,
}
// Rollback restores the image recorded in durable update state. It is safe for interrupted runs.
func Rollback(ctx context.Context, runner Runner, statePath string, confirm bool) (Result, error) {
func Rollback(ctx context.Context, runner Runner, statePath string, confirm bool) (result Result, retErr error) {
lock, err := acquireLock(statePath)
if err != nil {
return Result{StatePath: statePath}, err
@@ -175,6 +175,16 @@ func Rollback(ctx context.Context, runner Runner, statePath string, confirm bool
if !confirm {
return Result{StatePath: statePath}, ErrConfirmationRequired
}
if err := setMaintenance(ctx, runner, true); err != nil { return Result{StatePath: statePath}, err }
defer func() {
if clearErr := setMaintenance(context.Background(), runner, false); clearErr != nil {
result = Result{Phase: PhaseFailed, StatePath: statePath}
if retErr == nil { retErr = errors.New("maintenance admission gate could not be cleared: recovery required")
} else { retErr = fmt.Errorf("%w; maintenance admission gate could not be cleared: recovery required", retErr) }
}
}()
if active, err := activeSessions(ctx, runner); err != nil { return Result{StatePath: statePath}, err
} else if active { return Result{StatePath: statePath}, ErrActiveSessions }
state, err := readState(statePath)
if err != nil {
return Result{StatePath: statePath}, err
@@ -232,30 +242,34 @@ func canonicalDigestReference(value string) (string, error) {
return reference.FamiliarString(canonical), nil
}
// The command text is fixed; no operator input or host path is interpolated into the core shell.
// The marker lives alongside SETTINGS_FILE's named/bind-mounted directory and is read by backend.
func setMaintenance(ctx context.Context, runner Runner, enabled bool) error {
command := "mkdir -p /data/settings && : > /data/settings/maintenance.json && chmod 600 /data/settings/maintenance.json"
if !enabled { command = "rm -f /data/settings/maintenance.json" }
result, err := runCompose(ctx, runner, "exec", "-T", "core", "sh", "-ceu", command)
path := "deactivate"
if enabled { path = "activate" }
args := append([]string{"exec", "-T", "core", "curl", "-fsS", "-X", "POST"}, internalIdentityHeaders...)
args = append(args, "http://127.0.0.1:8787/internal/maintenance/"+path)
result, err := runCompose(ctx, runner, args...)
if err != nil { return commandError("maintenance admission gate", result, err) }
var status struct { Active bool `json:"active"`; Admissions int `json:"admissions"` }
if json.Unmarshal([]byte(result.Stdout), &status) != nil || status.Active != enabled || status.Admissions != 0 { return errors.New("maintenance admission gate did not acknowledge a quiescent state") }
return nil
}
func activeSessions(ctx context.Context, runner Runner) (bool, error) {
result, err := runCompose(ctx, runner, "exec", "-T", "core", "tht", "session", "list", "--json")
args := append([]string{"exec", "-T", "core", "curl", "-fsS"}, internalIdentityHeaders...)
args = append(args, "http://127.0.0.1:8787/sessions?scope=all")
result, err := runCompose(ctx, runner, args...)
if err != nil {
return false, commandError("active-session check", result, err)
}
var sessions []struct {
var payload struct { Sessions []struct {
Status string `json:"status"`
Archived bool `json:"archived"`
}
if err := json.Unmarshal([]byte(result.Stdout), &sessions); err != nil {
} `json:"sessions"` }
if err := json.Unmarshal([]byte(result.Stdout), &payload); err != nil {
return false, errors.New("active-session check returned invalid session data")
}
for _, session := range sessions {
if !session.Archived && session.Status == "open" {
for _, session := range payload.Sessions {
if !session.Archived && session.Status != "finalized" && session.Status != "closed" {
return true, nil
}
}
tools/thothctl/internal/pi/update_test.go
+11 -5
View File
@@ -132,7 +132,9 @@ func TestUpdateRequiresConfirmationAndDrainsActiveSessions(t *testing.T) {
if err != nil {
t.Fatalf("Update() with drain error = %v", err)
}
assertCalled(t, fake.calls, "compose exec -T core tht session list --json")
assertCalled(t, fake.calls, "http://127.0.0.1:8787/sessions?scope=all")
assertCalled(t, fake.calls, "/internal/maintenance/activate")
assertCalled(t, fake.calls, "/internal/maintenance/deactivate")
}
func TestRollbackRestoresInterruptedOrPreviouslyRecordedState(t *testing.T) {
@@ -162,7 +164,7 @@ func TestRollbackRestoresInterruptedOrPreviouslyRecordedState(t *testing.T) {
func TestUpdateRefusesToOverwriteInterruptedRecoveryState(t *testing.T) {
fake := newFakeRunner()
statePath := filepath.Join(t.TempDir(), "state.json")
writeStateForTest(t, statePath, State{Phase: PhaseRecreated, Previous: Image{ID: "sha256:old", Reference: "thothii-core:local", Volumes: []string{"settings"}, MountFingerprint: "recorded"}})
writeStateForTest(t, statePath, State{Phase: PhaseRecreated, Previous: Image{ID: "sha256:old", Reference: "thothii-core:local", Volumes: []string{"settings"}, MountFingerprint: mountFingerprint(nil)}})
_, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true})
if !errors.Is(err, ErrInterruptedUpdate) {
t.Fatalf("Update() error = %v, want interrupted update error", err)
@@ -242,12 +244,16 @@ func (f *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose
return compose.Result{Stdout: f.mountsJSON}, nil
}
return compose.Result{Stdout: `[{"Type":"volume","Name":"settings","Source":"settings","Destination":"/data/settings","RW":true},{"Type":"volume","Name":"pi-state","Source":"pi-state","Destination":"/home/thoth/.pi","RW":true},{"Type":"volume","Name":"sessions","Source":"sessions","Destination":"/data/sessions","RW":true},{"Type":"volume","Name":"workspace-registry","Source":"workspace-registry","Destination":"/data/workspace-registry","RW":true}]`}, nil
case strings.Contains(call, "tht session list --json"):
case strings.Contains(call, "/internal/maintenance/activate"):
return compose.Result{Stdout: `{"active":true,"admissions":0}`}, nil
case strings.Contains(call, "/internal/maintenance/deactivate"):
return compose.Result{Stdout: `{"active":false,"admissions":0}`}, nil
case strings.Contains(call, "/sessions?scope=all"):
if f.activeSessions {
f.activeSessions = false
return compose.Result{Stdout: `[{"status":"open","archived":false}]`}, nil
return compose.Result{Stdout: `{"sessions":[{"status":"open","archived":false}]}`}, nil
}
return compose.Result{Stdout: "[]"}, nil
return compose.Result{Stdout: `{"sessions":[]}`}, nil
case strings.Contains(call, "compose build"):
f.built = true
f.version = "0.81.0"