fix: acknowledge pi maintenance barrier

tools/thothctl/cmd/thothctl/main.go

@@ -172,7 +172,9 @@ func piCommand(ctx context.Context, installation config.Installation, runner com
 		if err := pi.Configure(ctx, controlled, defaults); err != nil {
 			return piFailure(stderr, err, secretValues)
 		}
-		fmt.Fprintln(stdout, "Pi defaults applied and read back. Put credentials only in PI_AUTH_FILE (/home/thoth/.pi/agent/auth.json, mode 0600); never pass credentials to thothctl.")
+		authFile, _ := installation.EnvironmentValue("PI_AUTH_FILE")
+		if authFile == "" { authFile = "the host path declared by PI_AUTH_FILE" }
+		fmt.Fprintf(stdout, "Pi defaults applied and read back. Put credentials only in PI_AUTH_FILE=%s (mode 0600); expected variables/files are PI_AUTH_FILE and /home/thoth/.pi/agent/auth.json. Never pass credentials to thothctl.\n", authFile)
 		return 0
 	case "update":
 		request, err := parsePiUpdateArgs(args[1:], filepath.Join(installation.ProjectDirectory, ".thothctl", "update-state.json"))

tools/thothctl/internal/config/installation.go

@@ -164,6 +164,16 @@ func (i Installation) SecretFiles() ([]string, error) {
 	return files, nil
 }
 
+// EnvironmentValue returns one declared installation value without exposing dotenv parsing to
+// callers. It is used only for operator-visible file locations, never for secret content.
+func (i Installation) EnvironmentValue(name string) (string, error) {
+	contents, err := safeio.ReadCanonicalRegular(i.EnvFile, maxEnvironmentFileBytes)
+	if err != nil { return "", errors.New("installation environment could not be read") }
+	values, err := parseComposeDotenv(contents)
+	if err != nil { return "", errors.New("installation environment could not be read") }
+	return values[name], nil
+}
+
 func parseComposeDotenv(contents []byte) (map[string]string, error) {
 	dotenvParseMu.Lock()
 	defer dotenvParseMu.Unlock()

tools/thothctl/internal/pi/commands.go

@@ -30,7 +30,7 @@ var internalIdentityHeaders = []string{
 
 // Configure changes the backend's real installation settings through a core-side helper. It
 // deliberately has no secret or endpoint input: external endpoints remain Compose-owned.
-func Configure(ctx context.Context, runner Runner, value Defaults) error {
+func Configure(ctx context.Context, runner Runner, value Defaults) (retErr error) {
 	if !choicePattern.MatchString(value.Provider) || !choicePattern.MatchString(value.Model) {
 		return errors.New("provider and model must be supported identifiers")
 	}
@@ -63,10 +63,22 @@ func Configure(ctx context.Context, runner Runner, value Defaults) error {
 	if !found {
 		return errors.New("provider/model is not in Pi options")
 	}
-	result, err := runCompose(ctx, runner, "exec", "-T", "core", "node", "/app/backend/dist/settings/settings-cli.js", "--provider", value.Provider, "--model", value.Model, "--thinking", value.Thinking)
-	if err != nil { return commandError("Pi installation settings write", result, err) }
 	settingsArgs := append([]string{"exec", "-T", "core", "curl", "-fsS"}, internalIdentityHeaders...)
 	settingsArgs = append(settingsArgs, "http://127.0.0.1:8787/settings")
+	oldResult, err := runCompose(ctx, runner, settingsArgs...)
+	if err != nil { return commandError("Pi installation settings capture", oldResult, err) }
+	var old Defaults
+	if json.Unmarshal([]byte(oldResult.Stdout), &old) != nil || old.Provider == "" || old.Model == "" || old.Thinking == "" { return errors.New("Pi installation settings capture is invalid") }
+	wrote := false
+	defer func() {
+		if retErr != nil && wrote {
+			result, restoreErr := runCompose(context.Background(), runner, "exec", "-T", "core", "node", "/app/backend/dist/settings/settings-cli.js", "--provider", old.Provider, "--model", old.Model, "--thinking", old.Thinking)
+			if restoreErr != nil || result.ExitCode != 0 { retErr = fmt.Errorf("%w; previous Pi settings could not be restored: recovery required", retErr) }
+		}
+	}()
+	result, err := runCompose(ctx, runner, "exec", "-T", "core", "node", "/app/backend/dist/settings/settings-cli.js", "--provider", value.Provider, "--model", value.Model, "--thinking", value.Thinking)
+	if err != nil { return commandError("Pi installation settings write", result, err) }
+	wrote = true
 	settings, err := runCompose(ctx, runner, settingsArgs...)
 	if err != nil { return commandError("Pi installation settings read-back", settings, err) }
 	var saved Defaults

tools/thothctl/internal/pi/state.go

@@ -79,6 +79,9 @@ func readState(path string) (State, error) {
 	if state.Version != stateFileVersion || state.Previous.ID == "" || state.Previous.Reference == "" || state.Previous.MountFingerprint == "" {
 		return State{}, errors.New("update recovery state is incomplete")
 	}
+	if mountFingerprint(state.Previous.Mounts) != state.Previous.MountFingerprint || (state.Candidate.ID != "" && mountFingerprint(state.Candidate.Mounts) != state.Candidate.MountFingerprint) {
+		return State{}, errors.New("update recovery state mount fingerprint is invalid")
+	}
 	return state, nil
 }
 
@@ -144,9 +147,10 @@ func acquireLock(statePath string) (*updateLock, error) {
 		return nil, errors.New("could not create Pi update recovery directory")
 	}
 	path := statePath + ".lock"
-	if err := os.Mkdir(path, 0o700); err != nil {
+	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
+	if err != nil {
 		if errors.Is(err, os.ErrExist) {
-			if reclaimDeadLocalLock(path) {
+			if reclaimDeadLocalLock(path, statePath) {
 				return acquireLock(statePath)
 			}
 			return nil, ErrLockHeld
@@ -154,28 +158,36 @@ func acquireLock(statePath string) (*updateLock, error) {
 		return nil, errors.New("could not acquire Pi update lock")
 	}
 	host, err := os.Hostname()
-	if err != nil { _ = os.Remove(path); return nil, errors.New("could not identify Pi update lock owner") }
+	if err != nil { _ = file.Close(); _ = os.Remove(path); return nil, errors.New("could not identify Pi update lock owner") }
 	owner := lockOwner{PID: os.Getpid(), Host: host, StartedAt: time.Now().UTC(), Transaction: fmt.Sprintf("%d-%d", os.Getpid(), time.Now().UnixNano())}
 	contents, err := json.Marshal(owner)
-	if err != nil { _ = os.Remove(path); return nil, errors.New("could not record Pi update lock owner") }
-	if err := writeFileDurably(filepath.Join(path, "owner.json"), ".owner-", append(contents, '\n')); err != nil {
-		_ = os.Remove(path)
+	if err != nil { _ = file.Close(); _ = os.Remove(path); return nil, errors.New("could not record Pi update lock owner") }
+	if _, err := file.Write(append(contents, '\n')); err != nil || file.Sync() != nil || file.Close() != nil {
+		_ = file.Close(); _ = os.Remove(path)
 		return nil, errors.New("could not record Pi update lock owner")
 	}
 	return &updateLock{path: path}, nil
 }
-func (l *updateLock) Release() { _ = os.Remove(filepath.Join(l.path, "owner.json")); _ = os.Remove(l.path) }
+func (l *updateLock) Release() { _ = os.Remove(l.path) }
 
 // reclaimDeadLocalLock is deliberately conservative: a malformed, remote, or merely old lock
 // is recovery-required. Only a process we can prove is gone on this machine is reclaimed.
-func reclaimDeadLocalLock(path string) bool {
-	contents, err := os.ReadFile(filepath.Join(path, "owner.json"))
-	if err != nil { return false }
+func reclaimDeadLocalLock(path, statePath string) bool {
+	info, err := os.Stat(path)
+	if err != nil || time.Since(info.ModTime()) < 5*time.Minute || !hasPendingRecoveryState(statePath) { return false }
+	contents, err := os.ReadFile(path)
+	if err != nil { return os.Remove(path) == nil }
 	var owner lockOwner
 	if json.Unmarshal(contents, &owner) != nil || owner.PID <= 0 || owner.Host == "" { return false }
 	host, err := os.Hostname()
 	if err != nil || owner.Host != host { return false }
 	if processAlive(owner.PID) { return false }
-	if err := os.Remove(filepath.Join(path, "owner.json")); err != nil { return false }
 	return os.Remove(path) == nil
 }
+
+func hasPendingRecoveryState(path string) bool {
+	contents, err := os.ReadFile(path); if err != nil { return false }
+	var state State
+	if json.Unmarshal(contents, &state) != nil { return false }
+	return state.Phase != PhaseVerified && state.Phase != PhaseRolledBack && state.Phase != PhaseNoop
+}

tools/thothctl/internal/pi/update.go

@@ -166,7 +166,7 @@ func Update(ctx context.Context, runner Runner, request Request) (result Result,
 }
 
 // Rollback restores the image recorded in durable update state. It is safe for interrupted runs.
-func Rollback(ctx context.Context, runner Runner, statePath string, confirm bool) (Result, error) {
+func Rollback(ctx context.Context, runner Runner, statePath string, confirm bool) (result Result, retErr error) {
 	lock, err := acquireLock(statePath)
 	if err != nil {
 		return Result{StatePath: statePath}, err
@@ -175,6 +175,16 @@ func Rollback(ctx context.Context, runner Runner, statePath string, confirm bool
 	if !confirm {
 		return Result{StatePath: statePath}, ErrConfirmationRequired
 	}
+	if err := setMaintenance(ctx, runner, true); err != nil { return Result{StatePath: statePath}, err }
+	defer func() {
+		if clearErr := setMaintenance(context.Background(), runner, false); clearErr != nil {
+			result = Result{Phase: PhaseFailed, StatePath: statePath}
+			if retErr == nil { retErr = errors.New("maintenance admission gate could not be cleared: recovery required")
+			} else { retErr = fmt.Errorf("%w; maintenance admission gate could not be cleared: recovery required", retErr) }
+		}
+	}()
+	if active, err := activeSessions(ctx, runner); err != nil { return Result{StatePath: statePath}, err
+	} else if active { return Result{StatePath: statePath}, ErrActiveSessions }
 	state, err := readState(statePath)
 	if err != nil {
 		return Result{StatePath: statePath}, err
@@ -232,30 +242,34 @@ func canonicalDigestReference(value string) (string, error) {
 	return reference.FamiliarString(canonical), nil
 }
 
-// The command text is fixed; no operator input or host path is interpolated into the core shell.
-// The marker lives alongside SETTINGS_FILE's named/bind-mounted directory and is read by backend.
 func setMaintenance(ctx context.Context, runner Runner, enabled bool) error {
-	command := "mkdir -p /data/settings && : > /data/settings/maintenance.json && chmod 600 /data/settings/maintenance.json"
-	if !enabled { command = "rm -f /data/settings/maintenance.json" }
-	result, err := runCompose(ctx, runner, "exec", "-T", "core", "sh", "-ceu", command)
+	path := "deactivate"
+	if enabled { path = "activate" }
+	args := append([]string{"exec", "-T", "core", "curl", "-fsS", "-X", "POST"}, internalIdentityHeaders...)
+	args = append(args, "http://127.0.0.1:8787/internal/maintenance/"+path)
+	result, err := runCompose(ctx, runner, args...)
 	if err != nil { return commandError("maintenance admission gate", result, err) }
+	var status struct { Active bool `json:"active"`; Admissions int `json:"admissions"` }
+	if json.Unmarshal([]byte(result.Stdout), &status) != nil || status.Active != enabled || status.Admissions != 0 { return errors.New("maintenance admission gate did not acknowledge a quiescent state") }
 	return nil
 }
 
 func activeSessions(ctx context.Context, runner Runner) (bool, error) {
-	result, err := runCompose(ctx, runner, "exec", "-T", "core", "tht", "session", "list", "--json")
+	args := append([]string{"exec", "-T", "core", "curl", "-fsS"}, internalIdentityHeaders...)
+	args = append(args, "http://127.0.0.1:8787/sessions?scope=all")
+	result, err := runCompose(ctx, runner, args...)
 	if err != nil {
 		return false, commandError("active-session check", result, err)
 	}
-	var sessions []struct {
+	var payload struct { Sessions []struct {
 		Status   string `json:"status"`
 		Archived bool   `json:"archived"`
-	}
-	if err := json.Unmarshal([]byte(result.Stdout), &sessions); err != nil {
+	} `json:"sessions"` }
+	if err := json.Unmarshal([]byte(result.Stdout), &payload); err != nil {
 		return false, errors.New("active-session check returned invalid session data")
 	}
-	for _, session := range sessions {
-		if !session.Archived && session.Status == "open" {
+	for _, session := range payload.Sessions {
+		if !session.Archived && session.Status != "finalized" && session.Status != "closed" {
 			return true, nil
 		}
 	}

tools/thothctl/internal/pi/update_test.go

@@ -132,7 +132,9 @@ func TestUpdateRequiresConfirmationAndDrainsActiveSessions(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Update() with drain error = %v", err)
 	}
-	assertCalled(t, fake.calls, "compose exec -T core tht session list --json")
+	assertCalled(t, fake.calls, "http://127.0.0.1:8787/sessions?scope=all")
+	assertCalled(t, fake.calls, "/internal/maintenance/activate")
+	assertCalled(t, fake.calls, "/internal/maintenance/deactivate")
 }
 
 func TestRollbackRestoresInterruptedOrPreviouslyRecordedState(t *testing.T) {
@@ -162,7 +164,7 @@ func TestRollbackRestoresInterruptedOrPreviouslyRecordedState(t *testing.T) {
 func TestUpdateRefusesToOverwriteInterruptedRecoveryState(t *testing.T) {
 	fake := newFakeRunner()
 	statePath := filepath.Join(t.TempDir(), "state.json")
-	writeStateForTest(t, statePath, State{Phase: PhaseRecreated, Previous: Image{ID: "sha256:old", Reference: "thothii-core:local", Volumes: []string{"settings"}, MountFingerprint: "recorded"}})
+	writeStateForTest(t, statePath, State{Phase: PhaseRecreated, Previous: Image{ID: "sha256:old", Reference: "thothii-core:local", Volumes: []string{"settings"}, MountFingerprint: mountFingerprint(nil)}})
 	_, err := Update(context.Background(), fake, Request{StatePath: statePath, Version: "0.81.0", Source: BuildSource, Confirm: true})
 	if !errors.Is(err, ErrInterruptedUpdate) {
 		t.Fatalf("Update() error = %v, want interrupted update error", err)
@@ -242,12 +244,16 @@ func (f *fakeRunner) Run(_ context.Context, args []string, _ io.Reader) (compose
 			return compose.Result{Stdout: f.mountsJSON}, nil
 		}
 		return compose.Result{Stdout: `[{"Type":"volume","Name":"settings","Source":"settings","Destination":"/data/settings","RW":true},{"Type":"volume","Name":"pi-state","Source":"pi-state","Destination":"/home/thoth/.pi","RW":true},{"Type":"volume","Name":"sessions","Source":"sessions","Destination":"/data/sessions","RW":true},{"Type":"volume","Name":"workspace-registry","Source":"workspace-registry","Destination":"/data/workspace-registry","RW":true}]`}, nil
-	case strings.Contains(call, "tht session list --json"):
+	case strings.Contains(call, "/internal/maintenance/activate"):
+		return compose.Result{Stdout: `{"active":true,"admissions":0}`}, nil
+	case strings.Contains(call, "/internal/maintenance/deactivate"):
+		return compose.Result{Stdout: `{"active":false,"admissions":0}`}, nil
+	case strings.Contains(call, "/sessions?scope=all"):
 		if f.activeSessions {
 			f.activeSessions = false
-			return compose.Result{Stdout: `[{"status":"open","archived":false}]`}, nil
+			return compose.Result{Stdout: `{"sessions":[{"status":"open","archived":false}]}`}, nil
 		}
-		return compose.Result{Stdout: "[]"}, nil
+		return compose.Result{Stdout: `{"sessions":[]}`}, nil
 	case strings.Contains(call, "compose build"):
 		f.built = true
 		f.version = "0.81.0"