fix: harden Pi lifecycle recovery

2026-08-04 20:28:09 +02:00
parent 5b3ce93e31
commit 5ba2821a1b
24 changed files with 1764 additions and 384 deletions
@@ -21,6 +21,11 @@ type Defaults struct {
 	Thinking string `json:"thinking"`
 }

+type ModelOption struct {
+	Provider string `json:"provider"`
+	ID       string `json:"id"`
+}
+
 var internalIdentityHeaders = []string{
 	"-H", "x-thoth-principal-issuer: thothctl",
 	"-H", "x-thoth-principal-subject: thothctl-maintenance",
@@ -30,7 +35,7 @@ var internalIdentityHeaders = []string{

 // Configure changes the backend's real installation settings through a core-side helper. It
 // deliberately has no secret or endpoint input: external endpoints remain Compose-owned.
-func Configure(ctx context.Context, runner Runner, value Defaults) (retErr error) {
+func Configure(ctx context.Context, runner Runner, value Defaults) error {
 	if !choicePattern.MatchString(value.Provider) || !choicePattern.MatchString(value.Model) {
 		return errors.New("provider and model must be supported identifiers")
 	}
@@ -38,24 +43,15 @@ func Configure(ctx context.Context, runner Runner, value Defaults) (retErr error
 		return errors.New("thinking must be low, medium, or high")
 	}
 	before, err := renderedCore(ctx, runner)
-	if err != nil { return err }
-	args := append([]string{"exec", "-T", "core", "curl", "-fsS"}, internalIdentityHeaders...)
-	args = append(args, "http://127.0.0.1:8787/models")
-	models, err := runCompose(ctx, runner, args...)
 	if err != nil {
-		return commandError("Pi options check", models, err)
+		return err
 	}
-	var payload struct {
-		Models []struct {
-			Provider string `json:"provider"`
-			ID       string `json:"id"`
-		} `json:"models"`
-	}
-	if json.Unmarshal([]byte(models.Stdout), &payload) != nil || len(payload.Models) == 0 {
-		return errors.New("Pi options response is invalid or empty")
+	options, err := ConfigurationOptions(ctx, runner)
+	if err != nil {
+		return err
 	}
 	found := false
-	for _, model := range payload.Models {
+	for _, model := range options {
 		if model.Provider == value.Provider && model.ID == value.Model {
 			found = true
 		}
@@ -66,31 +62,86 @@ func Configure(ctx context.Context, runner Runner, value Defaults) (retErr error
 	settingsArgs := append([]string{"exec", "-T", "core", "curl", "-fsS"}, internalIdentityHeaders...)
 	settingsArgs = append(settingsArgs, "http://127.0.0.1:8787/settings")
 	oldResult, err := runCompose(ctx, runner, settingsArgs...)
-	if err != nil { return commandError("Pi installation settings capture", oldResult, err) }
+	if err != nil {
+		return commandError("Pi installation settings capture", oldResult, err)
+	}
 	var old Defaults
-	if json.Unmarshal([]byte(oldResult.Stdout), &old) != nil || old.Provider == "" || old.Model == "" || old.Thinking == "" { return errors.New("Pi installation settings capture is invalid") }
-	wrote := false
-	defer func() {
-		if retErr != nil && wrote {
-			result, restoreErr := runCompose(context.Background(), runner, "exec", "-T", "core", "node", "/app/backend/dist/settings/settings-cli.js", "--provider", old.Provider, "--model", old.Model, "--thinking", old.Thinking)
-			if restoreErr != nil || result.ExitCode != 0 { retErr = fmt.Errorf("%w; previous Pi settings could not be restored: recovery required", retErr) }
+	if json.Unmarshal([]byte(oldResult.Stdout), &old) != nil || old.Provider == "" || old.Model == "" || old.Thinking == "" {
+		return errors.New("Pi installation settings capture is invalid")
+	}
+	restore := func(cause error) error {
+		result, restoreErr := writeDefaults(context.Background(), runner, old)
+		if restoreErr != nil {
+			return fmt.Errorf("%w; previous Pi settings could not be restored: recovery required", cause)
 		}
-	}()
-	result, err := runCompose(ctx, runner, "exec", "-T", "core", "node", "/app/backend/dist/settings/settings-cli.js", "--provider", value.Provider, "--model", value.Model, "--thinking", value.Thinking)
-	if err != nil { return commandError("Pi installation settings write", result, err) }
-	wrote = true
+		if result.ExitCode != 0 {
+			return fmt.Errorf("%w; previous Pi settings could not be restored: recovery required", cause)
+		}
+		verified, readErr := readDefaults(context.Background(), runner, settingsArgs)
+		if readErr != nil || verified != old {
+			return fmt.Errorf("%w; previous Pi settings restoration could not be verified: recovery required", cause)
+		}
+		return cause
+	}
+	result, err := writeDefaults(ctx, runner, value)
+	if err != nil {
+		return restore(commandError("Pi installation settings write", result, err))
+	}
 	settings, err := runCompose(ctx, runner, settingsArgs...)
-	if err != nil { return commandError("Pi installation settings read-back", settings, err) }
+	if err != nil {
+		return restore(commandError("Pi installation settings read-back", settings, err))
+	}
 	var saved Defaults
 	if json.Unmarshal([]byte(settings.Stdout), &saved) != nil || saved.Provider != value.Provider || saved.Model != value.Model || saved.Thinking != value.Thinking {
-		return errors.New("Pi installation settings read-back did not match requested provider, model, and thinking")
+		return restore(errors.New("Pi installation settings read-back did not match requested provider, model, and thinking"))
 	}
 	after, err := renderedCore(ctx, runner)
-	if err != nil { return err }
-	if before.ConfigurationSHA != after.ConfigurationSHA { return errors.New("external endpoint configuration changed while configuring Pi") }
+	if err != nil {
+		return restore(err)
+	}
+	if before.ConfigurationSHA != after.ConfigurationSHA {
+		return restore(errors.New("external endpoint configuration changed while configuring Pi"))
+	}
 	return nil
 }

+func ConfigurationOptions(ctx context.Context, runner Runner) ([]ModelOption, error) {
+	args := append([]string{"exec", "-T", "core", "curl", "-fsS"}, internalIdentityHeaders...)
+	args = append(args, "http://127.0.0.1:8787/models")
+	models, err := runCompose(ctx, runner, args...)
+	if err != nil {
+		return nil, commandError("Pi options check", models, err)
+	}
+	var payload struct {
+		Models []ModelOption `json:"models"`
+	}
+	if json.Unmarshal([]byte(models.Stdout), &payload) != nil || len(payload.Models) == 0 {
+		return nil, errors.New("Pi options response is invalid or empty")
+	}
+	for _, option := range payload.Models {
+		if !choicePattern.MatchString(option.Provider) || !choicePattern.MatchString(option.ID) {
+			return nil, errors.New("Pi options response contains an invalid provider/model")
+		}
+	}
+	return payload.Models, nil
+}
+
+func writeDefaults(ctx context.Context, runner Runner, value Defaults) (compose.Result, error) {
+	return runCompose(ctx, runner, "exec", "-T", "core", "node", "/app/backend/dist/settings/settings-cli.js", "--provider", value.Provider, "--model", value.Model, "--thinking", value.Thinking)
+}
+
+func readDefaults(ctx context.Context, runner Runner, args []string) (Defaults, error) {
+	result, err := runCompose(ctx, runner, args...)
+	if err != nil {
+		return Defaults{}, commandError("Pi installation settings restoration read-back", result, err)
+	}
+	var value Defaults
+	if json.Unmarshal([]byte(result.Stdout), &value) != nil {
+		return Defaults{}, errors.New("Pi installation settings restoration read-back is invalid")
+	}
+	return value, nil
+}
+
 // Runner is the narrow, shell-free command boundary shared with thothctl.
 type Runner interface {
 	Run(context.Context, []string, io.Reader) (compose.Result, error)
@@ -135,64 +186,70 @@ func Test(ctx context.Context, runner Runner) error {
 	if _, err := Status(ctx, runner); err != nil {
 		return err
 	}
-	for _, path := range []string{"health", "models", "settings"} {
-		args := []string{"exec", "-T", "core", "curl", "-fsS"}
-		if path != "health" { args = append(args, internalIdentityHeaders...) }
-		args = append(args, "http://127.0.0.1:8787/"+path)
-		result, err := runCompose(ctx, runner, args...)
-		if err != nil {
-			return commandError("Pi smoke check", result, err)
-		}
-		var payload any
-		if err := json.Unmarshal([]byte(result.Stdout), &payload); err != nil {
-			return fmt.Errorf("Pi smoke check returned invalid %s response", path)
-		}
-		object, ok := payload.(map[string]any)
-		if !ok {
-			return fmt.Errorf("Pi smoke check returned invalid %s response", path)
-		}
-		switch path {
-		case "health":
-			if object["status"] != "ok" { return errors.New("Pi smoke health response is not ready") }
-		case "models":
-			models, ok := object["models"].([]any)
-			if !ok || len(models) == 0 { return errors.New("Pi smoke models response is empty") }
-			valid := false
-			for _, item := range models { if model, ok := item.(map[string]any); ok && stringField(model, "provider") != "" && stringField(model, "id") != "" { valid = true; break } }
-			if !valid { return errors.New("Pi smoke models response has no provider/model choices") }
-		case "settings":
-			if stringField(object, "provider") == "" || stringField(object, "model") == "" || stringField(object, "thinking") == "" { return errors.New("Pi smoke settings response is incomplete") }
+	health, err := runCompose(ctx, runner, "exec", "-T", "core", "curl", "-fsS", "http://127.0.0.1:8787/health")
+	if err != nil {
+		return commandError("Pi smoke check", health, err)
+	}
+	var healthPayload struct {
+		Status string `json:"status"`
+	}
+	if json.Unmarshal([]byte(health.Stdout), &healthPayload) != nil || healthPayload.Status != "ok" {
+		return errors.New("Pi smoke health response is not ready")
+	}
+	models, err := ConfigurationOptions(ctx, runner)
+	if err != nil {
+		return err
+	}
+	settingsArgs := append([]string{"exec", "-T", "core", "curl", "-fsS"}, internalIdentityHeaders...)
+	settingsArgs = append(settingsArgs, "http://127.0.0.1:8787/settings")
+	settings, err := runCompose(ctx, runner, settingsArgs...)
+	if err != nil {
+		return commandError("Pi smoke settings check", settings, err)
+	}
+	var selected Defaults
+	if json.Unmarshal([]byte(settings.Stdout), &selected) != nil || !choicePattern.MatchString(selected.Provider) || !choicePattern.MatchString(selected.Model) || (selected.Thinking != "low" && selected.Thinking != "medium" && selected.Thinking != "high") {
+		return errors.New("Pi smoke settings response is incomplete")
+	}
+	for _, model := range models {
+		if model.Provider == selected.Provider && model.ID == selected.Model {
+			return nil
 		}
 	}
-	return nil
+	return errors.New("configured provider/model does not match an available Pi model entry")
 }

-func stringField(value map[string]any, key string) string { text, _ := value[key].(string); return strings.TrimSpace(text) }
-
 func renderedCore(ctx context.Context, runner Runner) (Image, error) {
 	result, err := runCompose(ctx, runner, "config", "--format", "json")
 	if err != nil {
 		return Image{}, commandError("Compose configuration check", result, err)
 	}
-	var document struct {
-		Services map[string]struct {
-			Image       string         `json:"image"`
-			Environment map[string]any `json:"environment"`
-		} `json:"services"`
-	}
+	var document map[string]any
 	if err := json.Unmarshal([]byte(result.Stdout), &document); err != nil {
 		return Image{}, errors.New("Compose returned invalid rendered configuration")
 	}
-	core, exists := document.Services["core"]
-	if !exists || core.Image == "" {
+	services, ok := document["services"].(map[string]any)
+	if !ok {
+		return Image{}, errors.New("rendered Compose configuration has no services")
+	}
+	core, ok := services["core"].(map[string]any)
+	reference, _ := core["image"].(string)
+	if !ok || reference == "" {
 		return Image{}, errors.New("rendered Compose configuration has no core image")
 	}
-	endpoint, exists := core.Environment["THT_LLM_URL"].(string)
+	environment, _ := core["environment"].(map[string]any)
+	endpoint, exists := environment["THT_LLM_URL"].(string)
 	if !exists || strings.TrimSpace(endpoint) == "" {
 		return Image{}, errors.New("THT_LLM_URL must be configured before Pi lifecycle operations")
 	}
-	digest := sha256.Sum256([]byte(result.Stdout))
-	return Image{Reference: core.Image, ConfigurationSHA: fmt.Sprintf("%x", digest[:])}, nil
+	// Lifecycle overrides intentionally replace only core.image. Normalize that field so the
+	// non-secret configuration digest continues to detect endpoint/mount/configuration drift.
+	core["image"] = "<lifecycle-image>"
+	normalized, err := json.Marshal(document)
+	if err != nil {
+		return Image{}, errors.New("Compose configuration could not be normalized")
+	}
+	digest := sha256.Sum256(normalized)
+	return Image{Reference: reference, ConfigurationSHA: fmt.Sprintf("%x", digest[:])}, nil
 }

 func runCompose(ctx context.Context, runner Runner, args ...string) (compose.Result, error) {