fix(setup): harden verification and recovery
This commit is contained in:
@@ -70,19 +70,19 @@ func Run(ctx context.Context, runner compose.Runner, request Request, input io.R
|
||||
}
|
||||
result.Built = true
|
||||
if err := runCompose(ctx, runner, installation, "up", "--detach", "--remove-orphans"); err != nil {
|
||||
return Result{}, fmt.Errorf("setup stack start: %w", err)
|
||||
return Result{}, withStartupRecovery(fmt.Errorf("setup stack start: %w", err), "core")
|
||||
}
|
||||
result.Started = true
|
||||
if err := waitForHealthyServices(ctx, runner, installation); err != nil {
|
||||
return Result{}, err
|
||||
return Result{}, withStartupRecovery(err, recoveryService(err))
|
||||
}
|
||||
result.Healthy = true
|
||||
if err := aggregateDoctor(ctx, runner, installation); err != nil {
|
||||
return Result{}, err
|
||||
return Result{}, withStartupRecovery(err, "core")
|
||||
}
|
||||
controlled := compose.InstallationRunner{Installation: installation, Runner: runner}
|
||||
if err := pi.Doctor(ctx, controlled); err != nil {
|
||||
return Result{}, fmt.Errorf("setup Pi doctor: %w", err)
|
||||
return Result{}, withStartupRecovery(fmt.Errorf("setup Pi doctor: %w", err), "core")
|
||||
}
|
||||
fmt.Fprintf(output, "ThothII is ready at %s\nInstallation descriptor: %s\nNext: tht status\n", frontendURL(installation), result.DescriptorPath)
|
||||
return result, nil
|
||||
@@ -147,6 +147,21 @@ func aggregateDoctor(ctx context.Context, runner compose.Runner, installation co
|
||||
return nil
|
||||
}
|
||||
|
||||
func withStartupRecovery(cause error, service string) error {
|
||||
if service == "" {
|
||||
service = "core"
|
||||
}
|
||||
return fmt.Errorf("%w; containers were left running for diagnosis: tht logs %s; then run tht status", cause, service)
|
||||
}
|
||||
|
||||
func recoveryService(cause error) string {
|
||||
var healthFailure healthFailure
|
||||
if errors.As(cause, &healthFailure) && healthFailure.service != "" {
|
||||
return healthFailure.service
|
||||
}
|
||||
return "core"
|
||||
}
|
||||
|
||||
type serviceStatus struct {
|
||||
Service string `json:"Service"`
|
||||
State string `json:"State"`
|
||||
@@ -154,6 +169,15 @@ type serviceStatus struct {
|
||||
ExitCode json.RawMessage `json:"ExitCode"`
|
||||
}
|
||||
|
||||
type healthFailure struct {
|
||||
service string
|
||||
state string
|
||||
}
|
||||
|
||||
func (e healthFailure) Error() string {
|
||||
return fmt.Sprintf("setup health timed out waiting for %s (last state: %s)", e.service, e.state)
|
||||
}
|
||||
|
||||
func waitForHealthyServices(ctx context.Context, runner compose.Runner, installation config.Installation) error {
|
||||
healthContext, cancel := context.WithTimeout(ctx, setupHealthTimeout)
|
||||
defer cancel()
|
||||
@@ -178,7 +202,7 @@ func waitForHealthyServices(ctx context.Context, runner compose.Runner, installa
|
||||
}
|
||||
select {
|
||||
case <-healthContext.Done():
|
||||
return fmt.Errorf("setup health timed out waiting for %s (last state: %s); containers were left running for diagnosis: tht logs %s; then run tht status", lastService, lastState, lastService)
|
||||
return healthFailure{service: lastService, state: lastState}
|
||||
case <-time.After(healthPollInterval):
|
||||
}
|
||||
}
|
||||
@@ -264,31 +288,59 @@ func requireVolumes(renderedConfig string) error {
|
||||
if err := json.Unmarshal([]byte(renderedConfig), &document); err != nil {
|
||||
return errors.New("Compose returned invalid rendered configuration")
|
||||
}
|
||||
if len(document.Volumes) == 0 {
|
||||
return errors.New("rendered Compose configuration declares no volumes")
|
||||
for _, volume := range []string{"settings", "pi-state", "workspace-registry", "workspace-secrets", "sessions", "qdrant-data", "embedding-models"} {
|
||||
if _, exists := document.Volumes[volume]; !exists {
|
||||
return fmt.Errorf("rendered Compose configuration is missing required volume %s", volume)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func requireLF(root string) error {
|
||||
return filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error {
|
||||
for _, path := range []string{filepath.Join(root, "compose.yaml"), filepath.Join(root, "deploy"), filepath.Join(root, "docker")} {
|
||||
if err := requireLFPath(path); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func requireLFPath(path string) error {
|
||||
info, err := os.Lstat(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
if err != nil || info.Mode()&os.ModeSymlink != 0 {
|
||||
return nil
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return requireLFFile(path, info.Mode())
|
||||
}
|
||||
return filepath.WalkDir(path, func(path string, entry os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if entry.IsDir() || entry.Type()&os.ModeSymlink != 0 || !requiresLF(entry.Name()) {
|
||||
if entry.IsDir() || entry.Type()&os.ModeSymlink != 0 {
|
||||
return nil
|
||||
}
|
||||
contents, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.Contains(string(contents), "\r\n") {
|
||||
return fmt.Errorf("CRLF line endings found in %s", filepath.Base(path))
|
||||
}
|
||||
return nil
|
||||
return requireLFFile(path, entry.Type())
|
||||
})
|
||||
}
|
||||
|
||||
func requireLFFile(path string, mode os.FileMode) error {
|
||||
if !mode.IsRegular() || !requiresLF(filepath.Base(path)) {
|
||||
return nil
|
||||
}
|
||||
contents, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.Contains(string(contents), "\r\n") {
|
||||
return fmt.Errorf("CRLF line endings found in %s", filepath.Base(path))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func requiresLF(name string) bool {
|
||||
if name == "Dockerfile" || strings.HasPrefix(name, "Dockerfile.") || strings.HasSuffix(name, ".Dockerfile") {
|
||||
return true
|
||||
|
||||
Reference in New Issue
Block a user