fix(cli): strengthen aggregate doctor diagnostics
This commit is contained in:
@@ -10,6 +10,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/compose"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
||||
@@ -24,6 +25,10 @@ const (
|
||||
StatusSkipped = "skipped"
|
||||
)
|
||||
|
||||
const probeTimeout = 5 * time.Second
|
||||
|
||||
const registryValidationProgram = `const fs=require("node:fs");const path="/data/workspace-registry/state/active.json";const s=JSON.parse(fs.readFileSync(path,"utf8"));const hex=/^[0-9a-f]{40}$/;if(!hex.test(s.head)||!Array.isArray(s.revisions)||s.revisions.some((r)=>!r||typeof r.id!=="string"||!r.id||!hex.test(r.commit)||!hex.test(r.blob))){process.exit(1)}for(const r of s.revisions){fs.accessSync("/data/workspace-registry/snapshots/"+r.commit+"/"+r.id+".yaml",fs.constants.R_OK)}`
|
||||
|
||||
// Check is one named, redacted diagnostic outcome.
|
||||
type Check struct {
|
||||
Name string `json:"name"`
|
||||
@@ -42,14 +47,58 @@ type Runner interface {
|
||||
Run(context.Context, []string, io.Reader) (compose.Result, error)
|
||||
}
|
||||
|
||||
// HTTPProbeTarget identifies one local service endpoint that must answer a bounded request.
|
||||
type HTTPProbeTarget struct {
|
||||
Name string
|
||||
Service string
|
||||
URL string
|
||||
}
|
||||
|
||||
// HTTPProbe is injectable so reachability failures remain independently testable.
|
||||
type HTTPProbe interface {
|
||||
Probe(context.Context, HTTPProbeTarget) error
|
||||
}
|
||||
|
||||
type composeHTTPProbe struct {
|
||||
installation config.Installation
|
||||
runner Runner
|
||||
}
|
||||
|
||||
func (p composeHTTPProbe) Probe(ctx context.Context, target HTTPProbeTarget) error {
|
||||
probeContext, cancel := context.WithTimeout(ctx, probeTimeout)
|
||||
defer cancel()
|
||||
var command []string
|
||||
switch target.Name {
|
||||
case "core":
|
||||
command = []string{"exec", "-T", target.Service, "curl", "-fsS", "--max-time", "5", target.URL}
|
||||
case "frontend":
|
||||
command = []string{"exec", "-T", target.Service, "wget", "-q", "-T", "5", "-O", "/dev/null", target.URL}
|
||||
default:
|
||||
return errors.New("unknown HTTP probe target")
|
||||
}
|
||||
result, err := p.runner.Run(probeContext, p.installation.ComposeArgs(command...), nil)
|
||||
if err != nil {
|
||||
return errors.New(commandDetail(target.Name+" HTTP probe", result, err, nil))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Run performs diagnostics only. Expected environmental failures become failed checks so that
|
||||
// callers can always render a complete report; unexpected local read errors are also reported.
|
||||
func Run(ctx context.Context, installation config.Installation, runner Runner) (Report, error) {
|
||||
return RunWithProbe(ctx, installation, runner, composeHTTPProbe{installation: installation, runner: runner})
|
||||
}
|
||||
|
||||
// RunWithProbe performs diagnostics only, with an injectable bounded HTTP probe.
|
||||
func RunWithProbe(ctx context.Context, installation config.Installation, runner Runner, probe HTTPProbe) (Report, error) {
|
||||
if runner == nil {
|
||||
return Report{}, errors.New("doctor requires a Docker command runner")
|
||||
}
|
||||
if probe == nil {
|
||||
return Report{}, errors.New("doctor requires an HTTP probe")
|
||||
}
|
||||
secretValues := secretValues(installation)
|
||||
report := Report{Checks: make([]Check, 0, 10)}
|
||||
report := Report{Checks: make([]Check, 0, 11)}
|
||||
add := func(name, status, detail string) {
|
||||
report.Checks = append(report.Checks, Check{Name: name, Status: status, Detail: output.SanitizeDetail(detail, secretValues)})
|
||||
}
|
||||
@@ -59,12 +108,18 @@ func Run(ctx context.Context, installation config.Installation, runner Runner) (
|
||||
} else {
|
||||
add("descriptor", StatusPassed, "installation descriptor is loaded")
|
||||
}
|
||||
if err := filePermissions(installation); err != nil {
|
||||
add("files", StatusFailed, err.Error())
|
||||
} else {
|
||||
add("files", StatusPassed, "declared host files have safe permissions")
|
||||
}
|
||||
|
||||
if !commandCheck(ctx, runner, []string{"version", "--format", "{{.Client.Version}}"}, secretValues, add, "docker", "Docker Engine") {
|
||||
add("compose", StatusSkipped, "Docker Engine is unavailable")
|
||||
add("configuration", StatusSkipped, "Docker Engine is unavailable")
|
||||
add("files", StatusSkipped, "Docker Engine is unavailable")
|
||||
add("services", StatusSkipped, "Docker Engine is unavailable")
|
||||
add("core-http", StatusSkipped, "core is unavailable")
|
||||
add("frontend-http", StatusSkipped, "frontend is unavailable")
|
||||
add("workspace-registry", StatusSkipped, "core is unavailable")
|
||||
add("workflow", StatusSkipped, "core is unavailable")
|
||||
add("pi", StatusSkipped, "core is unavailable")
|
||||
@@ -72,8 +127,9 @@ func Run(ctx context.Context, installation config.Installation, runner Runner) (
|
||||
}
|
||||
if !commandCheck(ctx, runner, []string{"compose", "version", "--short"}, secretValues, add, "compose", "Docker Compose") {
|
||||
add("configuration", StatusSkipped, "Docker Compose is unavailable")
|
||||
add("files", StatusSkipped, "Docker Compose is unavailable")
|
||||
add("services", StatusSkipped, "Docker Compose is unavailable")
|
||||
add("core-http", StatusSkipped, "core is unavailable")
|
||||
add("frontend-http", StatusSkipped, "frontend is unavailable")
|
||||
add("workspace-registry", StatusSkipped, "core is unavailable")
|
||||
add("workflow", StatusSkipped, "core is unavailable")
|
||||
add("pi", StatusSkipped, "core is unavailable")
|
||||
@@ -94,12 +150,6 @@ func Run(ctx context.Context, installation config.Installation, runner Runner) (
|
||||
add("configuration", StatusPassed, "Compose configuration and required volumes are valid")
|
||||
}
|
||||
|
||||
if err := filePermissions(installation); err != nil {
|
||||
add("files", StatusFailed, err.Error())
|
||||
} else {
|
||||
add("files", StatusPassed, "declared host files have safe permissions")
|
||||
}
|
||||
|
||||
status, statusAvailable := serviceStatus(ctx, installation, runner, secretValues, add)
|
||||
coreRunning := false
|
||||
if statusAvailable {
|
||||
@@ -110,22 +160,51 @@ func Run(ctx context.Context, installation config.Installation, runner Runner) (
|
||||
}
|
||||
}
|
||||
if !coreRunning {
|
||||
add("core-http", StatusSkipped, "core is not running")
|
||||
add("frontend-http", StatusSkipped, "core is not running")
|
||||
add("workspace-registry", StatusSkipped, "core is not running")
|
||||
add("workflow", StatusSkipped, "core is not running")
|
||||
add("pi", StatusSkipped, "core is not running")
|
||||
return finalize(report), nil
|
||||
}
|
||||
|
||||
if configReady && workspaceRegistryConfigured(rendered) {
|
||||
add("workspace-registry", StatusPassed, "workspace-registry volume is configured")
|
||||
} else {
|
||||
add("workspace-registry", StatusFailed, "workspace-registry volume is not configured")
|
||||
}
|
||||
reachabilityChecks(ctx, probe, secretValues, add)
|
||||
registryCheck(ctx, installation, runner, secretValues, add, configReady, rendered)
|
||||
workflowCheck(ctx, installation, runner, secretValues, add)
|
||||
piCheck(ctx, installation, runner, secretValues, add)
|
||||
return finalize(report), nil
|
||||
}
|
||||
|
||||
func reachabilityChecks(ctx context.Context, probe HTTPProbe, secrets []string, add func(string, string, string)) {
|
||||
for _, target := range []HTTPProbeTarget{
|
||||
{Name: "core", Service: "core", URL: "http://127.0.0.1:8787/health"},
|
||||
{Name: "frontend", Service: "frontend", URL: "http://127.0.0.1:8080/"},
|
||||
} {
|
||||
if err := probe.Probe(ctx, target); err != nil {
|
||||
add(target.Name+"-http", StatusFailed, output.SanitizeDetail(err.Error(), secrets))
|
||||
continue
|
||||
}
|
||||
add(target.Name+"-http", StatusPassed, target.Name+" answered a bounded HTTP probe")
|
||||
}
|
||||
}
|
||||
|
||||
func registryCheck(ctx context.Context, installation config.Installation, runner Runner, secrets []string, add func(string, string, string), configReady bool, rendered string) {
|
||||
if !configReady {
|
||||
add("workspace-registry", StatusFailed, "workspace-registry cannot be checked because Compose configuration is invalid")
|
||||
return
|
||||
}
|
||||
if !workspaceRegistryDeclared(rendered) {
|
||||
add("workspace-registry", StatusFailed, "workspace-registry volume is not configured")
|
||||
return
|
||||
}
|
||||
result, err := runner.Run(ctx, installation.ComposeArgs("exec", "-T", "core", "node", "-e", registryValidationProgram), nil)
|
||||
if err != nil {
|
||||
add("workspace-registry", StatusFailed, commandDetail("container-local workspace registry", result, err, secrets))
|
||||
return
|
||||
}
|
||||
add("workspace-registry", StatusPassed, "container-local active registry state and snapshots are valid")
|
||||
}
|
||||
|
||||
func finalize(report Report) Report {
|
||||
report.OK = len(report.Checks) > 0
|
||||
for _, check := range report.Checks {
|
||||
@@ -238,7 +317,7 @@ func ValidateVolumes(rendered string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func workspaceRegistryConfigured(rendered string) bool {
|
||||
func workspaceRegistryDeclared(rendered string) bool {
|
||||
var document struct {
|
||||
Volumes map[string]json.RawMessage `json:"volumes"`
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user