feat(cli): add version diagnostics and build-aware start
This commit is contained in:
@@ -0,0 +1,256 @@
|
||||
// Package doctor aggregates non-mutating host and container diagnostics for one installation.
|
||||
package doctor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/compose"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/output"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/pi"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/service"
|
||||
)
|
||||
|
||||
const (
|
||||
StatusPassed = "passed"
|
||||
StatusFailed = "failed"
|
||||
StatusSkipped = "skipped"
|
||||
)
|
||||
|
||||
// Check is one named, redacted diagnostic outcome.
|
||||
type Check struct {
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
Detail string `json:"detail"`
|
||||
}
|
||||
|
||||
// Report is the typed, machine-readable diagnostic result.
|
||||
type Report struct {
|
||||
OK bool `json:"ok"`
|
||||
Checks []Check `json:"checks"`
|
||||
}
|
||||
|
||||
// Runner is the shell-free Docker boundary used for all host and in-container checks.
|
||||
type Runner interface {
|
||||
Run(context.Context, []string, io.Reader) (compose.Result, error)
|
||||
}
|
||||
|
||||
// Run performs diagnostics only. Expected environmental failures become failed checks so that
|
||||
// callers can always render a complete report; unexpected local read errors are also reported.
|
||||
func Run(ctx context.Context, installation config.Installation, runner Runner) (Report, error) {
|
||||
if runner == nil {
|
||||
return Report{}, errors.New("doctor requires a Docker command runner")
|
||||
}
|
||||
secretValues := secretValues(installation)
|
||||
report := Report{Checks: make([]Check, 0, 10)}
|
||||
add := func(name, status, detail string) {
|
||||
report.Checks = append(report.Checks, Check{Name: name, Status: status, Detail: output.SanitizeDetail(detail, secretValues)})
|
||||
}
|
||||
|
||||
if err := validateInstallation(installation); err != nil {
|
||||
add("descriptor", StatusFailed, err.Error())
|
||||
} else {
|
||||
add("descriptor", StatusPassed, "installation descriptor is loaded")
|
||||
}
|
||||
|
||||
if !commandCheck(ctx, runner, []string{"version", "--format", "{{.Client.Version}}"}, secretValues, add, "docker", "Docker Engine") {
|
||||
add("compose", StatusSkipped, "Docker Engine is unavailable")
|
||||
add("configuration", StatusSkipped, "Docker Engine is unavailable")
|
||||
add("files", StatusSkipped, "Docker Engine is unavailable")
|
||||
add("services", StatusSkipped, "Docker Engine is unavailable")
|
||||
add("workspace-registry", StatusSkipped, "core is unavailable")
|
||||
add("workflow", StatusSkipped, "core is unavailable")
|
||||
add("pi", StatusSkipped, "core is unavailable")
|
||||
return finalize(report), nil
|
||||
}
|
||||
if !commandCheck(ctx, runner, []string{"compose", "version", "--short"}, secretValues, add, "compose", "Docker Compose") {
|
||||
add("configuration", StatusSkipped, "Docker Compose is unavailable")
|
||||
add("files", StatusSkipped, "Docker Compose is unavailable")
|
||||
add("services", StatusSkipped, "Docker Compose is unavailable")
|
||||
add("workspace-registry", StatusSkipped, "core is unavailable")
|
||||
add("workflow", StatusSkipped, "core is unavailable")
|
||||
add("pi", StatusSkipped, "core is unavailable")
|
||||
return finalize(report), nil
|
||||
}
|
||||
|
||||
configReady := false
|
||||
rendered := ""
|
||||
if result, err := runner.Run(ctx, installation.ComposeArgs("config", "--quiet"), nil); err != nil {
|
||||
add("configuration", StatusFailed, commandDetail("Compose configuration", result, err, secretValues))
|
||||
} else if result, err := runner.Run(ctx, installation.ComposeArgs("config", "--format", "json"), nil); err != nil {
|
||||
add("configuration", StatusFailed, commandDetail("Compose rendering", result, err, secretValues))
|
||||
} else if err := ValidateVolumes(result.Stdout); err != nil {
|
||||
add("configuration", StatusFailed, err.Error())
|
||||
} else {
|
||||
rendered = result.Stdout
|
||||
configReady = true
|
||||
add("configuration", StatusPassed, "Compose configuration and required volumes are valid")
|
||||
}
|
||||
|
||||
if err := filePermissions(installation); err != nil {
|
||||
add("files", StatusFailed, err.Error())
|
||||
} else {
|
||||
add("files", StatusPassed, "declared host files have safe permissions")
|
||||
}
|
||||
|
||||
status, statusAvailable := serviceStatus(ctx, installation, runner, secretValues, add)
|
||||
coreRunning := false
|
||||
if statusAvailable {
|
||||
var err error
|
||||
coreRunning, err = service.CoreRunning(status)
|
||||
if err != nil {
|
||||
coreRunning = false
|
||||
}
|
||||
}
|
||||
if !coreRunning {
|
||||
add("workspace-registry", StatusSkipped, "core is not running")
|
||||
add("workflow", StatusSkipped, "core is not running")
|
||||
add("pi", StatusSkipped, "core is not running")
|
||||
return finalize(report), nil
|
||||
}
|
||||
|
||||
if configReady && workspaceRegistryConfigured(rendered) {
|
||||
add("workspace-registry", StatusPassed, "workspace-registry volume is configured")
|
||||
} else {
|
||||
add("workspace-registry", StatusFailed, "workspace-registry volume is not configured")
|
||||
}
|
||||
workflowCheck(ctx, installation, runner, secretValues, add)
|
||||
piCheck(ctx, installation, runner, secretValues, add)
|
||||
return finalize(report), nil
|
||||
}
|
||||
|
||||
func finalize(report Report) Report {
|
||||
report.OK = len(report.Checks) > 0
|
||||
for _, check := range report.Checks {
|
||||
if check.Status != StatusPassed {
|
||||
report.OK = false
|
||||
break
|
||||
}
|
||||
}
|
||||
return report
|
||||
}
|
||||
|
||||
func commandCheck(ctx context.Context, runner Runner, args []string, secrets []string, add func(string, string, string), name, label string) bool {
|
||||
result, err := runner.Run(ctx, args, nil)
|
||||
if err != nil || strings.TrimSpace(result.Stdout) == "" {
|
||||
add(name, StatusFailed, commandDetail(label, result, err, secrets))
|
||||
return false
|
||||
}
|
||||
add(name, StatusPassed, strings.TrimSpace(result.Stdout))
|
||||
return true
|
||||
}
|
||||
|
||||
func serviceStatus(ctx context.Context, installation config.Installation, runner Runner, secrets []string, add func(string, string, string)) (string, bool) {
|
||||
result, err := runner.Run(ctx, installation.ComposeArgs("ps", "--all", "--format", "json"), nil)
|
||||
if err != nil {
|
||||
add("services", StatusFailed, commandDetail("Compose service status", result, err, secrets))
|
||||
return "", false
|
||||
}
|
||||
if err := service.Healthy(result.Stdout); err != nil {
|
||||
add("services", StatusFailed, err.Error())
|
||||
return result.Stdout, true
|
||||
}
|
||||
add("services", StatusPassed, "required services are running and reachable through Docker health checks")
|
||||
return result.Stdout, true
|
||||
}
|
||||
|
||||
func workflowCheck(ctx context.Context, installation config.Installation, runner Runner, secrets []string, add func(string, string, string)) {
|
||||
result, err := runner.Run(ctx, installation.ComposeArgs("exec", "-T", "core", "tht", "doctor", "--json"), nil)
|
||||
if err != nil {
|
||||
add("workflow", StatusFailed, commandDetail("container-local workflow doctor", result, err, secrets))
|
||||
return
|
||||
}
|
||||
var payload struct {
|
||||
OK bool `json:"ok"`
|
||||
}
|
||||
if json.Unmarshal([]byte(result.Stdout), &payload) != nil || !payload.OK {
|
||||
add("workflow", StatusFailed, "container-local workflow doctor returned an invalid or failing report")
|
||||
return
|
||||
}
|
||||
add("workflow", StatusPassed, "container-local workflow doctor passed")
|
||||
}
|
||||
|
||||
func piCheck(ctx context.Context, installation config.Installation, runner Runner, secrets []string, add func(string, string, string)) {
|
||||
controlled := compose.InstallationRunner{Installation: installation, Runner: runner}
|
||||
if err := pi.Doctor(ctx, controlled); err != nil {
|
||||
add("pi", StatusFailed, output.SanitizeDetail(err.Error(), secrets))
|
||||
return
|
||||
}
|
||||
add("pi", StatusPassed, "Pi doctor passed")
|
||||
}
|
||||
|
||||
func validateInstallation(installation config.Installation) error {
|
||||
if installation.Path == "" || installation.ProjectDirectory == "" || installation.EnvFile == "" {
|
||||
return errors.New("installation descriptor is incomplete")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func secretValues(installation config.Installation) []string {
|
||||
files, err := installation.SecretFiles()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
values, err := output.SecretValuesFromFiles(files)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func filePermissions(installation config.Installation) error {
|
||||
files, err := installation.SecretFiles()
|
||||
if err != nil {
|
||||
return errors.New("declared secret files could not be read")
|
||||
}
|
||||
for _, path := range append([]string{installation.EnvFile}, files...) {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil || !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("required host file is unavailable: %s", filepath.Base(path))
|
||||
}
|
||||
if info.Mode().Perm()&0o077 != 0 {
|
||||
return fmt.Errorf("required host file has unsafe permissions: %s", filepath.Base(path))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateVolumes checks the seven persistent volumes required by a ThothII installation.
|
||||
func ValidateVolumes(rendered string) error {
|
||||
var document struct {
|
||||
Volumes map[string]json.RawMessage `json:"volumes"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(rendered), &document); err != nil {
|
||||
return errors.New("Compose returned invalid rendered configuration")
|
||||
}
|
||||
for _, name := range []string{"settings", "pi-state", "workspace-registry", "workspace-secrets", "sessions", "qdrant-data", "embedding-models"} {
|
||||
if _, exists := document.Volumes[name]; !exists {
|
||||
return fmt.Errorf("rendered Compose configuration is missing required volume %s", name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func workspaceRegistryConfigured(rendered string) bool {
|
||||
var document struct {
|
||||
Volumes map[string]json.RawMessage `json:"volumes"`
|
||||
}
|
||||
return json.Unmarshal([]byte(rendered), &document) == nil && document.Volumes["workspace-registry"] != nil
|
||||
}
|
||||
|
||||
func commandDetail(label string, result compose.Result, err error, secrets []string) string {
|
||||
if result.ExitCode != 0 {
|
||||
return output.SanitizeDetail(fmt.Sprintf("%s failed (exit %d): %s", label, result.ExitCode, result.Stderr), secrets)
|
||||
}
|
||||
if err != nil {
|
||||
return output.SanitizeDetail(fmt.Sprintf("%s failed: %s", label, result.Stderr), secrets)
|
||||
}
|
||||
return output.SanitizeDetail(label+" returned no output", secrets)
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
package doctor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/compose"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
||||
)
|
||||
|
||||
// Catches treating an unavailable Docker executable as a successful diagnosis.
|
||||
func TestRunReportsUnavailableDockerWithoutReturningAnExecutionError(t *testing.T) {
|
||||
installation := doctorInstallation(t, "")
|
||||
runner := &doctorRunner{dockerUnavailable: true}
|
||||
|
||||
report, err := Run(context.Background(), installation, runner)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want report", err)
|
||||
}
|
||||
if report.OK || checkStatus(report, "docker") != "failed" {
|
||||
t.Fatalf("Run() report = %#v, want failed Docker check", report)
|
||||
}
|
||||
}
|
||||
|
||||
// Catches attempts to run in-container diagnostics when core is not running.
|
||||
func TestRunSkipsContainerDiagnosticsWhenCoreIsStopped(t *testing.T) {
|
||||
installation := doctorInstallation(t, "")
|
||||
runner := &doctorRunner{services: stoppedServices}
|
||||
|
||||
report, err := Run(context.Background(), installation, runner)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.OK || checkStatus(report, "workflow") != "skipped" || checkStatus(report, "pi") != "skipped" {
|
||||
t.Fatalf("Run() report = %#v, want stopped-core skips", report)
|
||||
}
|
||||
if strings.Contains(strings.Join(runner.calls, "\n"), " exec -T core ") {
|
||||
t.Fatalf("Run() invoked a container diagnostic while core was stopped: %v", runner.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// Catches host-Python diagnostics or omission of workflow/Pi checks once core is healthy.
|
||||
func TestRunUsesOnlyContainerLocalWorkflowAndPiDiagnosticsWhenCoreRuns(t *testing.T) {
|
||||
installation := doctorInstallation(t, "")
|
||||
runner := &doctorRunner{services: healthyServices}
|
||||
|
||||
report, err := Run(context.Background(), installation, runner)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !report.OK || checkStatus(report, "workflow") != "passed" || checkStatus(report, "pi") != "passed" {
|
||||
t.Fatalf("Run() report = %#v, want successful container diagnostics", report)
|
||||
}
|
||||
calls := strings.Join(runner.calls, "\n")
|
||||
if !strings.Contains(calls, "exec -T core tht doctor --json") {
|
||||
t.Fatalf("Run() calls = %s, want core-local workflow doctor", calls)
|
||||
}
|
||||
if strings.Contains(calls, ".venv") || strings.Contains(calls, "python") {
|
||||
t.Fatalf("Run() calls = %s, must not require host Python", calls)
|
||||
}
|
||||
}
|
||||
|
||||
// Catches a workflow failure leaking a credential from a declared secret file into a report.
|
||||
func TestRunRedactsWorkflowDiagnosticFailures(t *testing.T) {
|
||||
installation := doctorInstallation(t, "WORKFLOW_TOKEN_FILE=%s\n")
|
||||
secretPath := filepath.Join(filepath.Dir(installation.EnvFile), "workflow-token")
|
||||
if err := os.WriteFile(secretPath, []byte("workflow-secret-value"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
contents := "WORKFLOW_TOKEN_FILE=" + secretPath + "\n"
|
||||
if err := os.WriteFile(installation.EnvFile, []byte(contents), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runner := &doctorRunner{services: healthyServices, workflowFailure: "workflow-secret-value"}
|
||||
|
||||
report, err := Run(context.Background(), installation, runner)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.OK || checkStatus(report, "workflow") != "failed" {
|
||||
t.Fatalf("Run() report = %#v, want failed workflow check", report)
|
||||
}
|
||||
if strings.Contains(reportText(report), "workflow-secret-value") {
|
||||
t.Fatalf("Run() report exposed a secret: %#v", report)
|
||||
}
|
||||
}
|
||||
|
||||
func doctorInstallation(t *testing.T, _ string) config.Installation {
|
||||
t.Helper()
|
||||
base, err := filepath.EvalSymlinks(os.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
root, err := os.MkdirTemp(base, "tht-doctor-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = os.RemoveAll(root) })
|
||||
project := filepath.Join(root, "project")
|
||||
for _, path := range []string{project, filepath.Join(project, "deploy"), filepath.Join(project, "docker")} {
|
||||
if err := os.MkdirAll(path, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, path := range []string{filepath.Join(project, "compose.yaml"), filepath.Join(project, "deploy", "compose.local.yaml"), filepath.Join(project, "docker", "core.Dockerfile")} {
|
||||
if err := os.WriteFile(path, []byte("services: {}\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
envFile := filepath.Join(root, "operator.env")
|
||||
if err := os.WriteFile(envFile, []byte("SAFE_VALUE=1\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return config.Installation{Path: filepath.Join(root, "thothii-installation.yaml"), Profile: "local", ProjectDirectory: project, EnvFile: envFile}
|
||||
}
|
||||
|
||||
type doctorRunner struct {
|
||||
calls []string
|
||||
dockerUnavailable bool
|
||||
services string
|
||||
workflowFailure string
|
||||
}
|
||||
|
||||
func (r *doctorRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
|
||||
call := strings.Join(args, " ")
|
||||
r.calls = append(r.calls, call)
|
||||
if r.dockerUnavailable {
|
||||
return compose.Result{ExitCode: 127}, errors.New("docker unavailable")
|
||||
}
|
||||
switch {
|
||||
case strings.Contains(call, "version --format {{.Client.Version}}"):
|
||||
return compose.Result{Stdout: "26.0.0\n"}, nil
|
||||
case strings.Contains(call, "compose version --short"):
|
||||
return compose.Result{Stdout: "v2.30.0\n"}, nil
|
||||
case strings.Contains(call, "config --quiet"):
|
||||
return compose.Result{}, nil
|
||||
case strings.Contains(call, "config --format json"):
|
||||
return compose.Result{Stdout: renderedConfig}, nil
|
||||
case strings.Contains(call, "ps --all --format json"):
|
||||
if r.services == "" {
|
||||
return compose.Result{Stdout: healthyServices}, nil
|
||||
}
|
||||
return compose.Result{Stdout: r.services}, nil
|
||||
case strings.Contains(call, "tht doctor --json"):
|
||||
if r.workflowFailure != "" {
|
||||
return compose.Result{Stderr: r.workflowFailure, ExitCode: 23}, errors.New("workflow failed")
|
||||
}
|
||||
return compose.Result{Stdout: `{"ok":true,"checks":[]}`}, nil
|
||||
case strings.Contains(call, "pi --version") || strings.Contains(call, "PI_VERSION") || strings.Contains(call, "io.thothii.pi.version"):
|
||||
return compose.Result{Stdout: "0.80.3\n"}, nil
|
||||
case strings.Contains(call, "test -w /home/thoth/.pi") || strings.Contains(call, "test -r /home/thoth/.pi/agent/auth.json") || strings.Contains(call, "/health"):
|
||||
return compose.Result{Stdout: `{"ready":true}`}, nil
|
||||
case strings.Contains(call, "/pi-management/test"):
|
||||
return compose.Result{Stdout: `{"ready":true}`}, nil
|
||||
case strings.Contains(call, "ps -q core"):
|
||||
return compose.Result{Stdout: "core-id\n"}, nil
|
||||
}
|
||||
return compose.Result{}, nil
|
||||
}
|
||||
|
||||
func checkStatus(report Report, name string) string {
|
||||
for _, check := range report.Checks {
|
||||
if check.Name == name {
|
||||
return check.Status
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func reportText(report Report) string {
|
||||
parts := make([]string, 0, len(report.Checks))
|
||||
for _, check := range report.Checks {
|
||||
parts = append(parts, check.Name+" "+check.Status+" "+check.Detail)
|
||||
}
|
||||
return strings.Join(parts, "\n")
|
||||
}
|
||||
|
||||
const renderedConfig = `{"volumes":{"settings":{},"pi-state":{},"workspace-registry":{},"workspace-secrets":{},"sessions":{},"qdrant-data":{},"embedding-models":{}},"services":{"core":{"image":"thothii-core:local","environment":{"THT_LLM_URL":"https://llm.example.invalid"}}}}`
|
||||
|
||||
const healthyServices = `[
|
||||
{"Service":"core","State":"running","Health":"healthy"},
|
||||
{"Service":"frontend","State":"running","Health":"healthy"},
|
||||
{"Service":"qdrant","State":"running","Health":"healthy"},
|
||||
{"Service":"embedding","State":"running","Health":"healthy"},
|
||||
{"Service":"embedding-model-init","State":"exited","ExitCode":0}
|
||||
]`
|
||||
|
||||
const stoppedServices = `[
|
||||
{"Service":"core","State":"exited","Health":""},
|
||||
{"Service":"frontend","State":"running","Health":"healthy"}
|
||||
]`
|
||||
Reference in New Issue
Block a user