feat(cli): add version diagnostics and build-aware start
This commit is contained in:
+88
-130
@@ -16,11 +16,14 @@ import (
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/compose"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/doctor"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/output"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/pi"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/project"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/serverops"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/service"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/setup"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/version"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/workspaceops"
|
||||
)
|
||||
|
||||
@@ -32,10 +35,11 @@ descriptor in the current project tree.
|
||||
Commands:
|
||||
setup [--configure-only] [--installation-id ID] [--profile local|server]
|
||||
Create or validate the local non-secret installation configuration.
|
||||
version [--json] Show the host CLI build identity.
|
||||
status Show the Compose service state.
|
||||
doctor Validate Docker, Compose, rendered configuration, line endings, volumes, and health.
|
||||
doctor [--json] Run non-mutating host, Compose, workflow, and Pi diagnostics.
|
||||
logs Show the latest 200 sanitized service log lines (bounded; no follow mode).
|
||||
start Start the installation in the background.
|
||||
start [--build] Start the installation; --build builds current-checkout images first.
|
||||
stop Stop the installation.
|
||||
update --check-only Validate the current installation without changing containers.
|
||||
sessions migrate --yes
|
||||
@@ -93,6 +97,9 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||
if command == "setup" {
|
||||
return setupCommand(ctx, installationPath, commandArgs, stdout, stderr)
|
||||
}
|
||||
if command == "version" {
|
||||
return versionCommand(commandArgs, stdout, stderr)
|
||||
}
|
||||
return commandUsageError(stderr, fmt.Sprintf("unknown command %q", command))
|
||||
}
|
||||
workingDirectory, err := os.Getwd()
|
||||
@@ -136,10 +143,15 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||
}
|
||||
result, err = runner.Run(ctx, installation.ComposeArgs(logArgs...), nil)
|
||||
case "start":
|
||||
if len(commandArgs) != 0 {
|
||||
return commandUsageError(stderr, "start does not accept arguments")
|
||||
build, argumentError := startArgs(commandArgs)
|
||||
if argumentError != nil {
|
||||
return commandUsageError(stderr, argumentError.Error())
|
||||
}
|
||||
result, err = runner.Run(ctx, installation.ComposeArgs("up", "--detach", "--remove-orphans"), nil)
|
||||
if err := service.Start(ctx, installation, runner, build); err != nil {
|
||||
return lifecycleFailure(stderr, err, secretValues)
|
||||
}
|
||||
fmt.Fprintln(stdout, "ThothII services are healthy.")
|
||||
return 0
|
||||
case "stop":
|
||||
if len(commandArgs) != 0 {
|
||||
return commandUsageError(stderr, "stop does not accept arguments")
|
||||
@@ -151,10 +163,7 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||
}
|
||||
result, err = runner.Run(ctx, installation.ComposeArgs("config", "--quiet"), nil)
|
||||
case "doctor":
|
||||
if len(commandArgs) != 0 {
|
||||
return commandUsageError(stderr, "doctor does not accept arguments")
|
||||
}
|
||||
return doctor(ctx, installation, runner, secretValues, stdout, stderr)
|
||||
return doctorCommand(ctx, installation, runner, commandArgs, stdout, stderr)
|
||||
case "pi":
|
||||
return piCommand(ctx, installation, runner, commandArgs, secretValues, stdout, stderr)
|
||||
case "sessions":
|
||||
@@ -762,6 +771,76 @@ func commandUsageError(stderr io.Writer, message string) int {
|
||||
return 2
|
||||
}
|
||||
|
||||
func versionCommand(args []string, stdout, stderr io.Writer) int {
|
||||
jsonMode := false
|
||||
if len(args) == 1 && args[0] == "--json" {
|
||||
jsonMode = true
|
||||
} else if len(args) != 0 {
|
||||
return commandUsageError(stderr, "version accepts only --json")
|
||||
}
|
||||
info := version.Current()
|
||||
if jsonMode {
|
||||
encoder := json.NewEncoder(stdout)
|
||||
encoder.SetEscapeHTML(false)
|
||||
if err := encoder.Encode(info); err != nil {
|
||||
fmt.Fprintln(stderr, "tht: version output could not be written")
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
fmt.Fprintf(stdout, "tht %s\ncommit: %s\nbuilt: %s\nplatform: %s/%s\n", info.Version, info.Commit, info.BuildTime, info.OS, info.Arch)
|
||||
return 0
|
||||
}
|
||||
|
||||
func startArgs(args []string) (bool, error) {
|
||||
if len(args) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
if len(args) == 1 && args[0] == "--build" {
|
||||
return true, nil
|
||||
}
|
||||
return false, errors.New("start accepts only --build")
|
||||
}
|
||||
|
||||
func doctorCommand(ctx context.Context, installation config.Installation, runner compose.Runner, args []string, stdout, stderr io.Writer) int {
|
||||
jsonMode := false
|
||||
if len(args) == 1 && args[0] == "--json" {
|
||||
jsonMode = true
|
||||
} else if len(args) != 0 {
|
||||
return commandUsageError(stderr, "doctor accepts only --json")
|
||||
}
|
||||
report, err := doctor.Run(ctx, installation, runner)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "tht: %s\n", output.Sanitize(err.Error(), nil))
|
||||
return 1
|
||||
}
|
||||
if jsonMode {
|
||||
encoder := json.NewEncoder(stdout)
|
||||
encoder.SetEscapeHTML(false)
|
||||
if err := encoder.Encode(report); err != nil {
|
||||
fmt.Fprintln(stderr, "tht: doctor report could not be written")
|
||||
return 1
|
||||
}
|
||||
} else {
|
||||
for _, check := range report.Checks {
|
||||
fmt.Fprintf(stdout, "%s: %s", check.Name, check.Status)
|
||||
if check.Detail != "" {
|
||||
fmt.Fprintf(stdout, " - %s", check.Detail)
|
||||
}
|
||||
fmt.Fprintln(stdout)
|
||||
}
|
||||
}
|
||||
if report.OK {
|
||||
return 0
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
func lifecycleFailure(stderr io.Writer, err error, secretValues []string) int {
|
||||
fmt.Fprintf(stderr, "tht: %s\n", output.Sanitize(err.Error(), secretValues))
|
||||
return 1
|
||||
}
|
||||
|
||||
func writeResult(result compose.Result, err error, secretValues []string, stdout, stderr io.Writer) int {
|
||||
if result.Stdout != "" {
|
||||
fmt.Fprint(stdout, output.Sanitize(result.Stdout, secretValues))
|
||||
@@ -780,124 +859,3 @@ func writeResult(result compose.Result, err error, secretValues []string, stdout
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
func doctor(ctx context.Context, installation config.Installation, runner compose.Runner, secretValues []string, stdout, stderr io.Writer) int {
|
||||
checks := [][]string{
|
||||
{"version", "--format", "{{.Client.Version}}"},
|
||||
{"compose", "version", "--short"},
|
||||
installation.ComposeArgs("config", "--quiet"),
|
||||
installation.ComposeArgs("config", "--format", "json"),
|
||||
installation.ComposeArgs("ps", "--format", "json"),
|
||||
}
|
||||
var renderedConfig, status string
|
||||
for index, args := range checks {
|
||||
result, err := runner.Run(ctx, args, nil)
|
||||
if err != nil {
|
||||
return writeResult(result, err, secretValues, stdout, stderr)
|
||||
}
|
||||
if index == 3 {
|
||||
renderedConfig = result.Stdout
|
||||
}
|
||||
if index == 4 {
|
||||
status = result.Stdout
|
||||
}
|
||||
}
|
||||
if err := requireLF(installation.ProjectDirectory); err != nil {
|
||||
fmt.Fprintf(stderr, "tht: %s\n", err)
|
||||
return 1
|
||||
}
|
||||
if err := requireVolumes(renderedConfig); err != nil {
|
||||
fmt.Fprintf(stderr, "tht: %s\n", err)
|
||||
return 1
|
||||
}
|
||||
if err := requireHealthyServices(status); err != nil {
|
||||
fmt.Fprintf(stderr, "tht: %s\n", err)
|
||||
return 1
|
||||
}
|
||||
fmt.Fprintln(stdout, "Doctor checks passed.")
|
||||
return 0
|
||||
}
|
||||
|
||||
func requireLF(root string) error {
|
||||
return filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if entry.IsDir() || entry.Type()&os.ModeSymlink != 0 || !requiresLF(entry.Name()) {
|
||||
return nil
|
||||
}
|
||||
contents, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.Contains(string(contents), "\r\n") {
|
||||
return fmt.Errorf("CRLF line endings found in %s", filepath.Base(path))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func requiresLF(name string) bool {
|
||||
if name == "Dockerfile" || strings.HasPrefix(name, "Dockerfile.") || strings.HasSuffix(name, ".Dockerfile") {
|
||||
return true
|
||||
}
|
||||
for _, suffix := range []string{".sh", ".yml", ".yaml"} {
|
||||
if strings.HasSuffix(name, suffix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func requireVolumes(renderedConfig string) error {
|
||||
var document struct {
|
||||
Volumes map[string]json.RawMessage `json:"volumes"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(renderedConfig), &document); err != nil {
|
||||
return fmt.Errorf("Compose returned invalid rendered configuration")
|
||||
}
|
||||
if len(document.Volumes) == 0 {
|
||||
return errors.New("rendered Compose configuration declares no volumes")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func requireHealthyServices(status string) error {
|
||||
type serviceStatus struct {
|
||||
Service string `json:"Service"`
|
||||
State string `json:"State"`
|
||||
Health string `json:"Health"`
|
||||
}
|
||||
var services []serviceStatus
|
||||
if err := json.Unmarshal([]byte(status), &services); err != nil {
|
||||
decoder := json.NewDecoder(strings.NewReader(status))
|
||||
for {
|
||||
var service serviceStatus
|
||||
if err := decoder.Decode(&service); errors.Is(err, io.EOF) {
|
||||
break
|
||||
} else if err != nil {
|
||||
return errors.New("Compose returned invalid service status")
|
||||
}
|
||||
services = append(services, service)
|
||||
}
|
||||
if len(services) == 0 {
|
||||
return errors.New("Compose returned invalid service status")
|
||||
}
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, service := range services {
|
||||
if service.Service != "core" && service.Service != "frontend" {
|
||||
continue
|
||||
}
|
||||
if service.State != "running" || service.Health != "healthy" {
|
||||
return fmt.Errorf("%s is not healthy", service.Service)
|
||||
}
|
||||
seen[service.Service] = true
|
||||
}
|
||||
for _, service := range []string{"core", "frontend"} {
|
||||
if !seen[service] {
|
||||
return fmt.Errorf("%s service is not running", service)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Catches version regressing into a descriptor-dependent operator command.
|
||||
func TestVersionCommandIsDescriptorFreeAndMachineReadable(t *testing.T) {
|
||||
projectRoot := newProjectWithoutInstallation(t)
|
||||
previousDirectory, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chdir(filepath.Join(projectRoot, "frontend")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = os.Chdir(previousDirectory) })
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := run(context.Background(), []string{"version", "--json"}, &stdout, &stderr); code != 0 {
|
||||
t.Fatalf("run(version --json) = %d, stderr = %q", code, stderr.String())
|
||||
}
|
||||
var value struct {
|
||||
Version string `json:"version"`
|
||||
Commit string `json:"commit"`
|
||||
OS string `json:"os"`
|
||||
Arch string `json:"arch"`
|
||||
}
|
||||
decoder := json.NewDecoder(&stdout)
|
||||
if err := decoder.Decode(&value); err != nil || value.Version == "" || value.Commit == "" || value.OS == "" || value.Arch == "" {
|
||||
t.Fatalf("version JSON = %q, decoded %#v, err = %v", stdout.String(), value, err)
|
||||
}
|
||||
if err := decoder.Decode(&struct{}{}); err != io.EOF {
|
||||
t.Fatalf("version stdout contains more than one JSON document: %q", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Catches doctor --json writing a banner, progress, or Docker diagnostics to stdout.
|
||||
func TestDoctorJSONWritesOnlyOneReportDocument(t *testing.T) {
|
||||
fixture := newCLIFixture(t, "SAFE_VALUE=1\n")
|
||||
fixture.setEnvironment(t)
|
||||
fullHealth := `[{"Service":"core","State":"running","Health":"healthy"},{"Service":"frontend","State":"running","Health":"healthy"},{"Service":"qdrant","State":"running","Health":"healthy"},{"Service":"embedding","State":"running","Health":"healthy"},{"Service":"embedding-model-init","State":"exited","ExitCode":0}]`
|
||||
t.Setenv("THT_FAKE_PS", fullHealth)
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := run(context.Background(), []string{"--installation", fixture.installationPath, "doctor", "--json"}, &stdout, &stderr)
|
||||
if code != 0 && code != 1 {
|
||||
t.Fatalf("run(doctor --json) = %d, stderr = %q", code, stderr.String())
|
||||
}
|
||||
var report struct {
|
||||
OK *bool `json:"ok"`
|
||||
Checks []json.RawMessage `json:"checks"`
|
||||
}
|
||||
decoder := json.NewDecoder(&stdout)
|
||||
if err := decoder.Decode(&report); err != nil || report.OK == nil || report.Checks == nil {
|
||||
t.Fatalf("doctor JSON = %q, decoded %#v, err = %v", stdout.String(), report, err)
|
||||
}
|
||||
if err := decoder.Decode(&struct{}{}); err != io.EOF {
|
||||
t.Fatalf("doctor stdout contains more than one JSON document: %q", stdout.String())
|
||||
}
|
||||
if strings.Contains(stdout.String(), "Doctor checks passed") || strings.Contains(stdout.String(), "tht:") {
|
||||
t.Fatalf("doctor JSON stdout contains presentation text: %q", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Catches start --build being parsed as an argument but never reaching the shared lifecycle runner.
|
||||
func TestStartBuildOptionBuildsBeforeUpWhilePlainStartDoesNotBuild(t *testing.T) {
|
||||
fullHealth := `[{"Service":"core","State":"running","Health":"healthy"},{"Service":"frontend","State":"running","Health":"healthy"},{"Service":"qdrant","State":"running","Health":"healthy"},{"Service":"embedding","State":"running","Health":"healthy"},{"Service":"embedding-model-init","State":"exited","ExitCode":0}]`
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
args []string
|
||||
wantBuild bool
|
||||
}{
|
||||
{name: "plain", args: []string{"start"}, wantBuild: false},
|
||||
{name: "build", args: []string{"start", "--build"}, wantBuild: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
fixture := newCLIFixture(t, "SAFE_VALUE=1\n")
|
||||
fixture.setEnvironment(t)
|
||||
t.Setenv("THT_FAKE_PS", fullHealth)
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := run(context.Background(), append([]string{"--installation", fixture.installationPath}, test.args...), &stdout, &stderr); code != 0 {
|
||||
t.Fatalf("run(%v) = %d, stderr = %q", test.args, code, stderr.String())
|
||||
}
|
||||
calls := fixture.invocations(t)
|
||||
joined := make([]string, 0, len(calls))
|
||||
for _, call := range calls {
|
||||
joined = append(joined, strings.Join(call, " "))
|
||||
}
|
||||
sequence := strings.Join(joined, "\n")
|
||||
hasBuild := strings.Contains(sequence, " build")
|
||||
if hasBuild != test.wantBuild {
|
||||
t.Fatalf("start calls = %s, build present = %t, want %t", sequence, hasBuild, test.wantBuild)
|
||||
}
|
||||
if test.wantBuild && strings.Index(sequence, " build") > strings.Index(sequence, " up --detach --remove-orphans") {
|
||||
t.Fatalf("start calls = %s, want build before up", sequence)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -53,8 +53,8 @@ func TestRootCommandIdentity(t *testing.T) {
|
||||
{
|
||||
name: "version path",
|
||||
args: []string{"--installation", fixture.installationPath, "version"},
|
||||
wantCode: 2,
|
||||
wantText: `tht: unknown command "version"`,
|
||||
wantCode: 0,
|
||||
wantText: "tht 0.0.0-dev",
|
||||
},
|
||||
{
|
||||
name: "retired command is not an alias",
|
||||
@@ -1265,6 +1265,7 @@ case " $* " in
|
||||
exit "$THT_FAKE_MIGRATION_EXIT"
|
||||
fi
|
||||
printf '%s\n' '{"applied":[],"drifted":[],"pending":[]}' ;;
|
||||
*" ps --all --format json "*) printf '%s\n' "$THT_FAKE_PS" ;;
|
||||
*" ps --format json "*) printf '%s\n' "$THT_FAKE_PS" ;;
|
||||
*" ps -q core "*) printf '%s\n' 'core-id' ;;
|
||||
*" image inspect --format {{.Id}} "*) printf '%s\n' "${THT_FAKE_IMAGE_ID:-sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb}" ;;
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
// Package doctor aggregates non-mutating host and container diagnostics for one installation.
|
||||
package doctor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/compose"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/output"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/pi"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/service"
|
||||
)
|
||||
|
||||
const (
|
||||
StatusPassed = "passed"
|
||||
StatusFailed = "failed"
|
||||
StatusSkipped = "skipped"
|
||||
)
|
||||
|
||||
// Check is one named, redacted diagnostic outcome.
|
||||
type Check struct {
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
Detail string `json:"detail"`
|
||||
}
|
||||
|
||||
// Report is the typed, machine-readable diagnostic result.
|
||||
type Report struct {
|
||||
OK bool `json:"ok"`
|
||||
Checks []Check `json:"checks"`
|
||||
}
|
||||
|
||||
// Runner is the shell-free Docker boundary used for all host and in-container checks.
|
||||
type Runner interface {
|
||||
Run(context.Context, []string, io.Reader) (compose.Result, error)
|
||||
}
|
||||
|
||||
// Run performs diagnostics only. Expected environmental failures become failed checks so that
|
||||
// callers can always render a complete report; unexpected local read errors are also reported.
|
||||
func Run(ctx context.Context, installation config.Installation, runner Runner) (Report, error) {
|
||||
if runner == nil {
|
||||
return Report{}, errors.New("doctor requires a Docker command runner")
|
||||
}
|
||||
secretValues := secretValues(installation)
|
||||
report := Report{Checks: make([]Check, 0, 10)}
|
||||
add := func(name, status, detail string) {
|
||||
report.Checks = append(report.Checks, Check{Name: name, Status: status, Detail: output.SanitizeDetail(detail, secretValues)})
|
||||
}
|
||||
|
||||
if err := validateInstallation(installation); err != nil {
|
||||
add("descriptor", StatusFailed, err.Error())
|
||||
} else {
|
||||
add("descriptor", StatusPassed, "installation descriptor is loaded")
|
||||
}
|
||||
|
||||
if !commandCheck(ctx, runner, []string{"version", "--format", "{{.Client.Version}}"}, secretValues, add, "docker", "Docker Engine") {
|
||||
add("compose", StatusSkipped, "Docker Engine is unavailable")
|
||||
add("configuration", StatusSkipped, "Docker Engine is unavailable")
|
||||
add("files", StatusSkipped, "Docker Engine is unavailable")
|
||||
add("services", StatusSkipped, "Docker Engine is unavailable")
|
||||
add("workspace-registry", StatusSkipped, "core is unavailable")
|
||||
add("workflow", StatusSkipped, "core is unavailable")
|
||||
add("pi", StatusSkipped, "core is unavailable")
|
||||
return finalize(report), nil
|
||||
}
|
||||
if !commandCheck(ctx, runner, []string{"compose", "version", "--short"}, secretValues, add, "compose", "Docker Compose") {
|
||||
add("configuration", StatusSkipped, "Docker Compose is unavailable")
|
||||
add("files", StatusSkipped, "Docker Compose is unavailable")
|
||||
add("services", StatusSkipped, "Docker Compose is unavailable")
|
||||
add("workspace-registry", StatusSkipped, "core is unavailable")
|
||||
add("workflow", StatusSkipped, "core is unavailable")
|
||||
add("pi", StatusSkipped, "core is unavailable")
|
||||
return finalize(report), nil
|
||||
}
|
||||
|
||||
configReady := false
|
||||
rendered := ""
|
||||
if result, err := runner.Run(ctx, installation.ComposeArgs("config", "--quiet"), nil); err != nil {
|
||||
add("configuration", StatusFailed, commandDetail("Compose configuration", result, err, secretValues))
|
||||
} else if result, err := runner.Run(ctx, installation.ComposeArgs("config", "--format", "json"), nil); err != nil {
|
||||
add("configuration", StatusFailed, commandDetail("Compose rendering", result, err, secretValues))
|
||||
} else if err := ValidateVolumes(result.Stdout); err != nil {
|
||||
add("configuration", StatusFailed, err.Error())
|
||||
} else {
|
||||
rendered = result.Stdout
|
||||
configReady = true
|
||||
add("configuration", StatusPassed, "Compose configuration and required volumes are valid")
|
||||
}
|
||||
|
||||
if err := filePermissions(installation); err != nil {
|
||||
add("files", StatusFailed, err.Error())
|
||||
} else {
|
||||
add("files", StatusPassed, "declared host files have safe permissions")
|
||||
}
|
||||
|
||||
status, statusAvailable := serviceStatus(ctx, installation, runner, secretValues, add)
|
||||
coreRunning := false
|
||||
if statusAvailable {
|
||||
var err error
|
||||
coreRunning, err = service.CoreRunning(status)
|
||||
if err != nil {
|
||||
coreRunning = false
|
||||
}
|
||||
}
|
||||
if !coreRunning {
|
||||
add("workspace-registry", StatusSkipped, "core is not running")
|
||||
add("workflow", StatusSkipped, "core is not running")
|
||||
add("pi", StatusSkipped, "core is not running")
|
||||
return finalize(report), nil
|
||||
}
|
||||
|
||||
if configReady && workspaceRegistryConfigured(rendered) {
|
||||
add("workspace-registry", StatusPassed, "workspace-registry volume is configured")
|
||||
} else {
|
||||
add("workspace-registry", StatusFailed, "workspace-registry volume is not configured")
|
||||
}
|
||||
workflowCheck(ctx, installation, runner, secretValues, add)
|
||||
piCheck(ctx, installation, runner, secretValues, add)
|
||||
return finalize(report), nil
|
||||
}
|
||||
|
||||
func finalize(report Report) Report {
|
||||
report.OK = len(report.Checks) > 0
|
||||
for _, check := range report.Checks {
|
||||
if check.Status != StatusPassed {
|
||||
report.OK = false
|
||||
break
|
||||
}
|
||||
}
|
||||
return report
|
||||
}
|
||||
|
||||
func commandCheck(ctx context.Context, runner Runner, args []string, secrets []string, add func(string, string, string), name, label string) bool {
|
||||
result, err := runner.Run(ctx, args, nil)
|
||||
if err != nil || strings.TrimSpace(result.Stdout) == "" {
|
||||
add(name, StatusFailed, commandDetail(label, result, err, secrets))
|
||||
return false
|
||||
}
|
||||
add(name, StatusPassed, strings.TrimSpace(result.Stdout))
|
||||
return true
|
||||
}
|
||||
|
||||
func serviceStatus(ctx context.Context, installation config.Installation, runner Runner, secrets []string, add func(string, string, string)) (string, bool) {
|
||||
result, err := runner.Run(ctx, installation.ComposeArgs("ps", "--all", "--format", "json"), nil)
|
||||
if err != nil {
|
||||
add("services", StatusFailed, commandDetail("Compose service status", result, err, secrets))
|
||||
return "", false
|
||||
}
|
||||
if err := service.Healthy(result.Stdout); err != nil {
|
||||
add("services", StatusFailed, err.Error())
|
||||
return result.Stdout, true
|
||||
}
|
||||
add("services", StatusPassed, "required services are running and reachable through Docker health checks")
|
||||
return result.Stdout, true
|
||||
}
|
||||
|
||||
func workflowCheck(ctx context.Context, installation config.Installation, runner Runner, secrets []string, add func(string, string, string)) {
|
||||
result, err := runner.Run(ctx, installation.ComposeArgs("exec", "-T", "core", "tht", "doctor", "--json"), nil)
|
||||
if err != nil {
|
||||
add("workflow", StatusFailed, commandDetail("container-local workflow doctor", result, err, secrets))
|
||||
return
|
||||
}
|
||||
var payload struct {
|
||||
OK bool `json:"ok"`
|
||||
}
|
||||
if json.Unmarshal([]byte(result.Stdout), &payload) != nil || !payload.OK {
|
||||
add("workflow", StatusFailed, "container-local workflow doctor returned an invalid or failing report")
|
||||
return
|
||||
}
|
||||
add("workflow", StatusPassed, "container-local workflow doctor passed")
|
||||
}
|
||||
|
||||
func piCheck(ctx context.Context, installation config.Installation, runner Runner, secrets []string, add func(string, string, string)) {
|
||||
controlled := compose.InstallationRunner{Installation: installation, Runner: runner}
|
||||
if err := pi.Doctor(ctx, controlled); err != nil {
|
||||
add("pi", StatusFailed, output.SanitizeDetail(err.Error(), secrets))
|
||||
return
|
||||
}
|
||||
add("pi", StatusPassed, "Pi doctor passed")
|
||||
}
|
||||
|
||||
func validateInstallation(installation config.Installation) error {
|
||||
if installation.Path == "" || installation.ProjectDirectory == "" || installation.EnvFile == "" {
|
||||
return errors.New("installation descriptor is incomplete")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func secretValues(installation config.Installation) []string {
|
||||
files, err := installation.SecretFiles()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
values, err := output.SecretValuesFromFiles(files)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func filePermissions(installation config.Installation) error {
|
||||
files, err := installation.SecretFiles()
|
||||
if err != nil {
|
||||
return errors.New("declared secret files could not be read")
|
||||
}
|
||||
for _, path := range append([]string{installation.EnvFile}, files...) {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil || !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("required host file is unavailable: %s", filepath.Base(path))
|
||||
}
|
||||
if info.Mode().Perm()&0o077 != 0 {
|
||||
return fmt.Errorf("required host file has unsafe permissions: %s", filepath.Base(path))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateVolumes checks the seven persistent volumes required by a ThothII installation.
|
||||
func ValidateVolumes(rendered string) error {
|
||||
var document struct {
|
||||
Volumes map[string]json.RawMessage `json:"volumes"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(rendered), &document); err != nil {
|
||||
return errors.New("Compose returned invalid rendered configuration")
|
||||
}
|
||||
for _, name := range []string{"settings", "pi-state", "workspace-registry", "workspace-secrets", "sessions", "qdrant-data", "embedding-models"} {
|
||||
if _, exists := document.Volumes[name]; !exists {
|
||||
return fmt.Errorf("rendered Compose configuration is missing required volume %s", name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func workspaceRegistryConfigured(rendered string) bool {
|
||||
var document struct {
|
||||
Volumes map[string]json.RawMessage `json:"volumes"`
|
||||
}
|
||||
return json.Unmarshal([]byte(rendered), &document) == nil && document.Volumes["workspace-registry"] != nil
|
||||
}
|
||||
|
||||
func commandDetail(label string, result compose.Result, err error, secrets []string) string {
|
||||
if result.ExitCode != 0 {
|
||||
return output.SanitizeDetail(fmt.Sprintf("%s failed (exit %d): %s", label, result.ExitCode, result.Stderr), secrets)
|
||||
}
|
||||
if err != nil {
|
||||
return output.SanitizeDetail(fmt.Sprintf("%s failed: %s", label, result.Stderr), secrets)
|
||||
}
|
||||
return output.SanitizeDetail(label+" returned no output", secrets)
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
package doctor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/compose"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
||||
)
|
||||
|
||||
// Catches treating an unavailable Docker executable as a successful diagnosis.
|
||||
func TestRunReportsUnavailableDockerWithoutReturningAnExecutionError(t *testing.T) {
|
||||
installation := doctorInstallation(t, "")
|
||||
runner := &doctorRunner{dockerUnavailable: true}
|
||||
|
||||
report, err := Run(context.Background(), installation, runner)
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want report", err)
|
||||
}
|
||||
if report.OK || checkStatus(report, "docker") != "failed" {
|
||||
t.Fatalf("Run() report = %#v, want failed Docker check", report)
|
||||
}
|
||||
}
|
||||
|
||||
// Catches attempts to run in-container diagnostics when core is not running.
|
||||
func TestRunSkipsContainerDiagnosticsWhenCoreIsStopped(t *testing.T) {
|
||||
installation := doctorInstallation(t, "")
|
||||
runner := &doctorRunner{services: stoppedServices}
|
||||
|
||||
report, err := Run(context.Background(), installation, runner)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.OK || checkStatus(report, "workflow") != "skipped" || checkStatus(report, "pi") != "skipped" {
|
||||
t.Fatalf("Run() report = %#v, want stopped-core skips", report)
|
||||
}
|
||||
if strings.Contains(strings.Join(runner.calls, "\n"), " exec -T core ") {
|
||||
t.Fatalf("Run() invoked a container diagnostic while core was stopped: %v", runner.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// Catches host-Python diagnostics or omission of workflow/Pi checks once core is healthy.
|
||||
func TestRunUsesOnlyContainerLocalWorkflowAndPiDiagnosticsWhenCoreRuns(t *testing.T) {
|
||||
installation := doctorInstallation(t, "")
|
||||
runner := &doctorRunner{services: healthyServices}
|
||||
|
||||
report, err := Run(context.Background(), installation, runner)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !report.OK || checkStatus(report, "workflow") != "passed" || checkStatus(report, "pi") != "passed" {
|
||||
t.Fatalf("Run() report = %#v, want successful container diagnostics", report)
|
||||
}
|
||||
calls := strings.Join(runner.calls, "\n")
|
||||
if !strings.Contains(calls, "exec -T core tht doctor --json") {
|
||||
t.Fatalf("Run() calls = %s, want core-local workflow doctor", calls)
|
||||
}
|
||||
if strings.Contains(calls, ".venv") || strings.Contains(calls, "python") {
|
||||
t.Fatalf("Run() calls = %s, must not require host Python", calls)
|
||||
}
|
||||
}
|
||||
|
||||
// Catches a workflow failure leaking a credential from a declared secret file into a report.
|
||||
func TestRunRedactsWorkflowDiagnosticFailures(t *testing.T) {
|
||||
installation := doctorInstallation(t, "WORKFLOW_TOKEN_FILE=%s\n")
|
||||
secretPath := filepath.Join(filepath.Dir(installation.EnvFile), "workflow-token")
|
||||
if err := os.WriteFile(secretPath, []byte("workflow-secret-value"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
contents := "WORKFLOW_TOKEN_FILE=" + secretPath + "\n"
|
||||
if err := os.WriteFile(installation.EnvFile, []byte(contents), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runner := &doctorRunner{services: healthyServices, workflowFailure: "workflow-secret-value"}
|
||||
|
||||
report, err := Run(context.Background(), installation, runner)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.OK || checkStatus(report, "workflow") != "failed" {
|
||||
t.Fatalf("Run() report = %#v, want failed workflow check", report)
|
||||
}
|
||||
if strings.Contains(reportText(report), "workflow-secret-value") {
|
||||
t.Fatalf("Run() report exposed a secret: %#v", report)
|
||||
}
|
||||
}
|
||||
|
||||
func doctorInstallation(t *testing.T, _ string) config.Installation {
|
||||
t.Helper()
|
||||
base, err := filepath.EvalSymlinks(os.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
root, err := os.MkdirTemp(base, "tht-doctor-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = os.RemoveAll(root) })
|
||||
project := filepath.Join(root, "project")
|
||||
for _, path := range []string{project, filepath.Join(project, "deploy"), filepath.Join(project, "docker")} {
|
||||
if err := os.MkdirAll(path, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, path := range []string{filepath.Join(project, "compose.yaml"), filepath.Join(project, "deploy", "compose.local.yaml"), filepath.Join(project, "docker", "core.Dockerfile")} {
|
||||
if err := os.WriteFile(path, []byte("services: {}\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
envFile := filepath.Join(root, "operator.env")
|
||||
if err := os.WriteFile(envFile, []byte("SAFE_VALUE=1\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return config.Installation{Path: filepath.Join(root, "thothii-installation.yaml"), Profile: "local", ProjectDirectory: project, EnvFile: envFile}
|
||||
}
|
||||
|
||||
type doctorRunner struct {
|
||||
calls []string
|
||||
dockerUnavailable bool
|
||||
services string
|
||||
workflowFailure string
|
||||
}
|
||||
|
||||
func (r *doctorRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
|
||||
call := strings.Join(args, " ")
|
||||
r.calls = append(r.calls, call)
|
||||
if r.dockerUnavailable {
|
||||
return compose.Result{ExitCode: 127}, errors.New("docker unavailable")
|
||||
}
|
||||
switch {
|
||||
case strings.Contains(call, "version --format {{.Client.Version}}"):
|
||||
return compose.Result{Stdout: "26.0.0\n"}, nil
|
||||
case strings.Contains(call, "compose version --short"):
|
||||
return compose.Result{Stdout: "v2.30.0\n"}, nil
|
||||
case strings.Contains(call, "config --quiet"):
|
||||
return compose.Result{}, nil
|
||||
case strings.Contains(call, "config --format json"):
|
||||
return compose.Result{Stdout: renderedConfig}, nil
|
||||
case strings.Contains(call, "ps --all --format json"):
|
||||
if r.services == "" {
|
||||
return compose.Result{Stdout: healthyServices}, nil
|
||||
}
|
||||
return compose.Result{Stdout: r.services}, nil
|
||||
case strings.Contains(call, "tht doctor --json"):
|
||||
if r.workflowFailure != "" {
|
||||
return compose.Result{Stderr: r.workflowFailure, ExitCode: 23}, errors.New("workflow failed")
|
||||
}
|
||||
return compose.Result{Stdout: `{"ok":true,"checks":[]}`}, nil
|
||||
case strings.Contains(call, "pi --version") || strings.Contains(call, "PI_VERSION") || strings.Contains(call, "io.thothii.pi.version"):
|
||||
return compose.Result{Stdout: "0.80.3\n"}, nil
|
||||
case strings.Contains(call, "test -w /home/thoth/.pi") || strings.Contains(call, "test -r /home/thoth/.pi/agent/auth.json") || strings.Contains(call, "/health"):
|
||||
return compose.Result{Stdout: `{"ready":true}`}, nil
|
||||
case strings.Contains(call, "/pi-management/test"):
|
||||
return compose.Result{Stdout: `{"ready":true}`}, nil
|
||||
case strings.Contains(call, "ps -q core"):
|
||||
return compose.Result{Stdout: "core-id\n"}, nil
|
||||
}
|
||||
return compose.Result{}, nil
|
||||
}
|
||||
|
||||
func checkStatus(report Report, name string) string {
|
||||
for _, check := range report.Checks {
|
||||
if check.Name == name {
|
||||
return check.Status
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func reportText(report Report) string {
|
||||
parts := make([]string, 0, len(report.Checks))
|
||||
for _, check := range report.Checks {
|
||||
parts = append(parts, check.Name+" "+check.Status+" "+check.Detail)
|
||||
}
|
||||
return strings.Join(parts, "\n")
|
||||
}
|
||||
|
||||
const renderedConfig = `{"volumes":{"settings":{},"pi-state":{},"workspace-registry":{},"workspace-secrets":{},"sessions":{},"qdrant-data":{},"embedding-models":{}},"services":{"core":{"image":"thothii-core:local","environment":{"THT_LLM_URL":"https://llm.example.invalid"}}}}`
|
||||
|
||||
const healthyServices = `[
|
||||
{"Service":"core","State":"running","Health":"healthy"},
|
||||
{"Service":"frontend","State":"running","Health":"healthy"},
|
||||
{"Service":"qdrant","State":"running","Health":"healthy"},
|
||||
{"Service":"embedding","State":"running","Health":"healthy"},
|
||||
{"Service":"embedding-model-init","State":"exited","ExitCode":0}
|
||||
]`
|
||||
|
||||
const stoppedServices = `[
|
||||
{"Service":"core","State":"exited","Health":""},
|
||||
{"Service":"frontend","State":"running","Health":"healthy"}
|
||||
]`
|
||||
@@ -0,0 +1,184 @@
|
||||
// Package service contains the shared non-mutating health and lifecycle checks for Compose services.
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/compose"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
||||
)
|
||||
|
||||
const healthTimeout = 5 * time.Minute
|
||||
|
||||
var healthPollInterval = time.Second
|
||||
|
||||
// HealthFailure identifies the last non-ready service after a bounded health wait.
|
||||
type HealthFailure struct {
|
||||
Service string
|
||||
State string
|
||||
}
|
||||
|
||||
func (e HealthFailure) Error() string {
|
||||
return fmt.Sprintf("service health timed out waiting for %s (last state: %s)", e.Service, e.State)
|
||||
}
|
||||
|
||||
type status struct {
|
||||
Service string `json:"Service"`
|
||||
State string `json:"State"`
|
||||
Health string `json:"Health"`
|
||||
ExitCode json.RawMessage `json:"ExitCode"`
|
||||
}
|
||||
|
||||
// Start optionally builds the configured project, starts it in the background, then waits for
|
||||
// every required service to become healthy. It does not inspect or modify host state directly.
|
||||
func Start(ctx context.Context, installation config.Installation, runner compose.Runner, build bool) error {
|
||||
if runner == nil {
|
||||
return errors.New("start requires a Docker command runner")
|
||||
}
|
||||
if build {
|
||||
if err := runCompose(ctx, installation, runner, "build"); err != nil {
|
||||
return fmt.Errorf("image build: %w", err)
|
||||
}
|
||||
}
|
||||
if err := runCompose(ctx, installation, runner, "up", "--detach", "--remove-orphans"); err != nil {
|
||||
return fmt.Errorf("stack start: %w", err)
|
||||
}
|
||||
return WaitForHealthy(ctx, installation, runner)
|
||||
}
|
||||
|
||||
// WaitForHealthy waits for the complete ThothII Compose service set.
|
||||
func WaitForHealthy(ctx context.Context, installation config.Installation, runner compose.Runner) error {
|
||||
healthContext, cancel := context.WithTimeout(ctx, healthTimeout)
|
||||
defer cancel()
|
||||
lastService, lastState := "core", "unknown"
|
||||
for {
|
||||
result, err := runner.Run(healthContext, installation.ComposeArgs("ps", "--all", "--format", "json"), nil)
|
||||
if err == nil {
|
||||
statuses, parseErr := parseStatuses(result.Stdout)
|
||||
if parseErr == nil {
|
||||
if service, state, ready := healthy(statuses); ready {
|
||||
return nil
|
||||
} else {
|
||||
lastService, lastState = service, state
|
||||
}
|
||||
} else {
|
||||
lastState = "Compose returned invalid service status"
|
||||
}
|
||||
} else if result.ExitCode != 0 {
|
||||
lastState = fmt.Sprintf("Compose exited with status %d", result.ExitCode)
|
||||
} else {
|
||||
lastState = "Compose status command failed"
|
||||
}
|
||||
select {
|
||||
case <-healthContext.Done():
|
||||
return HealthFailure{Service: lastService, State: lastState}
|
||||
case <-time.After(healthPollInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// CoreRunning reports whether Compose currently identifies core as a running container.
|
||||
func CoreRunning(value string) (bool, error) {
|
||||
statuses, err := parseStatuses(value)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, item := range statuses {
|
||||
if item.Service == "core" {
|
||||
return strings.EqualFold(item.State, "running"), nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Healthy verifies the full expected Compose service set, including the one-shot model initializer.
|
||||
func Healthy(value string) error {
|
||||
statuses, err := parseStatuses(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
service, state, ready := healthy(statuses)
|
||||
if ready {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s is not healthy (%s)", service, state)
|
||||
}
|
||||
|
||||
func runCompose(ctx context.Context, installation config.Installation, runner compose.Runner, command ...string) error {
|
||||
result, err := runner.Run(ctx, installation.ComposeArgs(command...), nil)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if result.ExitCode != 0 {
|
||||
return fmt.Errorf("Docker exited with status %d", result.ExitCode)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func parseStatuses(value string) ([]status, error) {
|
||||
var statuses []status
|
||||
if err := json.Unmarshal([]byte(value), &statuses); err == nil && len(statuses) > 0 {
|
||||
return statuses, nil
|
||||
}
|
||||
decoder := json.NewDecoder(strings.NewReader(value))
|
||||
for {
|
||||
var item status
|
||||
err := decoder.Decode(&item)
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, errors.New("Compose returned invalid service status")
|
||||
}
|
||||
statuses = append(statuses, item)
|
||||
}
|
||||
if len(statuses) == 0 {
|
||||
return nil, errors.New("Compose returned invalid service status")
|
||||
}
|
||||
return statuses, nil
|
||||
}
|
||||
|
||||
func healthy(statuses []status) (string, string, bool) {
|
||||
wanted := map[string]bool{"core": false, "frontend": false, "qdrant": false, "embedding": false, "embedding-model-init": false}
|
||||
for _, item := range statuses {
|
||||
if _, required := wanted[item.Service]; !required {
|
||||
continue
|
||||
}
|
||||
if item.Service == "embedding-model-init" {
|
||||
if strings.EqualFold(item.State, "exited") && exitCodeZero(item.ExitCode) {
|
||||
wanted[item.Service] = true
|
||||
continue
|
||||
}
|
||||
return item.Service, item.State, false
|
||||
}
|
||||
if strings.EqualFold(item.State, "running") && strings.EqualFold(item.Health, "healthy") {
|
||||
wanted[item.Service] = true
|
||||
continue
|
||||
}
|
||||
return item.Service, strings.TrimSpace(item.State + "/" + item.Health), false
|
||||
}
|
||||
for _, name := range []string{"core", "frontend", "qdrant", "embedding", "embedding-model-init"} {
|
||||
if !wanted[name] {
|
||||
return name, "not reported by Docker Compose", false
|
||||
}
|
||||
}
|
||||
return "", "", true
|
||||
}
|
||||
|
||||
func exitCodeZero(value json.RawMessage) bool {
|
||||
if len(value) == 0 || string(value) == "null" {
|
||||
return false
|
||||
}
|
||||
var number int
|
||||
if json.Unmarshal(value, &number) == nil {
|
||||
return number == 0
|
||||
}
|
||||
var text string
|
||||
return json.Unmarshal(value, &text) == nil && strings.TrimSpace(text) == "0"
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/compose"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
||||
)
|
||||
|
||||
// Catches --build being ignored, which would run stale images after a checkout update.
|
||||
func TestStartBuildsBeforeStartingAndCheckingHealth(t *testing.T) {
|
||||
runner := &recordingRunner{}
|
||||
installation := testInstallation()
|
||||
|
||||
if err := Start(context.Background(), installation, runner, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := strings.Join(runner.stages, " -> "), "build -> up -> health"; got != want {
|
||||
t.Fatalf("Start(..., build=true) stages = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Catches a normal start unnecessarily rebuilding images.
|
||||
func TestStartSkipsBuildUnlessRequested(t *testing.T) {
|
||||
runner := &recordingRunner{}
|
||||
installation := testInstallation()
|
||||
|
||||
if err := Start(context.Background(), installation, runner, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := strings.Join(runner.stages, " -> "), "up -> health"; got != want {
|
||||
t.Fatalf("Start(..., build=false) stages = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
type recordingRunner struct{ stages []string }
|
||||
|
||||
func (r *recordingRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
|
||||
joined := strings.Join(args, " ")
|
||||
switch {
|
||||
case strings.HasSuffix(joined, " build"):
|
||||
r.stages = append(r.stages, "build")
|
||||
case strings.HasSuffix(joined, " up --detach --remove-orphans"):
|
||||
r.stages = append(r.stages, "up")
|
||||
case strings.HasSuffix(joined, " ps --all --format json"):
|
||||
r.stages = append(r.stages, "health")
|
||||
return compose.Result{Stdout: healthyServices}, nil
|
||||
}
|
||||
return compose.Result{}, nil
|
||||
}
|
||||
|
||||
func testInstallation() config.Installation {
|
||||
return config.Installation{
|
||||
Path: "/tmp/thothii/deploy/local/thothii-installation.yaml",
|
||||
Profile: "local",
|
||||
ProjectDirectory: "/tmp/thothii",
|
||||
EnvFile: "/tmp/thothii/deploy/local/operator.env",
|
||||
}
|
||||
}
|
||||
|
||||
const healthyServices = `[
|
||||
{"Service":"core","State":"running","Health":"healthy"},
|
||||
{"Service":"frontend","State":"running","Health":"healthy"},
|
||||
{"Service":"qdrant","State":"running","Health":"healthy"},
|
||||
{"Service":"embedding","State":"running","Health":"healthy"},
|
||||
{"Service":"embedding-model-init","State":"exited","ExitCode":0}
|
||||
]`
|
||||
+15
-159
@@ -3,7 +3,6 @@ package setup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -11,18 +10,14 @@ import (
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/compose"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/pi"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/doctor"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/project"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/service"
|
||||
)
|
||||
|
||||
const setupHealthTimeout = 5 * time.Minute
|
||||
|
||||
var healthPollInterval = time.Second
|
||||
|
||||
// Result records the completed setup phases. DescriptorPath always identifies the descriptor
|
||||
// selected by this invocation, including an idempotent rerun.
|
||||
type Result struct {
|
||||
@@ -65,24 +60,19 @@ func Run(ctx context.Context, runner compose.Runner, request Request, input io.R
|
||||
fmt.Fprintf(output, "Configuration is ready: %s\n", result.DescriptorPath)
|
||||
return result, nil
|
||||
}
|
||||
if err := runCompose(ctx, runner, installation, "build"); err != nil {
|
||||
return Result{}, fmt.Errorf("setup image build: %w", err)
|
||||
if err := service.Start(ctx, installation, runner, true); err != nil {
|
||||
if strings.Contains(err.Error(), "image build") {
|
||||
return Result{}, fmt.Errorf("setup %w", err)
|
||||
}
|
||||
result.Built = true
|
||||
if err := runCompose(ctx, runner, installation, "up", "--detach", "--remove-orphans"); err != nil {
|
||||
return Result{}, withStartupRecovery(fmt.Errorf("setup stack start: %w", err), "core")
|
||||
return Result{}, withStartupRecovery(fmt.Errorf("setup %w", err), recoveryService(err))
|
||||
}
|
||||
result.Started = true
|
||||
if err := waitForHealthyServices(ctx, runner, installation); err != nil {
|
||||
return Result{}, withStartupRecovery(err, recoveryService(err))
|
||||
result.Built, result.Started, result.Healthy = true, true, true
|
||||
report, err := doctor.Run(ctx, installation, runner)
|
||||
if err != nil {
|
||||
return Result{}, withStartupRecovery(fmt.Errorf("setup doctor: %w", err), "core")
|
||||
}
|
||||
result.Healthy = true
|
||||
if err := aggregateDoctor(ctx, runner, installation); err != nil {
|
||||
return Result{}, withStartupRecovery(err, "core")
|
||||
}
|
||||
controlled := compose.InstallationRunner{Installation: installation, Runner: runner}
|
||||
if err := pi.Doctor(ctx, controlled); err != nil {
|
||||
return Result{}, withStartupRecovery(fmt.Errorf("setup Pi doctor: %w", err), "core")
|
||||
if !report.OK {
|
||||
return Result{}, withStartupRecovery(errors.New("setup doctor reported failed checks"), "core")
|
||||
}
|
||||
fmt.Fprintf(output, "ThothII is ready at %s\nInstallation descriptor: %s\nNext: tht status\n", frontendURL(installation), result.DescriptorPath)
|
||||
return result, nil
|
||||
@@ -136,17 +126,6 @@ func composeFailure(result compose.Result, cause error) error {
|
||||
return cause
|
||||
}
|
||||
|
||||
func aggregateDoctor(ctx context.Context, runner compose.Runner, installation config.Installation) error {
|
||||
result, err := runner.Run(ctx, installation.ComposeArgs("config", "--format", "json"), nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("setup doctor: %w", composeFailure(result, err))
|
||||
}
|
||||
if err := requireVolumes(result.Stdout); err != nil {
|
||||
return fmt.Errorf("setup doctor: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func withStartupRecovery(cause error, service string) error {
|
||||
if service == "" {
|
||||
service = "core"
|
||||
@@ -155,121 +134,13 @@ func withStartupRecovery(cause error, service string) error {
|
||||
}
|
||||
|
||||
func recoveryService(cause error) string {
|
||||
var healthFailure healthFailure
|
||||
if errors.As(cause, &healthFailure) && healthFailure.service != "" {
|
||||
return healthFailure.service
|
||||
var healthFailure service.HealthFailure
|
||||
if errors.As(cause, &healthFailure) && healthFailure.Service != "" {
|
||||
return healthFailure.Service
|
||||
}
|
||||
return "core"
|
||||
}
|
||||
|
||||
type serviceStatus struct {
|
||||
Service string `json:"Service"`
|
||||
State string `json:"State"`
|
||||
Health string `json:"Health"`
|
||||
ExitCode json.RawMessage `json:"ExitCode"`
|
||||
}
|
||||
|
||||
type healthFailure struct {
|
||||
service string
|
||||
state string
|
||||
}
|
||||
|
||||
func (e healthFailure) Error() string {
|
||||
return fmt.Sprintf("setup health timed out waiting for %s (last state: %s)", e.service, e.state)
|
||||
}
|
||||
|
||||
func waitForHealthyServices(ctx context.Context, runner compose.Runner, installation config.Installation) error {
|
||||
healthContext, cancel := context.WithTimeout(ctx, setupHealthTimeout)
|
||||
defer cancel()
|
||||
lastService, lastState := "core", "unknown"
|
||||
for {
|
||||
result, err := runner.Run(healthContext, installation.ComposeArgs("ps", "--all", "--format", "json"), nil)
|
||||
if err == nil {
|
||||
statuses, parseErr := parseServiceStatuses(result.Stdout)
|
||||
if parseErr == nil {
|
||||
if service, state, ready := healthyServices(statuses); ready {
|
||||
return nil
|
||||
} else {
|
||||
lastService, lastState = service, state
|
||||
}
|
||||
} else {
|
||||
lastState = "Compose returned invalid status"
|
||||
}
|
||||
} else if result.ExitCode != 0 {
|
||||
lastState = fmt.Sprintf("Compose exited with status %d", result.ExitCode)
|
||||
} else {
|
||||
lastState = "Compose status command failed"
|
||||
}
|
||||
select {
|
||||
case <-healthContext.Done():
|
||||
return healthFailure{service: lastService, state: lastState}
|
||||
case <-time.After(healthPollInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseServiceStatuses(value string) ([]serviceStatus, error) {
|
||||
var statuses []serviceStatus
|
||||
if err := json.Unmarshal([]byte(value), &statuses); err == nil && len(statuses) > 0 {
|
||||
return statuses, nil
|
||||
}
|
||||
decoder := json.NewDecoder(strings.NewReader(value))
|
||||
for {
|
||||
var status serviceStatus
|
||||
err := decoder.Decode(&status)
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, errors.New("Compose returned invalid service status")
|
||||
}
|
||||
statuses = append(statuses, status)
|
||||
}
|
||||
if len(statuses) == 0 {
|
||||
return nil, errors.New("Compose returned invalid service status")
|
||||
}
|
||||
return statuses, nil
|
||||
}
|
||||
|
||||
func healthyServices(statuses []serviceStatus) (string, string, bool) {
|
||||
wanted := map[string]bool{"core": false, "frontend": false, "qdrant": false, "embedding": false, "embedding-model-init": false}
|
||||
for _, status := range statuses {
|
||||
if _, required := wanted[status.Service]; !required {
|
||||
continue
|
||||
}
|
||||
if status.Service == "embedding-model-init" {
|
||||
if strings.EqualFold(status.State, "exited") && exitCodeZero(status.ExitCode) {
|
||||
wanted[status.Service] = true
|
||||
continue
|
||||
}
|
||||
return status.Service, status.State, false
|
||||
}
|
||||
if strings.EqualFold(status.State, "running") && strings.EqualFold(status.Health, "healthy") {
|
||||
wanted[status.Service] = true
|
||||
continue
|
||||
}
|
||||
return status.Service, strings.TrimSpace(status.State + "/" + status.Health), false
|
||||
}
|
||||
for _, service := range []string{"core", "frontend", "qdrant", "embedding", "embedding-model-init"} {
|
||||
if !wanted[service] {
|
||||
return service, "not reported by Docker Compose", false
|
||||
}
|
||||
}
|
||||
return "", "", true
|
||||
}
|
||||
|
||||
func exitCodeZero(value json.RawMessage) bool {
|
||||
if len(value) == 0 || string(value) == "null" {
|
||||
return false
|
||||
}
|
||||
var number int
|
||||
if json.Unmarshal(value, &number) == nil {
|
||||
return number == 0
|
||||
}
|
||||
var text string
|
||||
return json.Unmarshal(value, &text) == nil && strings.TrimSpace(text) == "0"
|
||||
}
|
||||
|
||||
func frontendURL(installation config.Installation) string {
|
||||
port, err := installation.EnvironmentValue("THOTH_HTTP_PORT")
|
||||
if err != nil || strings.TrimSpace(port) == "" {
|
||||
@@ -281,21 +152,6 @@ func frontendURL(installation config.Installation) string {
|
||||
return "http://127.0.0.1:" + port
|
||||
}
|
||||
|
||||
func requireVolumes(renderedConfig string) error {
|
||||
var document struct {
|
||||
Volumes map[string]json.RawMessage `json:"volumes"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(renderedConfig), &document); err != nil {
|
||||
return errors.New("Compose returned invalid rendered configuration")
|
||||
}
|
||||
for _, volume := range []string{"settings", "pi-state", "workspace-registry", "workspace-secrets", "sessions", "qdrant-data", "embedding-models"} {
|
||||
if _, exists := document.Volumes[volume]; !exists {
|
||||
return fmt.Errorf("rendered Compose configuration is missing required volume %s", volume)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func requireLF(root string) error {
|
||||
for _, path := range []string{filepath.Join(root, "compose.yaml"), filepath.Join(root, "deploy"), filepath.Join(root, "docker")} {
|
||||
if err := requireLFPath(path); err != nil {
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/compose"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/doctor"
|
||||
)
|
||||
|
||||
func TestRunBuildsStartsAndVerifiesInOrder(t *testing.T) {
|
||||
@@ -35,7 +36,7 @@ func TestRunBuildsStartsAndVerifiesInOrder(t *testing.T) {
|
||||
|
||||
want := []string{
|
||||
"docker engine", "docker compose", "architecture", "compose config", "compose build",
|
||||
"compose up", "health", "health", "doctor", "pi doctor",
|
||||
"compose up", "health", "health", "doctor docker", "doctor compose", "compose config", "doctor config", "health", "workflow doctor", "pi doctor",
|
||||
}
|
||||
if got := collapseStages(runner.stages); strings.Join(got, " | ") != strings.Join(want, " | ") {
|
||||
t.Fatalf("runner stages = %v, want %v", got, want)
|
||||
@@ -119,11 +120,11 @@ func TestRunUpFailurePreservesCauseAndOffersRecovery(t *testing.T) {
|
||||
|
||||
func TestRunAggregateDoctorFailurePreservesCauseAndOffersRecovery(t *testing.T) {
|
||||
_, request := setupRunFixture(t, false)
|
||||
runner := &setupRunner{failureAt: "doctor"}
|
||||
runner := &setupRunner{failureAt: "doctor config"}
|
||||
|
||||
_, err := Run(context.Background(), runner, request, strings.NewReader(""), io.Discard)
|
||||
assertRecoveryFailure(t, err, "setup doctor", "core")
|
||||
assertSetupStages(t, runner, "docker engine", "docker compose", "architecture", "compose config", "compose build", "compose up", "health", "doctor")
|
||||
assertRecoveryFailure(t, err, "setup doctor reported failed checks", "core")
|
||||
assertSetupStages(t, runner, "docker engine", "docker compose", "architecture", "compose config", "compose build", "compose up", "health", "doctor docker", "doctor compose", "compose config", "doctor config", "health", "workflow doctor", "pi doctor")
|
||||
}
|
||||
|
||||
func TestRunPiDoctorFailurePreservesCauseAndOffersRecovery(t *testing.T) {
|
||||
@@ -131,8 +132,8 @@ func TestRunPiDoctorFailurePreservesCauseAndOffersRecovery(t *testing.T) {
|
||||
runner := &setupRunner{failureAt: "pi doctor"}
|
||||
|
||||
_, err := Run(context.Background(), runner, request, strings.NewReader(""), io.Discard)
|
||||
assertRecoveryFailure(t, err, "setup Pi doctor", "core")
|
||||
assertSetupStages(t, runner, "docker engine", "docker compose", "architecture", "compose config", "compose build", "compose up", "health", "doctor", "pi doctor")
|
||||
assertRecoveryFailure(t, err, "setup doctor reported failed checks", "core")
|
||||
assertSetupStages(t, runner, "docker engine", "docker compose", "architecture", "compose config", "compose build", "compose up", "health", "doctor docker", "doctor compose", "compose config", "doctor config", "health", "workflow doctor", "pi doctor")
|
||||
}
|
||||
|
||||
func TestRequireVolumesRequiresEveryInstallationVolume(t *testing.T) {
|
||||
@@ -145,16 +146,16 @@ func TestRequireVolumesRequiresEveryInstallationVolume(t *testing.T) {
|
||||
volumes = append(volumes, name)
|
||||
}
|
||||
}
|
||||
if err := requireVolumes(renderedConfigForVolumes(volumes...)); err == nil || !strings.Contains(err.Error(), missing) {
|
||||
t.Fatalf("requireVolumes() error = %v, want missing %q", err, missing)
|
||||
if err := doctor.ValidateVolumes(renderedConfigForVolumes(volumes...)); err == nil || !strings.Contains(err.Error(), missing) {
|
||||
t.Fatalf("ValidateVolumes() error = %v, want missing %q", err, missing)
|
||||
}
|
||||
})
|
||||
}
|
||||
if err := requireVolumes(renderedConfigForVolumes("unrelated")); err == nil {
|
||||
t.Fatal("requireVolumes() error = nil, want required-volume failure for unrelated-only configuration")
|
||||
if err := doctor.ValidateVolumes(renderedConfigForVolumes("unrelated")); err == nil {
|
||||
t.Fatal("ValidateVolumes() error = nil, want required-volume failure for unrelated-only configuration")
|
||||
}
|
||||
if err := requireVolumes(renderedConfigForVolumes(all...)); err != nil {
|
||||
t.Fatalf("requireVolumes() error = %v, want complete installation volume set", err)
|
||||
if err := doctor.ValidateVolumes(renderedConfigForVolumes(all...)); err != nil {
|
||||
t.Fatalf("ValidateVolumes() error = %v, want complete installation volume set", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -216,7 +217,10 @@ type setupRunner struct {
|
||||
|
||||
func (r *setupRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
|
||||
stage, result := setupStage(args)
|
||||
if stage == "doctor" && containsStage(r.stages, "doctor") {
|
||||
if stage == "docker compose" && containsStage(r.stages, "docker compose") {
|
||||
stage = "doctor compose"
|
||||
}
|
||||
if stage == "doctor config" && containsStage(r.stages, "workflow doctor") {
|
||||
stage = "pi doctor"
|
||||
}
|
||||
r.stages = append(r.stages, stage)
|
||||
@@ -258,6 +262,8 @@ func setupStage(args []string) (string, compose.Result) {
|
||||
switch {
|
||||
case joined == "version --format {{.Server.Version}}":
|
||||
return "docker engine", compose.Result{Stdout: "26.0.0\n"}
|
||||
case joined == "version --format {{.Client.Version}}":
|
||||
return "doctor docker", compose.Result{Stdout: "26.0.0\n"}
|
||||
case joined == "compose version --short":
|
||||
return "docker compose", compose.Result{Stdout: "v2.30.0\n"}
|
||||
case joined == "version --format {{.Server.Arch}}":
|
||||
@@ -271,7 +277,9 @@ func setupStage(args []string) (string, compose.Result) {
|
||||
case strings.HasSuffix(joined, " ps --all --format json"):
|
||||
return "health", compose.Result{}
|
||||
case strings.HasSuffix(joined, " config --format json"):
|
||||
return "doctor", compose.Result{Stdout: renderedSetupConfig}
|
||||
return "doctor config", compose.Result{Stdout: renderedSetupConfig}
|
||||
case strings.Contains(joined, "exec -T core tht doctor --json"):
|
||||
return "workflow doctor", compose.Result{Stdout: `{"ok":true,"components":{}}`}
|
||||
case strings.Contains(joined, " ps -q core"):
|
||||
return "pi doctor", compose.Result{Stdout: "core-id\n"}
|
||||
case strings.HasPrefix(joined, "inspect --format"):
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
// Package version exposes host CLI build identity without requiring an installation descriptor.
|
||||
package version
|
||||
|
||||
import "runtime"
|
||||
|
||||
var (
|
||||
semanticVersion = "0.0.0-dev"
|
||||
commit = "unknown"
|
||||
buildTime = "unknown"
|
||||
)
|
||||
|
||||
// Info is the stable, machine-readable identity of the host-side CLI.
|
||||
type Info struct {
|
||||
Version string `json:"version"`
|
||||
Commit string `json:"commit"`
|
||||
BuildTime string `json:"buildTime"`
|
||||
OS string `json:"os"`
|
||||
Arch string `json:"arch"`
|
||||
}
|
||||
|
||||
// Current returns linker-overridable build metadata plus the executing host platform.
|
||||
func Current() Info {
|
||||
return Info{
|
||||
Version: semanticVersion,
|
||||
Commit: commit,
|
||||
BuildTime: buildTime,
|
||||
OS: runtime.GOOS,
|
||||
Arch: runtime.GOARCH,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package version
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Catches a source build that reports no usable identity or the wrong host platform.
|
||||
func TestCurrentReportsUsableSourceBuildIdentity(t *testing.T) {
|
||||
info := Current()
|
||||
if info.Version == "" || info.Commit == "" || info.BuildTime == "" {
|
||||
t.Fatalf("Current() = %#v, want non-empty build identity", info)
|
||||
}
|
||||
if info.OS != runtime.GOOS || info.Arch != runtime.GOARCH {
|
||||
t.Fatalf("Current() platform = %s/%s, want %s/%s", info.OS, info.Arch, runtime.GOOS, runtime.GOARCH)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user