421 lines
17 KiB
Go
421 lines
17 KiB
Go
package doctor
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/aritmolab/thothii/tools/tht/internal/compose"
|
|
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
|
)
|
|
|
|
// Catches treating an unavailable Docker executable as a successful diagnosis.
|
|
func TestRunReportsUnavailableDockerWithoutReturningAnExecutionError(t *testing.T) {
|
|
installation := doctorInstallation(t, "")
|
|
runner := &doctorRunner{dockerUnavailable: true}
|
|
|
|
report, err := Run(context.Background(), installation, runner)
|
|
if err != nil {
|
|
t.Fatalf("Run() error = %v, want report", err)
|
|
}
|
|
if report.OK || checkStatus(report, "docker") != "failed" {
|
|
t.Fatalf("Run() report = %#v, want failed Docker check", report)
|
|
}
|
|
}
|
|
|
|
func TestRunReportsOneSanitizedRuntimeAuthProjectionFailureBeforeCompose(t *testing.T) {
|
|
installation := doctorInstallation(t, "")
|
|
installation.Profile = "server"
|
|
installation.Authentication.RuntimeProjection = &config.RuntimeProjection{
|
|
Directory: "/runtime-auth", UID: 10001, GID: 10001,
|
|
}
|
|
previous := requireRuntimeAuthProjectionReady
|
|
requireRuntimeAuthProjectionReady = func(config.Installation) error {
|
|
return errors.New("synthetic-runtime-projection-secret")
|
|
}
|
|
t.Cleanup(func() { requireRuntimeAuthProjectionReady = previous })
|
|
|
|
runner := &doctorRunner{services: healthyServices}
|
|
report, err := Run(context.Background(), installation, runner)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
failures := 0
|
|
for _, check := range report.Checks {
|
|
if check.Name != "auth-projection" {
|
|
continue
|
|
}
|
|
failures++
|
|
if check.Status != StatusFailed || check.Detail != "runtime authentication projection is unavailable" {
|
|
t.Fatalf("auth-projection check = %#v", check)
|
|
}
|
|
}
|
|
if failures != 1 {
|
|
t.Fatalf("auth-projection failures = %d, report = %#v", failures, report)
|
|
}
|
|
if strings.Contains(reportText(report), "synthetic-runtime-projection-secret") {
|
|
t.Fatalf("runtime projection report leaked internal detail: %#v", report)
|
|
}
|
|
if len(runner.calls) != 0 {
|
|
t.Fatalf("doctor reached Compose diagnostics after failed projection gate: %v", runner.calls)
|
|
}
|
|
}
|
|
|
|
func TestRunLeavesUnprojectedDoctorChecklistUnchanged(t *testing.T) {
|
|
report, err := Run(context.Background(), doctorInstallation(t, ""), &doctorRunner{services: healthyServices})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, check := range report.Checks {
|
|
if check.Name == "auth-projection" {
|
|
t.Fatalf("unprojected report unexpectedly contains auth-projection: %#v", report)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestValidateVolumesRequiresAuthState(t *testing.T) {
|
|
legacy := `{"volumes":{"settings":{},"pi-state":{},"workspace-registry":{},"workspace-secrets":{},"sessions":{},"qdrant-data":{},"embedding-models":{}}}`
|
|
if err := ValidateVolumes(legacy); err == nil || !strings.Contains(err.Error(), "auth-state") {
|
|
t.Fatalf("ValidateVolumes() error = %v, want missing auth-state", err)
|
|
}
|
|
withAuthState := `{"volumes":{"settings":{},"pi-state":{},"workspace-registry":{},"workspace-secrets":{},"sessions":{},"qdrant-data":{},"embedding-models":{},"auth-state":{}}}`
|
|
if err := ValidateVolumes(withAuthState); err != nil {
|
|
t.Fatalf("ValidateVolumes() error = %v, want complete volume set", err)
|
|
}
|
|
}
|
|
|
|
// Catches Docker availability short-circuiting a host file-permission failure.
|
|
func TestRunChecksUnsafeFilesEvenWhenDockerIsUnavailable(t *testing.T) {
|
|
installation := doctorInstallation(t, "")
|
|
if err := os.Chmod(installation.EnvFile, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
report, err := Run(context.Background(), installation, &doctorRunner{dockerUnavailable: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if checkStatus(report, "files") != StatusFailed {
|
|
t.Fatalf("Run() files check = %q, want failed; report = %#v", checkStatus(report, "files"), report)
|
|
}
|
|
}
|
|
|
|
// Catches attempts to run in-container diagnostics when core is not running.
|
|
func TestRunSkipsContainerDiagnosticsWhenCoreIsStopped(t *testing.T) {
|
|
installation := doctorInstallation(t, "")
|
|
runner := &doctorRunner{services: stoppedServices}
|
|
|
|
report, err := Run(context.Background(), installation, runner)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if report.OK || checkStatus(report, "authentication") != "skipped" || checkStatus(report, "workflow") != "skipped" || checkStatus(report, "pi") != "skipped" {
|
|
t.Fatalf("Run() report = %#v, want stopped-core skips", report)
|
|
}
|
|
if strings.Contains(strings.Join(runner.calls, "\n"), " exec -T core ") {
|
|
t.Fatalf("Run() invoked a container diagnostic while core was stopped: %v", runner.calls)
|
|
}
|
|
}
|
|
|
|
func TestRunFailsAuthenticationWithoutExecWhenCoreIsRunningButUnhealthy(t *testing.T) {
|
|
installation := doctorInstallation(t, "")
|
|
runner := &doctorRunner{services: unhealthyCoreServices}
|
|
|
|
report, err := Run(context.Background(), installation, runner)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if report.OK || checkStatus(report, "authentication") != StatusFailed {
|
|
t.Fatalf("Run() report = %#v, want deterministic failed authentication", report)
|
|
}
|
|
assertChecklist(t, report, []string{"descriptor", "files", "docker", "compose", "configuration", "authentication", "services", "core-http", "frontend-http", "workspace-registry", "workflow", "pi"})
|
|
if strings.Contains(strings.Join(runner.calls, "\n"), " exec -T ") {
|
|
t.Fatalf("Run() invoked exec -T while core was unhealthy: %v", runner.calls)
|
|
}
|
|
if detail := checkDetail(report, "authentication"); detail != "core is running but unhealthy" {
|
|
t.Fatalf("authentication detail = %q, want deterministic unhealthy detail", detail)
|
|
}
|
|
}
|
|
|
|
func TestRunExecutesOnlyAuthenticationWhenCoreIsHealthyButAnotherServiceIsUnhealthy(t *testing.T) {
|
|
installation := doctorInstallation(t, "")
|
|
runner := &doctorRunner{services: unhealthyFrontendServices}
|
|
|
|
report, err := Run(context.Background(), installation, runner)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if checkStatus(report, "authentication") != StatusPassed || checkStatus(report, "services") != StatusFailed {
|
|
t.Fatalf("Run() report = %#v, want auth passed before failed services", report)
|
|
}
|
|
calls := strings.Join(runner.calls, "\n")
|
|
if strings.Count(calls, " exec -T ") != 1 || !strings.Contains(calls, "exec -T core node dist/auth/diagnostic-command.js --json") {
|
|
t.Fatalf("Run() calls = %s, want only the healthy-core authentication exec", calls)
|
|
}
|
|
}
|
|
|
|
// Catches host-Python diagnostics or omission of workflow/Pi checks once core is healthy.
|
|
func TestRunUsesOnlyContainerLocalWorkflowAndPiDiagnosticsWhenCoreRuns(t *testing.T) {
|
|
installation := doctorInstallation(t, "")
|
|
runner := &doctorRunner{services: healthyServices}
|
|
|
|
report, err := Run(context.Background(), installation, runner)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !report.OK || checkStatus(report, "authentication") != "passed" || checkStatus(report, "workflow") != "passed" || checkStatus(report, "pi") != "passed" {
|
|
t.Fatalf("Run() report = %#v, want successful container diagnostics", report)
|
|
}
|
|
assertChecklist(t, report, []string{"descriptor", "files", "docker", "compose", "configuration", "authentication", "services", "core-http", "frontend-http", "workspace-registry", "workflow", "pi"})
|
|
calls := strings.Join(runner.calls, "\n")
|
|
if !strings.Contains(calls, "exec -T core node dist/auth/diagnostic-command.js --json") {
|
|
t.Fatalf("Run() calls = %s, want core-local authentication diagnostic", calls)
|
|
}
|
|
if !strings.Contains(calls, "exec -T core node dist/operator-command.js workflow-doctor") {
|
|
t.Fatalf("Run() calls = %s, want registry-bound core-local workflow doctor", calls)
|
|
}
|
|
if strings.Contains(calls, ".venv") || strings.Contains(calls, "python") {
|
|
t.Fatalf("Run() calls = %s, must not require host Python", calls)
|
|
}
|
|
}
|
|
|
|
// Catches a claimed valid registry based only on the rendered volume declaration.
|
|
func TestRunFailsAnInvalidContainerLocalRegistryState(t *testing.T) {
|
|
installation := doctorInstallation(t, "")
|
|
runner := &doctorRunner{services: healthyServices, registryInvalid: true}
|
|
|
|
report, err := Run(context.Background(), installation, runner)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if report.OK || checkStatus(report, "workspace-registry") != StatusFailed {
|
|
t.Fatalf("Run() report = %#v, want failed registry diagnostic", report)
|
|
}
|
|
if !strings.Contains(strings.Join(runner.calls, "\n"), "workspace-registry/state/active.json") {
|
|
t.Fatalf("Run() calls = %v, want an actual registry-state read", runner.calls)
|
|
}
|
|
}
|
|
|
|
// Catches Compose health being treated as proof that the HTTP listeners answer requests.
|
|
func TestRunReportsEachHTTPReachabilityProbeFailure(t *testing.T) {
|
|
installation := doctorInstallation(t, "")
|
|
for _, endpoint := range []string{"core", "frontend"} {
|
|
t.Run(endpoint, func(t *testing.T) {
|
|
probe := &probeStub{failures: map[string]error{endpoint: errors.New(endpoint + " unreachable")}}
|
|
report, err := RunWithProbe(context.Background(), installation, &doctorRunner{services: healthyServices}, probe)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if report.OK || checkStatus(report, endpoint+"-http") != StatusFailed {
|
|
t.Fatalf("RunWithProbe() report = %#v, want failed %s HTTP check", report, endpoint)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Catches a workflow failure leaking a credential from a declared secret file into a report.
|
|
func TestRunRedactsWorkflowDiagnosticFailures(t *testing.T) {
|
|
installation := doctorInstallation(t, "WORKFLOW_TOKEN_FILE=%s\n")
|
|
secretPath := filepath.Join(filepath.Dir(installation.EnvFile), "workflow-token")
|
|
if err := os.WriteFile(secretPath, []byte("workflow-secret-value"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
contents := "WORKFLOW_TOKEN_FILE=" + secretPath + "\n"
|
|
if err := os.WriteFile(installation.EnvFile, []byte(contents), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runner := &doctorRunner{services: healthyServices, workflowFailure: "workflow-secret-value"}
|
|
|
|
report, err := Run(context.Background(), installation, runner)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if report.OK || checkStatus(report, "workflow") != "failed" {
|
|
t.Fatalf("Run() report = %#v, want failed workflow check", report)
|
|
}
|
|
if strings.Contains(reportText(report), "workflow-secret-value") {
|
|
t.Fatalf("Run() report exposed a secret: %#v", report)
|
|
}
|
|
}
|
|
|
|
func TestRunFailsBeforeDockerWhenDeclaredSecretCorpusIsIncomplete(t *testing.T) {
|
|
installation := doctorInstallation(t, "")
|
|
missing := filepath.Join(filepath.Dir(installation.EnvFile), "missing-pi-auth.json")
|
|
if err := os.WriteFile(installation.EnvFile, []byte("PI_AUTH_FILE="+missing+"\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runner := &doctorRunner{services: healthyServices}
|
|
|
|
report, err := Run(context.Background(), installation, runner)
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if report.OK || checkStatus(report, "files") != StatusFailed || len(runner.calls) != 0 {
|
|
t.Fatalf("incomplete corpus was not refused before Docker: report=%#v calls=%v", report, runner.calls)
|
|
}
|
|
if strings.Contains(reportText(report), missing) {
|
|
t.Fatalf("incomplete corpus report exposed a secret path: %#v", report)
|
|
}
|
|
assertChecklist(t, report, []string{"descriptor", "files", "docker", "compose", "configuration", "authentication", "services", "core-http", "frontend-http", "workspace-registry", "workflow", "pi"})
|
|
}
|
|
|
|
func doctorInstallation(t *testing.T, _ string) config.Installation {
|
|
t.Helper()
|
|
base, err := filepath.EvalSymlinks(os.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
root, err := os.MkdirTemp(base, "tht-doctor-")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = os.RemoveAll(root) })
|
|
project := filepath.Join(root, "project")
|
|
for _, path := range []string{project, filepath.Join(project, "deploy"), filepath.Join(project, "docker")} {
|
|
if err := os.MkdirAll(path, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for _, path := range []string{filepath.Join(project, "compose.yaml"), filepath.Join(project, "deploy", "compose.local.yaml"), filepath.Join(project, "docker", "core.Dockerfile")} {
|
|
if err := os.WriteFile(path, []byte("services: {}\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
envFile := filepath.Join(root, "operator.env")
|
|
if err := os.WriteFile(envFile, []byte("SAFE_VALUE=1\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return config.Installation{Path: filepath.Join(root, "thothii-installation.yaml"), Profile: "local", ProjectDirectory: project, EnvFile: envFile}
|
|
}
|
|
|
|
type doctorRunner struct {
|
|
calls []string
|
|
dockerUnavailable bool
|
|
services string
|
|
workflowFailure string
|
|
registryInvalid bool
|
|
}
|
|
|
|
func (r *doctorRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
|
|
call := strings.Join(args, " ")
|
|
r.calls = append(r.calls, call)
|
|
if r.dockerUnavailable {
|
|
return compose.Result{ExitCode: 127}, errors.New("docker unavailable")
|
|
}
|
|
switch {
|
|
case strings.Contains(call, "version --format {{.Client.Version}}"):
|
|
return compose.Result{Stdout: "26.0.0\n"}, nil
|
|
case strings.Contains(call, "compose version --short"):
|
|
return compose.Result{Stdout: "v2.30.0\n"}, nil
|
|
case strings.Contains(call, "config --quiet"):
|
|
return compose.Result{}, nil
|
|
case strings.Contains(call, "config --format json"):
|
|
return compose.Result{Stdout: renderedConfig}, nil
|
|
case strings.Contains(call, "ps --all --format json"):
|
|
if r.services == "" {
|
|
return compose.Result{Stdout: healthyServices}, nil
|
|
}
|
|
return compose.Result{Stdout: r.services}, nil
|
|
case strings.Contains(call, "operator-command.js workflow-doctor"):
|
|
if r.workflowFailure != "" {
|
|
return compose.Result{Stderr: r.workflowFailure, ExitCode: 23}, errors.New("workflow failed")
|
|
}
|
|
return compose.Result{Stdout: `{"ready":true,"workspaces":1}`}, nil
|
|
case strings.Contains(call, "dist/auth/diagnostic-command.js --json"):
|
|
return compose.Result{Stdout: `{"ready":true,"mode":"oidc","checks":[{"level":"info","code":"auth_ready","message":"Authentication is ready."}]}`}, nil
|
|
case strings.Contains(call, "workspace-registry/state/active.json"):
|
|
if r.registryInvalid {
|
|
return compose.Result{ExitCode: 23, Stderr: "invalid workspace registry"}, errors.New("registry invalid")
|
|
}
|
|
return compose.Result{Stdout: "registry is valid\n"}, nil
|
|
case strings.Contains(call, "pi --version") || strings.Contains(call, "PI_VERSION") || strings.Contains(call, "io.thothii.pi.version"):
|
|
return compose.Result{Stdout: "0.80.3\n"}, nil
|
|
case strings.Contains(call, "test -w /home/thoth/.pi") || strings.Contains(call, "test -r /home/thoth/.pi/agent/auth.json") || strings.Contains(call, "/health"):
|
|
return compose.Result{Stdout: `{"ready":true}`}, nil
|
|
case strings.Contains(call, "operator-command.js pi-test"):
|
|
return compose.Result{Stdout: `{"ready":true}`}, nil
|
|
case strings.Contains(call, "ps -q core"):
|
|
return compose.Result{Stdout: "core-id\n"}, nil
|
|
}
|
|
return compose.Result{}, nil
|
|
}
|
|
|
|
type probeStub struct{ failures map[string]error }
|
|
|
|
func (p *probeStub) Probe(_ context.Context, endpoint HTTPProbeTarget) error {
|
|
return p.failures[endpoint.Name]
|
|
}
|
|
|
|
func checkStatus(report Report, name string) string {
|
|
for _, check := range report.Checks {
|
|
if check.Name == name {
|
|
return check.Status
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func checkDetail(report Report, name string) string {
|
|
for _, check := range report.Checks {
|
|
if check.Name == name {
|
|
return check.Detail
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func reportText(report Report) string {
|
|
parts := make([]string, 0, len(report.Checks))
|
|
for _, check := range report.Checks {
|
|
parts = append(parts, check.Name+" "+check.Status+" "+check.Detail)
|
|
}
|
|
return strings.Join(parts, "\n")
|
|
}
|
|
|
|
func assertChecklist(t *testing.T, report Report, want []string) {
|
|
t.Helper()
|
|
got := make([]string, 0, len(report.Checks))
|
|
for _, check := range report.Checks {
|
|
got = append(got, check.Name)
|
|
}
|
|
if strings.Join(got, ",") != strings.Join(want, ",") {
|
|
t.Fatalf("doctor checklist = %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
const renderedConfig = `{"volumes":{"settings":{},"pi-state":{},"workspace-registry":{},"workspace-secrets":{},"sessions":{},"qdrant-data":{},"embedding-models":{},"auth-state":{}},"services":{"core":{"image":"thothii-core:local","environment":{"THT_LLM_URL":"https://llm.example.invalid"}}}}`
|
|
|
|
const healthyServices = `[
|
|
{"Service":"core","State":"running","Health":"healthy"},
|
|
{"Service":"frontend","State":"running","Health":"healthy"},
|
|
{"Service":"qdrant","State":"running","Health":"healthy"},
|
|
{"Service":"embedding","State":"running","Health":"healthy"},
|
|
{"Service":"embedding-model-init","State":"exited","ExitCode":0}
|
|
]`
|
|
|
|
const stoppedServices = `[
|
|
{"Service":"core","State":"exited","Health":""},
|
|
{"Service":"frontend","State":"running","Health":"healthy"}
|
|
]`
|
|
|
|
const unhealthyCoreServices = `[
|
|
{"Service":"core","State":"running","Health":"unhealthy"},
|
|
{"Service":"frontend","State":"running","Health":"healthy"},
|
|
{"Service":"qdrant","State":"running","Health":"healthy"},
|
|
{"Service":"embedding","State":"running","Health":"healthy"},
|
|
{"Service":"embedding-model-init","State":"exited","ExitCode":0}
|
|
]`
|
|
|
|
const unhealthyFrontendServices = `[
|
|
{"Service":"core","State":"running","Health":"healthy"},
|
|
{"Service":"frontend","State":"running","Health":"unhealthy"},
|
|
{"Service":"qdrant","State":"running","Health":"healthy"},
|
|
{"Service":"embedding","State":"running","Health":"healthy"},
|
|
{"Service":"embedding-model-init","State":"exited","ExitCode":0}
|
|
]`
|