Files
ThothII/tools/tht/internal/setup/run_test.go
T

542 lines
23 KiB
Go

package setup
import (
"bytes"
"context"
"errors"
"io"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/aritmolab/thothii/tools/tht/internal/authconfig"
"github.com/aritmolab/thothii/tools/tht/internal/authprojection"
"github.com/aritmolab/thothii/tools/tht/internal/compose"
"github.com/aritmolab/thothii/tools/tht/internal/config"
"github.com/aritmolab/thothii/tools/tht/internal/doctor"
)
func TestRunBuildsStartsAndVerifiesInOrder(t *testing.T) {
projectRoot, request := setupRunFixture(t, false)
runner := &setupRunner{health: []string{
unhealthyServicesJSON,
healthyServicesJSON,
}}
var output bytes.Buffer
result, err := Run(context.Background(), runner, request, strings.NewReader(""), &output)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if !result.Configured || !result.Built || !result.Started || !result.Healthy {
t.Fatalf("Run() result = %#v, want all lifecycle phases complete", result)
}
if result.ProjectName == "" || result.DescriptorPath != filepath.Join(projectRoot, "deploy", "ci", "thothii-installation.yaml") {
t.Fatalf("Run() result = %#v, want descriptor below the project deployment directory", result)
}
want := []string{
"docker engine", "docker compose", "architecture", "compose config", "compose build",
"compose up", "health", "health", "doctor docker", "doctor compose", "compose config", "doctor config", "health", "authentication", "core HTTP", "frontend HTTP", "workspace registry", "workflow doctor", "pi doctor",
}
if got := collapseStages(runner.stages); strings.Join(got, " | ") != strings.Join(want, " | ") {
t.Fatalf("runner stages = %v, want %v", got, want)
}
for _, text := range []string{"ThothII is ready", "http://127.0.0.1:8080", result.DescriptorPath} {
if !strings.Contains(output.String(), text) {
t.Errorf("output = %q, want %q", output.String(), text)
}
}
}
func TestRunConfigureOnlyStopsAfterRenderedConfiguration(t *testing.T) {
_, request := setupRunFixture(t, true)
runner := &setupRunner{}
result, err := Run(context.Background(), runner, request, strings.NewReader(""), io.Discard)
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if !result.Configured || result.Built || result.Started || result.Healthy {
t.Fatalf("Run() result = %#v, want only configuration", result)
}
want := []string{"docker engine", "docker compose", "architecture", "compose config"}
if got := runner.stages; strings.Join(got, " | ") != strings.Join(want, " | ") {
t.Fatalf("runner stages = %v, want %v", got, want)
}
}
func TestRunCompleteMigratesCatalogPullsWorkspaceAndTestsLLM(t *testing.T) {
root, request := setupRunFixture(t, false)
request.Complete = true
secretRoot := filepath.Join(root, "deploy", "ci", "secrets")
for path, contents := range map[string]string{
filepath.Join(secretRoot, "pi-auth.json"): "{\"deepseek\":{\"apiKey\":\"configured\"}}\n",
filepath.Join(secretRoot, "workspace-git-key"): "private-key\n",
filepath.Join(secretRoot, "workspace-git-known-hosts"): "git.example.invalid ssh-ed25519 AAAA\n",
} {
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
t.Fatal(err)
}
}
runner := &setupRunner{health: []string{healthyServicesJSON}}
if _, err := Run(context.Background(), runner, request, strings.NewReader(""), io.Discard); err != nil {
t.Fatal(err)
}
want := []string{
"docker engine", "docker compose", "architecture", "compose config", "compose build",
"catalog db", "catalog migrate", "compose up", "health", "workspace pull", "pi doctor",
"doctor docker", "doctor compose", "compose config", "doctor config", "health",
"authentication", "core HTTP", "frontend HTTP", "workspace registry", "workflow doctor", "pi doctor",
}
if got := collapseStages(runner.stages); strings.Join(got, " | ") != strings.Join(want, " | ") {
t.Fatalf("complete setup stages = %v, want %v", got, want)
}
}
func TestRunConfiguresAndStaticallyValidatesLocalAuthBeforeComposeRender(t *testing.T) {
projectRoot, request := setupRunFixture(t, true)
passwordFile := filepath.Join(projectRoot, "initial-admin-password")
if err := os.WriteFile(passwordFile, []byte("correct horse battery staple"), 0o600); err != nil {
t.Fatal(err)
}
request.NonInteractive = true
request.Answers.AuthMode = "local"
request.Answers.AuthPublicURL = "http://127.0.0.1:8080"
request.Answers.AuthAdminUser = "admin"
request.Answers.AuthAdminDisplayName = "Initial Admin"
request.Answers.AuthPasswordFile = passwordFile
runner := &setupRunner{}
if _, err := Run(context.Background(), runner, request, strings.NewReader(""), io.Discard); err != nil {
t.Fatal(err)
}
installationPath := filepath.Join(projectRoot, "deploy", "ci", "thothii-installation.yaml")
installation, err := config.Load(installationPath)
if err != nil {
t.Fatal(err)
}
configuration, registry, err := authconfig.Load(installation.AuthenticationDirectory())
if err != nil {
t.Fatalf("setup did not create a statically valid authentication configuration: %v", err)
}
if configuration.Mode != "local" || len(registry.Users) != 1 || registry.Users[0].Username != "admin" {
t.Fatalf("authentication configuration = %#v registry = %#v", configuration, registry)
}
assertSetupStages(t, runner, "docker engine", "docker compose", "architecture", "compose config")
}
func TestRunConfigureOnlyPublishesInitialProjectedServerAuthentication(t *testing.T) {
requireProjectedServerTestHost(t)
projectRoot, request := setupRunFixture(t, true)
request.Profile = "server"
if err := os.WriteFile(filepath.Join(projectRoot, "deploy", "compose.auth-runtime-projection.yaml"), []byte("services: {}\n"), 0o600); err != nil {
t.Fatal(err)
}
if _, err := Run(context.Background(), &setupRunner{}, request, strings.NewReader(""), io.Discard); err != nil {
t.Fatal(err)
}
installation, err := config.Load(filepath.Join(projectRoot, "deploy", "ci", "thothii-installation.yaml"))
if err != nil {
t.Fatal(err)
}
projection := installation.RuntimeAuthProjection()
if projection == nil {
t.Fatal("generated server installation has no runtime auth projection")
}
status, err := authprojection.Inspect(authprojection.Spec{RuntimeRoot: projection.Directory, UID: projection.UID, GID: projection.GID})
if err != nil || status.Selector.State != "ready" {
t.Fatalf("initial runtime auth projection = %#v, %v; want ready", status, err)
}
}
func TestRunConfigureOnlyLeavesProjectedAuthenticationBlockedWhenInitialPublicationFails(t *testing.T) {
requireProjectedServerTestHost(t)
projectRoot, request := setupRunFixture(t, true)
request.Profile = "server"
if err := os.WriteFile(filepath.Join(projectRoot, "deploy", "compose.auth-runtime-projection.yaml"), []byte("services: {}\n"), 0o600); err != nil {
t.Fatal(err)
}
previous := publishProjectedCanonical
publishProjectedCanonical = func(ctx context.Context, canonicalRoot string, spec authconfig.ProjectionSpec) (authconfig.ProjectionStatus, error) {
transaction, err := authconfig.BeginExternalProjectionTransaction(ctx, canonicalRoot, spec)
if err != nil {
return authconfig.ProjectionStatus{}, err
}
if err := transaction.Close(); err != nil {
return authconfig.ProjectionStatus{}, err
}
return authconfig.ProjectionStatus{}, errors.New("synthetic-password-sentinel")
}
t.Cleanup(func() { publishProjectedCanonical = previous })
_, err := Run(context.Background(), &setupRunner{}, request, strings.NewReader(""), io.Discard)
if err == nil || strings.Contains(err.Error(), "synthetic-password-sentinel") {
t.Fatalf("Run() error = %v, want sanitized publication failure", err)
}
installation, loadErr := config.Load(filepath.Join(projectRoot, "deploy", "ci", "thothii-installation.yaml"))
if loadErr != nil {
t.Fatal(loadErr)
}
projection := installation.RuntimeAuthProjection()
if projection == nil {
t.Fatal("generated server installation has no runtime auth projection")
}
_, inspectErr := authprojection.Inspect(authprojection.Spec{RuntimeRoot: projection.Directory, UID: projection.UID, GID: projection.GID})
if !errors.Is(inspectErr, authprojection.ErrBlocked) {
t.Fatalf("Inspect() error = %v, want blocked projection", inspectErr)
}
}
func TestRunConfigureOnlyVerifiesProjectedAuthenticationAfterPublication(t *testing.T) {
requireProjectedServerTestHost(t)
projectRoot, request := setupRunFixture(t, true)
request.Profile = "server"
if err := os.WriteFile(filepath.Join(projectRoot, "deploy", "compose.auth-runtime-projection.yaml"), []byte("services: {}\n"), 0o600); err != nil {
t.Fatal(err)
}
previous := requireRuntimeAuthProjectionReady
calls := 0
requireRuntimeAuthProjectionReady = func(installation config.Installation) error {
calls++
projection := installation.RuntimeAuthProjection()
if projection == nil {
t.Fatal("post-publication readiness received an unprojected installation")
}
status, err := authprojection.Inspect(authprojection.Spec{
RuntimeRoot: projection.Directory,
UID: projection.UID,
GID: projection.GID,
})
if err != nil || status.Selector.State != "ready" {
t.Fatalf("post-publication projection = %#v, %v; want ready", status, err)
}
return errors.New("synthetic-readiness-secret")
}
t.Cleanup(func() { requireRuntimeAuthProjectionReady = previous })
_, err := Run(context.Background(), &setupRunner{}, request, strings.NewReader(""), io.Discard)
if err == nil || strings.Contains(err.Error(), "synthetic-readiness-secret") {
t.Fatalf("Run() error = %v, want sanitized post-publication readiness failure", err)
}
if calls != 1 {
t.Fatalf("post-publication readiness calls = %d, want 1", calls)
}
}
func TestRunRejectsIncompleteNonInteractiveLocalAuthenticationBeforeComposeRender(t *testing.T) {
_, request := setupRunFixture(t, true)
request.NonInteractive = true
request.Answers.AuthMode = "local"
request.Answers.AuthPublicURL = ""
request.Answers.AuthAdminUser = ""
request.Answers.AuthAdminDisplayName = ""
request.Answers.AuthPasswordFile = ""
runner := &setupRunner{}
_, err := Run(context.Background(), runner, request, strings.NewReader(""), io.Discard)
if err == nil || !strings.Contains(err.Error(), "non-interactive local authentication") {
t.Fatalf("Run() error = %v, want non-interactive local authentication guidance", err)
}
assertSetupStages(t, runner, "docker engine", "docker compose", "architecture")
}
func TestRunPropagatesPreflightFailureBeforeWritingConfiguration(t *testing.T) {
projectRoot, request := setupRunFixture(t, false)
runner := &setupRunner{failureAt: "docker engine"}
_, err := Run(context.Background(), runner, request, strings.NewReader(""), io.Discard)
if err == nil || !strings.Contains(err.Error(), "Docker Engine") {
t.Fatalf("Run() error = %v, want Docker Engine failure", err)
}
if _, statErr := os.Stat(filepath.Join(projectRoot, "deploy", "ci", "thothii-installation.yaml")); !errors.Is(statErr, os.ErrNotExist) {
t.Fatalf("descriptor was written after preflight failure: %v", statErr)
}
}
func TestRunTimesOutWithPartialStartupGuidance(t *testing.T) {
_, request := setupRunFixture(t, false)
runner := &setupRunner{health: []string{unhealthyServicesJSON}}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Millisecond)
defer cancel()
_, err := Run(ctx, runner, request, strings.NewReader(""), io.Discard)
if err == nil {
t.Fatal("Run() error = nil, want health timeout")
}
for _, text := range []string{"frontend", "tht logs frontend", "tht status", "left running"} {
if !strings.Contains(err.Error(), text) {
t.Errorf("Run() error = %q, want %q", err, text)
}
}
}
func TestRunBuildFailureDoesNotAttemptContainerStartup(t *testing.T) {
_, request := setupRunFixture(t, false)
runner := &setupRunner{failureAt: "compose build"}
_, err := Run(context.Background(), runner, request, strings.NewReader(""), io.Discard)
if err == nil || !strings.Contains(err.Error(), "setup image build") {
t.Fatalf("Run() error = %v, want original build failure", err)
}
if strings.Contains(err.Error(), "tht status") {
t.Fatalf("Run() error = %q, must not offer partial-start recovery before compose up", err)
}
assertSetupStages(t, runner, "docker engine", "docker compose", "architecture", "compose config", "compose build")
}
func TestRunUpFailurePreservesCauseAndOffersRecovery(t *testing.T) {
_, request := setupRunFixture(t, false)
runner := &setupRunner{failureAt: "compose up"}
_, err := Run(context.Background(), runner, request, strings.NewReader(""), io.Discard)
assertRecoveryFailure(t, err, "setup stack start", "core")
assertSetupStages(t, runner, "docker engine", "docker compose", "architecture", "compose config", "compose build", "compose up")
}
func TestRunAggregateDoctorFailurePreservesCauseAndOffersRecovery(t *testing.T) {
_, request := setupRunFixture(t, false)
runner := &setupRunner{failureAt: "doctor config"}
_, err := Run(context.Background(), runner, request, strings.NewReader(""), io.Discard)
assertRecoveryFailure(t, err, "setup doctor reported failed checks", "core")
assertSetupStages(t, runner, "docker engine", "docker compose", "architecture", "compose config", "compose build", "compose up", "health", "doctor docker", "doctor compose", "compose config", "doctor config", "health", "core HTTP", "frontend HTTP", "workflow doctor", "pi doctor")
}
func TestRunPiDoctorFailurePreservesCauseAndOffersRecovery(t *testing.T) {
_, request := setupRunFixture(t, false)
runner := &setupRunner{failureAt: "pi doctor"}
_, err := Run(context.Background(), runner, request, strings.NewReader(""), io.Discard)
assertRecoveryFailure(t, err, "setup doctor reported failed checks", "core")
assertSetupStages(t, runner, "docker engine", "docker compose", "architecture", "compose config", "compose build", "compose up", "health", "doctor docker", "doctor compose", "compose config", "doctor config", "health", "authentication", "core HTTP", "frontend HTTP", "workspace registry", "workflow doctor", "pi doctor")
}
func TestRequireVolumesRequiresEveryInstallationVolume(t *testing.T) {
all := []string{"settings", "pi-state", "workspace-registry", "workspace-secrets", "sessions", "qdrant-data", "embedding-models", "auth-state"}
for _, missing := range all {
t.Run("missing "+missing, func(t *testing.T) {
volumes := make([]string, 0, len(all)-1)
for _, name := range all {
if name != missing {
volumes = append(volumes, name)
}
}
if err := doctor.ValidateVolumes(renderedConfigForVolumes(volumes...)); err == nil || !strings.Contains(err.Error(), missing) {
t.Fatalf("ValidateVolumes() error = %v, want missing %q", err, missing)
}
})
}
if err := doctor.ValidateVolumes(renderedConfigForVolumes("unrelated")); err == nil {
t.Fatal("ValidateVolumes() error = nil, want required-volume failure for unrelated-only configuration")
}
if err := doctor.ValidateVolumes(renderedConfigForVolumes(all...)); err != nil {
t.Fatalf("ValidateVolumes() error = %v, want complete installation volume set", err)
}
}
func TestRunIgnoresIrrelevantCRLFFilesDuringLineEndingCheck(t *testing.T) {
projectRoot, request := setupRunFixture(t, true)
irrelevant := filepath.Join(projectRoot, "node_modules", "unrelated", "generated.yml")
if err := os.MkdirAll(filepath.Dir(irrelevant), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(irrelevant, []byte("generated: true\r\n"), 0o600); err != nil {
t.Fatal(err)
}
if _, err := Run(context.Background(), &setupRunner{}, request, strings.NewReader(""), io.Discard); err != nil {
t.Fatalf("Run() error = %v, want irrelevant CRLF file ignored", err)
}
}
func assertRecoveryFailure(t *testing.T, err error, cause, service string) {
t.Helper()
if err == nil {
t.Fatal("Run() error = nil, want failure")
}
for _, text := range []string{cause, "tht logs " + service, "tht status", "left running"} {
if !strings.Contains(err.Error(), text) {
t.Errorf("Run() error = %q, want %q", err, text)
}
}
}
func assertSetupStages(t *testing.T, runner *setupRunner, want ...string) {
t.Helper()
if got := collapseStages(runner.stages); strings.Join(got, " | ") != strings.Join(want, " | ") {
t.Fatalf("runner stages = %v, want %v", got, want)
}
}
const unhealthyServicesJSON = `[
{"Service":"core","State":"running","Health":"healthy"},
{"Service":"frontend","State":"running","Health":"starting"},
{"Service":"qdrant","State":"running","Health":"healthy"},
{"Service":"embedding","State":"running","Health":"healthy"},
{"Service":"embedding-model-init","State":"exited","ExitCode":0}
]`
const healthyServicesJSON = `[
{"Service":"core","State":"running","Health":"healthy"},
{"Service":"frontend","State":"running","Health":"healthy"},
{"Service":"qdrant","State":"running","Health":"healthy"},
{"Service":"embedding","State":"running","Health":"healthy"},
{"Service":"embedding-model-init","State":"exited","ExitCode":0}
]`
type setupRunner struct {
stages []string
health []string
failureAt string
}
func (r *setupRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
stage, result := setupStage(args)
if stage == "docker compose" && containsStage(r.stages, "docker compose") {
stage = "doctor compose"
}
if stage == "doctor config" && containsStage(r.stages, "workflow doctor") {
stage = "pi doctor"
}
r.stages = append(r.stages, stage)
if stage == r.failureAt {
return compose.Result{ExitCode: 41}, errors.New("fixture failure")
}
if stage == "health" {
if len(r.health) == 0 {
result.Stdout = healthyServicesJSON
} else {
result.Stdout, r.health = r.health[0], r.health[1:]
}
}
return result, nil
}
func containsStage(stages []string, wanted string) bool {
for _, stage := range stages {
if stage == wanted {
return true
}
}
return false
}
func collapseStages(stages []string) []string {
collapsed := make([]string, 0, len(stages))
for _, stage := range stages {
if stage == "pi doctor" && len(collapsed) > 0 && collapsed[len(collapsed)-1] == stage {
continue
}
collapsed = append(collapsed, stage)
}
return collapsed
}
func setupStage(args []string) (string, compose.Result) {
joined := strings.Join(args, " ")
switch {
case joined == "version --format {{.Server.Version}}":
return "docker engine", compose.Result{Stdout: "26.0.0\n"}
case joined == "version --format {{.Client.Version}}":
return "doctor docker", compose.Result{Stdout: "26.0.0\n"}
case joined == "compose version --short":
return "docker compose", compose.Result{Stdout: "v2.30.0\n"}
case joined == "version --format {{.Server.Arch}}":
return "architecture", compose.Result{Stdout: "arm64\n"}
case strings.HasSuffix(joined, " config --quiet"):
return "compose config", compose.Result{}
case strings.HasSuffix(joined, " up --detach catalog-db"):
return "catalog db", compose.Result{}
case strings.HasSuffix(joined, " --profile catalog-maintenance run --rm catalog-migrate"):
return "catalog migrate", compose.Result{}
case strings.HasSuffix(joined, " build"):
return "compose build", compose.Result{}
case strings.HasSuffix(joined, " up --detach --remove-orphans"):
return "compose up", compose.Result{}
case strings.HasSuffix(joined, " ps --all --format json"):
return "health", compose.Result{}
case strings.HasSuffix(joined, " config --format json"):
return "doctor config", compose.Result{Stdout: renderedSetupConfig}
case strings.Contains(joined, "exec -T core node dist/auth/diagnostic-command.js --json"):
return "authentication", compose.Result{Stdout: `{"ready":true,"mode":"oidc","checks":[{"level":"info","code":"auth_ready","message":"Authentication is ready."}]}`}
case strings.Contains(joined, "exec -T core node dist/operator-command.js workflow-doctor"):
return "workflow doctor", compose.Result{Stdout: `{"ready":true,"workspaces":1}`}
case strings.Contains(joined, "operator-command.js workspace-pull"):
return "workspace pull", compose.Result{Stdout: `{"ready":true,"status":"succeeded"}`}
case strings.Contains(joined, "exec -T core curl -fsS --max-time 5 http://127.0.0.1:8787/health"):
return "core HTTP", compose.Result{}
case strings.Contains(joined, "exec -T frontend wget -q -T 5 -O /dev/null http://127.0.0.1:8080/"):
return "frontend HTTP", compose.Result{}
case strings.Contains(joined, "workspace-registry/state/active.json"):
return "workspace registry", compose.Result{}
case strings.Contains(joined, " ps -q core"):
return "pi doctor", compose.Result{Stdout: "core-id\n"}
case strings.HasPrefix(joined, "inspect --format"):
return "pi doctor", compose.Result{Stdout: "0.80.3\n"}
case strings.Contains(joined, "pi --version") || strings.Contains(joined, "PI_VERSION") || strings.Contains(joined, "test -w") || strings.Contains(joined, "test -r") || strings.Contains(joined, "127.0.0.1:8787/health"):
return "pi doctor", compose.Result{Stdout: "0.80.3\n"}
case strings.Contains(joined, "operator-command.js pi-test"):
return "pi doctor", compose.Result{Stdout: `{"ready":true}`}
default:
return "unexpected: " + joined, compose.Result{}
}
}
const renderedSetupConfig = `{"volumes":{"settings":{},"pi-state":{},"workspace-registry":{},"workspace-secrets":{},"sessions":{},"qdrant-data":{},"embedding-models":{},"auth-state":{}},"services":{"core":{"image":"thothii-core:local","environment":{"THT_LLM_URL":"https://llm.example.invalid"}}}}`
func renderedConfigForVolumes(volumes ...string) string {
entries := make([]string, 0, len(volumes))
for _, volume := range volumes {
entries = append(entries, `"`+volume+`":{}`)
}
return `{"volumes":{` + strings.Join(entries, ",") + `}}`
}
func setupRunFixture(t *testing.T, configureOnly bool) (string, Request) {
t.Helper()
temporaryRoot, err := filepath.EvalSymlinks(os.TempDir())
if err != nil {
t.Fatal(err)
}
root, err := os.MkdirTemp(temporaryRoot, "tht-setup-run-")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.RemoveAll(root) })
for _, directory := range []string{".git", "backend", "frontend", "harness", "tools", "deploy", "docker"} {
if err := os.MkdirAll(filepath.Join(root, directory), 0o755); err != nil {
t.Fatal(err)
}
}
for _, path := range []string{
"compose.yaml", "deploy/compose.local.yaml", "deploy/compose.server.yaml", "deploy/compose.git-https.yaml",
} {
if err := os.WriteFile(filepath.Join(root, path), []byte("services: {}\n"), 0o600); err != nil {
t.Fatal(err)
}
}
secrets := filepath.Join(root, "deploy", "ci", "secrets")
if err := os.MkdirAll(secrets, 0o700); err != nil {
t.Fatal(err)
}
passwordFile := filepath.Join(secrets, "initial-admin-password")
if err := os.WriteFile(passwordFile, []byte("fixture authentication password"), 0o600); err != nil {
t.Fatal(err)
}
return root, Request{
ProjectRoot: root, InstallationID: "ci", Profile: "local", ConfigureOnly: configureOnly, NonInteractive: true,
Answers: Answers{
WorkspaceRemote: "https://git.example.invalid/thothii-workspaces.git", WorkspaceBranch: "main", WorkspaceAccess: "https",
SecretsFile: filepath.Join(secrets, "thothii.secrets"), PiAuthFile: filepath.Join(secrets, "pi-auth.json"),
GitCredentialsFile: filepath.Join(secrets, "git-credentials"), GitCAFile: filepath.Join(secrets, "git-ca.pem"),
AuthMode: "local", AuthPublicURL: "http://127.0.0.1:8080", AuthAdminUser: "admin",
AuthAdminDisplayName: "Initial Admin", AuthPasswordFile: passwordFile,
CreateSecretTemplates: true,
},
}
}