wip: guided standalone installation and workspace checks

tools/tht/internal/setup/files.go

@@ -3,6 +3,8 @@ package setup
 import (
 	"bufio"
 	"bytes"
+	"crypto/rand"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"io"
@@ -13,6 +15,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"unicode"
 
 	"github.com/aritmolab/thothii/tools/tht/internal/config"
 	"github.com/aritmolab/thothii/tools/tht/internal/safeio"
@@ -26,6 +29,7 @@ const (
 )
 
 var installationIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]*$`)
+var secretBundleKeyPattern = regexp.MustCompile(`^[A-Z][A-Z0-9_]{0,127}$`)
 
 // atomicWriteNewFile is a seam for failure testing. Its implementation never replaces an existing
 // file and leaves no final target until all content is synced.
@@ -44,6 +48,7 @@ type answers struct {
 	secretsFile, piAuthFile          string
 	gitCredentialsFile, gitCAFile    string
 	gitSSHKeyFile, gitKnownHostsFile string
+	complete                         bool
 	createSecretTemplates            bool
 }
 
@@ -116,6 +121,11 @@ func EnsureFiles(request Request, input io.Reader, output io.Writer) (FilesResul
 	if err := validateOrCreateSecretFiles(values, output); err != nil {
 		return FilesResult{}, err
 	}
+	if values.complete {
+		if err := validateCompleteProtectedFiles(values); err != nil {
+			return FilesResult{}, err
+		}
+	}
 
 	created := make([]string, 0, 2)
 	cleanup := func() {
@@ -163,6 +173,27 @@ func collectAnswers(request Request, input io.Reader, output io.Writer, root str
 	value := answersFromRequest(request)
 	value.installationID = firstNonEmpty(request.InstallationID, os.Getenv("THT_SETUP_INSTALLATION_ID"), "local")
 	value.profile = firstNonEmpty(request.Profile, os.Getenv("THT_SETUP_PROFILE"), "local")
+	value.complete = request.Complete
+	if request.Complete {
+		// The complete path has one predictable protected directory. The user only fills the
+		// bundle and any repository credential that is genuinely required; catalog passwords
+		// are generated below and never appear in the questionnaire.
+		directory := filepath.Join(root, "deploy", value.installationID, "secrets")
+		value.workspaceBranch = firstNonEmpty(value.workspaceBranch, "main")
+		value.secretsFile = firstNonEmpty(value.secretsFile, filepath.Join(directory, "thothii.secrets"))
+		value.piAuthFile = firstNonEmpty(value.piAuthFile, filepath.Join(directory, "pi-auth.json"))
+		if request.NonInteractive {
+			value.workspaceAccess = firstNonEmpty(value.workspaceAccess, accessForRemote(value.workspaceRemote))
+			if value.workspaceAccess == "ssh" {
+				value.gitSSHKeyFile = firstNonEmpty(value.gitSSHKeyFile, filepath.Join(directory, "workspace-git-key"))
+				value.gitKnownHostsFile = firstNonEmpty(value.gitKnownHostsFile, filepath.Join(directory, "workspace-git-known-hosts"))
+			} else {
+				value.gitCredentialsFile = firstNonEmpty(value.gitCredentialsFile, filepath.Join(directory, "workspace-git-credentials"))
+				value.gitCAFile = firstNonEmpty(value.gitCAFile, filepath.Join(directory, "workspace-git-ca.pem"))
+			}
+		}
+		value.createSecretTemplates = true
+	}
 	if request.NonInteractive {
 		return requireNonInteractiveAnswers(value)
 	}
@@ -190,6 +221,18 @@ func collectAnswers(request Request, input io.Reader, output io.Writer, root str
 		return answers{}, err
 	}
 	directory := filepath.Join(root, "deploy", value.installationID, "secrets")
+	if request.Complete {
+		value.secretsFile = firstNonEmpty(value.secretsFile, filepath.Join(directory, "thothii.secrets"))
+		value.piAuthFile = firstNonEmpty(value.piAuthFile, filepath.Join(directory, "pi-auth.json"))
+		if value.workspaceAccess == "ssh" {
+			value.gitSSHKeyFile = firstNonEmpty(value.gitSSHKeyFile, filepath.Join(directory, "workspace-git-key"))
+			value.gitKnownHostsFile = firstNonEmpty(value.gitKnownHostsFile, filepath.Join(directory, "workspace-git-known-hosts"))
+		} else {
+			value.gitCredentialsFile = firstNonEmpty(value.gitCredentialsFile, filepath.Join(directory, "workspace-git-credentials"))
+			value.gitCAFile = firstNonEmpty(value.gitCAFile, filepath.Join(directory, "workspace-git-ca.pem"))
+		}
+		return value, nil
+	}
 	if value.secretsFile, err = prompt(scanner, output, "Secret file location", firstNonEmpty(value.secretsFile, filepath.Join(directory, "thothii.secrets"))); err != nil {
 		return answers{}, err
 	}
@@ -216,11 +259,15 @@ func collectAnswers(request Request, input io.Reader, output io.Writer, root str
 		return answers{}, missingErr
 	}
 	if len(missing) > 0 {
-		answer, promptErr := prompt(scanner, output, "Create blank secret-file templates for the missing locations? Type yes to confirm", "no")
-		if promptErr != nil {
-			return answers{}, promptErr
+		if request.Complete {
+			value.createSecretTemplates = true
+		} else {
+			answer, promptErr := prompt(scanner, output, "Create blank secret-file templates for the missing locations? Type yes to confirm", "no")
+			if promptErr != nil {
+				return answers{}, promptErr
+			}
+			value.createSecretTemplates = strings.EqualFold(answer, "yes")
 		}
-		value.createSecretTemplates = strings.EqualFold(answer, "yes")
 	}
 	return value, nil
 }
@@ -379,6 +426,13 @@ func render(root, descriptorPath string, value answers) ([]byte, []byte, error)
 	if value.llmURL != "" {
 		lines = append(lines, "THT_LLM_URL="+dotenvValue(value.llmURL))
 	}
+	if value.complete {
+		passwordDirectory := filepath.Dir(value.secretsFile)
+		lines = append(lines,
+			"THT_CATALOG_RUNTIME_PASSWORD_SOURCE="+dotenvValue(filepath.Join(passwordDirectory, "catalog-runtime-password")),
+			"THT_CATALOG_MIGRATOR_PASSWORD_SOURCE="+dotenvValue(filepath.Join(passwordDirectory, "catalog-migrator-password")),
+		)
+	}
 	if value.profile == "server" {
 		installationDirectory := filepath.Dir(descriptorPath)
 		lines = append(lines,
@@ -474,10 +528,7 @@ func validateOrCreateSecretFiles(value answers, output io.Writer) error {
 	if err != nil {
 		return err
 	}
-	if len(missing) == 0 {
-		return nil
-	}
-	if !value.createSecretTemplates {
+	if len(missing) > 0 && !value.createSecretTemplates {
 		return fmt.Errorf("secret files are missing: %s; create them yourself or explicitly confirm blank secret-file templates", strings.Join(missing, ", "))
 	}
 	for _, path := range missing {
@@ -489,6 +540,104 @@ func validateOrCreateSecretFiles(value answers, output io.Writer) error {
 		}
 		fmt.Fprintf(output, "Created blank secret-file template: %s\n", path)
 	}
+	if value.complete {
+		for _, path := range catalogPasswordPaths(value) {
+			exists, err := inspectExistingSecretFile(path)
+			if err != nil {
+				return err
+			}
+			if exists {
+				continue
+			}
+			contents, err := generatedCatalogPassword()
+			if err != nil {
+				return fmt.Errorf("generate catalog password: %w", err)
+			}
+			if err := atomicWriteNewFile(path, contents, 0o600); err != nil {
+				return fmt.Errorf("create catalog password %s: %w", path, err)
+			}
+			fmt.Fprintf(output, "Created generated catalog password file: %s\n", path)
+		}
+	}
+	return nil
+}
+
+func catalogPasswordPaths(value answers) []string {
+	directory := filepath.Dir(value.secretsFile)
+	return []string{
+		filepath.Join(directory, "catalog-runtime-password"),
+		filepath.Join(directory, "catalog-migrator-password"),
+	}
+}
+
+func generatedCatalogPassword() ([]byte, error) {
+	value := make([]byte, 32)
+	if _, err := rand.Read(value); err != nil {
+		return nil, errors.New("secure random source is unavailable")
+	}
+	return []byte(hex.EncodeToString(value) + "\n"), nil
+}
+
+func validateCompleteProtectedFiles(value answers) error {
+	if err := validateSecretBundle(value.secretsFile); err != nil {
+		return err
+	}
+	// The generated catalog deliberately uses Pi's built-in provider. A syntactically empty
+	// auth store would let Docker start only to fail at the first provider check, so catch it
+	// before any image is built. Other model providers can be selected later in the descriptor.
+	contents, err := safeio.ReadCanonicalRegular(value.piAuthFile, maxSecretBytes)
+	if err != nil || strings.TrimSpace(string(contents)) == "" || strings.TrimSpace(string(contents)) == "{}" {
+		return fmt.Errorf("complete setup requires usable Pi credentials in %s", value.piAuthFile)
+	}
+	if value.workspaceAccess == "ssh" {
+		for name, path := range map[string]string{
+			"workspace Git SSH key":     value.gitSSHKeyFile,
+			"workspace Git known-hosts": value.gitKnownHostsFile,
+		} {
+			contents, readErr := safeio.ReadCanonicalRegular(path, maxSecretBytes)
+			if readErr != nil || strings.TrimSpace(string(contents)) == "" {
+				return fmt.Errorf("complete setup requires usable %s in %s", name, path)
+			}
+		}
+	} else {
+		for name, path := range map[string]string{
+			"workspace Git credentials": value.gitCredentialsFile,
+			"workspace Git CA":          value.gitCAFile,
+		} {
+			contents, readErr := safeio.ReadCanonicalRegular(path, maxSecretBytes)
+			if readErr != nil || strings.TrimSpace(string(contents)) == "" {
+				return fmt.Errorf("complete setup requires usable %s in %s", name, path)
+			}
+		}
+	}
+	return nil
+}
+
+func validateSecretBundle(path string) error {
+	contents, err := safeio.ReadCanonicalRegular(path, maxSecretBytes)
+	if err != nil {
+		return fmt.Errorf("complete setup cannot read the secret bundle %s", path)
+	}
+	seen := make(map[string]struct{})
+	for lineNumber, raw := range strings.Split(string(contents), "\n") {
+		line := strings.TrimSuffix(raw, "\r")
+		trimmed := strings.TrimSpace(line)
+		if trimmed == "" || strings.HasPrefix(trimmed, "#") {
+			continue
+		}
+		key, secret, found := strings.Cut(line, "=")
+		invalid := !found || !secretBundleKeyPattern.MatchString(key) || strings.TrimSpace(key) != key ||
+			secret == "" || strings.TrimSpace(secret) != secret ||
+			strings.Contains(strings.ToLower(secret), "replace-me") ||
+			strings.IndexFunc(secret, unicode.IsSpace) >= 0
+		if invalid {
+			return fmt.Errorf("complete setup found an invalid secret bundle entry at line %d", lineNumber+1)
+		}
+		if _, duplicate := seen[key]; duplicate {
+			return fmt.Errorf("complete setup found a duplicate secret bundle key %s", key)
+		}
+		seen[key] = struct{}{}
+	}
 	return nil
 }
 

tools/tht/internal/setup/files_test.go

@@ -207,6 +207,50 @@ func TestEnsureFilesRequiresExplicitNonInteractiveAnswers(t *testing.T) {
 	}
 }
 
+func TestCompleteSetupCreatesProtectedPlaceholdersThenRequiresUsableCredentials(t *testing.T) {
+	root := newProject(t, "complete setup")
+	for name, value := range map[string]string{
+		"THT_SETUP_WORKSPACE_REMOTE": "git@git.example.invalid:team/workspaces.git",
+		"THT_SETUP_WORKSPACE_BRANCH": "main",
+		"THT_SETUP_WORKSPACE_ACCESS": "ssh",
+	} {
+		t.Setenv(name, value)
+	}
+	request := Request{ProjectRoot: root, InstallationID: "local", Profile: "local", Complete: true, NonInteractive: true}
+	if _, err := EnsureFiles(request, strings.NewReader(""), ioDiscard{}); err == nil || !strings.Contains(err.Error(), "usable Pi credentials") {
+		t.Fatalf("first complete setup error = %v, want the placeholder guidance", err)
+	}
+	secretRoot := filepath.Join(root, "deploy", "local", "secrets")
+	for path, contents := range map[string]string{
+		filepath.Join(secretRoot, "pi-auth.json"):              "{\"deepseek\":{\"apiKey\":\"configured\"}}\n",
+		filepath.Join(secretRoot, "workspace-git-key"):         "private-key\n",
+		filepath.Join(secretRoot, "workspace-git-known-hosts"): "git.example.invalid ssh-ed25519 AAAA\n",
+	} {
+		if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	result, err := EnsureFiles(request, strings.NewReader(""), ioDiscard{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	environment, err := os.ReadFile(result.EnvironmentPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, name := range []string{"THT_CATALOG_RUNTIME_PASSWORD_SOURCE", "THT_CATALOG_MIGRATOR_PASSWORD_SOURCE"} {
+		if !strings.Contains(string(environment), name+"=") {
+			t.Fatalf("complete environment misses %s: %s", name, environment)
+		}
+	}
+	for _, name := range []string{"catalog-runtime-password", "catalog-migrator-password"} {
+		contents, readErr := os.ReadFile(filepath.Join(secretRoot, name))
+		if readErr != nil || len(strings.TrimSpace(string(contents))) < 32 {
+			t.Fatalf("generated catalog password %s is unavailable or too short", name)
+		}
+	}
+}
+
 func TestEnsureFilesIncludesServerStorageLocations(t *testing.T) {
 	requireProjectedServerTestHost(t)
 	root := newProject(t, "server profile")

tools/tht/internal/setup/request.go

@@ -1,12 +1,15 @@
 // Package setup creates the local, non-secret configuration selected by tht setup.
 package setup
 
-// Request contains the stable setup-file inputs. Task 5 will use ConfigureOnly when it adds
-// Compose validation and lifecycle orchestration.
+// Request contains the stable setup-file inputs.
 type Request struct {
 	ProjectRoot    string
 	InstallationID string
 	Profile        string
+	// Complete runs the installation-only steps that are safe to automate: catalog migration,
+	// stack startup, and the initial workspace pull. It intentionally does not invent database
+	// bindings or credentials that belong to the installation operator.
+	Complete       bool
 	ConfigureOnly  bool
 	NonInteractive bool
 	Answers        Answers

tools/tht/internal/setup/run.go

@@ -3,6 +3,7 @@ package setup
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -35,7 +36,8 @@ type Result struct {
 }
 
 // Run validates the host, creates or validates non-secret configuration, and by default builds,
-// starts, and verifies the current checkout. ConfigureOnly stops after Compose rendering.
+// starts, and verifies the current checkout. Complete additionally migrates the Catalog and
+// imports the configured workspace repository. ConfigureOnly stops after Compose rendering.
 func Run(ctx context.Context, runner compose.Runner, request Request, input io.Reader, output io.Writer) (Result, error) {
 	if runner == nil {
 		return Result{}, errors.New("setup requires a Docker command runner")
@@ -71,13 +73,33 @@ func Run(ctx context.Context, runner compose.Runner, request Request, input io.R
 		fmt.Fprintf(output, "Configuration is ready: %s\n", result.DescriptorPath)
 		return result, nil
 	}
-	if err := service.Start(ctx, installation, runner, true); err != nil {
+	if request.Complete {
+		if err := runCompose(ctx, runner, installation, "build"); err != nil {
+			return Result{}, fmt.Errorf("setup image build: %w", err)
+		}
+		if err := runCompose(ctx, runner, installation, "up", "--detach", "catalog-db"); err != nil {
+			return Result{}, fmt.Errorf("setup Catalog database start: %w", err)
+		}
+		if err := runCompose(ctx, runner, installation,
+			"--profile", "catalog-maintenance", "run", "--rm", "catalog-migrate"); err != nil {
+			return Result{}, fmt.Errorf("setup Catalog migration: %w", err)
+		}
+	}
+	if err := service.Start(ctx, installation, runner, !request.Complete); err != nil {
 		if strings.Contains(err.Error(), "image build") {
 			return Result{}, fmt.Errorf("setup %w", err)
 		}
 		return Result{}, withStartupRecovery(fmt.Errorf("setup %w", err), recoveryService(err))
 	}
 	result.Built, result.Started, result.Healthy = true, true, true
+	if request.Complete {
+		if err := runOperator(ctx, runner, installation, "workspace-pull"); err != nil {
+			return Result{}, withStartupRecovery(fmt.Errorf("setup workspace import: %w", err), "core")
+		}
+		if err := runOperator(ctx, runner, installation, "pi-test"); err != nil {
+			return Result{}, withStartupRecovery(fmt.Errorf("setup LLM credential test: %w", err), "core")
+		}
+	}
 	report, err := doctor.Run(ctx, installation, runner)
 	if err != nil {
 		return Result{}, withStartupRecovery(fmt.Errorf("setup doctor: %w", err), "core")
@@ -85,6 +107,9 @@ func Run(ctx context.Context, runner compose.Runner, request Request, input io.R
 	if !report.OK {
 		return Result{}, withStartupRecovery(errors.New("setup doctor reported failed checks"), "core")
 	}
+	if request.Complete {
+		fmt.Fprintln(output, "Workspace repository pulled and activated; run 'tht workspace test' after configuring each workspace database.")
+	}
 	fmt.Fprintf(output, "ThothII is ready at %s\nInstallation descriptor: %s\nNext: tht status\n", frontendURL(installation), result.DescriptorPath)
 	return result, nil
 }
@@ -219,6 +244,25 @@ func runCompose(ctx context.Context, runner compose.Runner, installation config.
 	return nil
 }
 
+func runOperator(ctx context.Context, runner compose.Runner, installation config.Installation, action string) error {
+	result, err := runner.Run(ctx, installation.ComposeArgs(
+		"exec", "-T", "core", "node", "dist/operator-command.js", action,
+	), nil)
+	if err != nil {
+		if result.ExitCode != 0 {
+			return fmt.Errorf("Docker exited with status %d", result.ExitCode)
+		}
+		return err
+	}
+	var payload struct {
+		Ready *bool `json:"ready"`
+	}
+	if err := json.Unmarshal([]byte(result.Stdout), &payload); err != nil || payload.Ready == nil || !*payload.Ready {
+		return fmt.Errorf("operator action %s reported failure", action)
+	}
+	return nil
+}
+
 func composeFailure(result compose.Result, cause error) error {
 	if result.ExitCode != 0 {
 		return fmt.Errorf("Docker exited with status %d", result.ExitCode)

tools/tht/internal/setup/run_test.go

@@ -68,6 +68,34 @@ func TestRunConfigureOnlyStopsAfterRenderedConfiguration(t *testing.T) {
 	}
 }
 
+func TestRunCompleteMigratesCatalogPullsWorkspaceAndTestsLLM(t *testing.T) {
+	root, request := setupRunFixture(t, false)
+	request.Complete = true
+	secretRoot := filepath.Join(root, "deploy", "ci", "secrets")
+	for path, contents := range map[string]string{
+		filepath.Join(secretRoot, "pi-auth.json"):              "{\"deepseek\":{\"apiKey\":\"configured\"}}\n",
+		filepath.Join(secretRoot, "workspace-git-key"):         "private-key\n",
+		filepath.Join(secretRoot, "workspace-git-known-hosts"): "git.example.invalid ssh-ed25519 AAAA\n",
+	} {
+		if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	runner := &setupRunner{health: []string{healthyServicesJSON}}
+	if _, err := Run(context.Background(), runner, request, strings.NewReader(""), io.Discard); err != nil {
+		t.Fatal(err)
+	}
+	want := []string{
+		"docker engine", "docker compose", "architecture", "compose config", "compose build",
+		"catalog db", "catalog migrate", "compose up", "health", "workspace pull", "pi doctor",
+		"doctor docker", "doctor compose", "compose config", "doctor config", "health",
+		"authentication", "core HTTP", "frontend HTTP", "workspace registry", "workflow doctor", "pi doctor",
+	}
+	if got := collapseStages(runner.stages); strings.Join(got, " | ") != strings.Join(want, " | ") {
+		t.Fatalf("complete setup stages = %v, want %v", got, want)
+	}
+}
+
 func TestRunConfiguresAndStaticallyValidatesLocalAuthBeforeComposeRender(t *testing.T) {
 	projectRoot, request := setupRunFixture(t, true)
 	passwordFile := filepath.Join(projectRoot, "initial-admin-password")
@@ -421,6 +449,10 @@ func setupStage(args []string) (string, compose.Result) {
 		return "architecture", compose.Result{Stdout: "arm64\n"}
 	case strings.HasSuffix(joined, " config --quiet"):
 		return "compose config", compose.Result{}
+	case strings.HasSuffix(joined, " up --detach catalog-db"):
+		return "catalog db", compose.Result{}
+	case strings.HasSuffix(joined, " --profile catalog-maintenance run --rm catalog-migrate"):
+		return "catalog migrate", compose.Result{}
 	case strings.HasSuffix(joined, " build"):
 		return "compose build", compose.Result{}
 	case strings.HasSuffix(joined, " up --detach --remove-orphans"):
@@ -433,6 +465,8 @@ func setupStage(args []string) (string, compose.Result) {
 		return "authentication", compose.Result{Stdout: `{"ready":true,"mode":"oidc","checks":[{"level":"info","code":"auth_ready","message":"Authentication is ready."}]}`}
 	case strings.Contains(joined, "exec -T core node dist/operator-command.js workflow-doctor"):
 		return "workflow doctor", compose.Result{Stdout: `{"ready":true,"workspaces":1}`}
+	case strings.Contains(joined, "operator-command.js workspace-pull"):
+		return "workspace pull", compose.Result{Stdout: `{"ready":true,"status":"succeeded"}`}
 	case strings.Contains(joined, "exec -T core curl -fsS --max-time 5 http://127.0.0.1:8787/health"):
 		return "core HTTP", compose.Result{}
 	case strings.Contains(joined, "exec -T frontend wget -q -T 5 -O /dev/null http://127.0.0.1:8080/"):