ThothII/tools/tht/internal/setup/files.go

package setup

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/aritmolab/thothii/tools/tht/internal/safeio"
	"gopkg.in/yaml.v3"
)

const (
	descriptorName  = "thothii-installation.yaml"
	environmentName = "operator.env"
	maxSecretBytes  = 64 << 10
)

var installationIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]*$`)

// atomicWriteNewFile is a seam for failure testing. Its implementation never replaces an existing
// file and leaves no final target until all content is synced.
var atomicWriteNewFile = writeNewFileAtomically

type answers struct {
	installationID, profile          string
	workspaceRemote, workspaceBranch string
	workspaceAccess                  string
	dwhRESTURL, llmURL               string
	secretsFile, piAuthFile          string
	gitCredentialsFile, gitCAFile    string
	gitSSHKeyFile, gitKnownHostsFile string
	createSecretTemplates            bool
}

type generatedDescriptor struct {
	Profile          string `yaml:"profile"`
	ProjectDirectory string `yaml:"projectDirectory"`
	EnvFile          string `yaml:"envFile"`
	Workspace        struct {
		Remote string `yaml:"remote"`
		Branch string `yaml:"branch"`
		Access string `yaml:"access"`
	} `yaml:"workspaceRepository"`
	Authentication struct {
		ConfigDirectory string `yaml:"configDirectory"`
	} `yaml:"authentication"`
	Overrides []string `yaml:"overrides"`
}

// EnsureFiles writes a descriptor and non-secret environment file below deploy/<installation-id>.
// Existing files are accepted only when their bytes exactly match the requested configuration.
func EnsureFiles(request Request, input io.Reader, output io.Writer) (FilesResult, error) {
	root, err := canonicalProjectRoot(request.ProjectRoot)
	if err != nil {
		return FilesResult{}, err
	}
	values, err := collectAnswers(request, input, output, root)
	if err != nil {
		return FilesResult{}, err
	}
	if err := validateAnswers(values); err != nil {
		return FilesResult{}, err
	}

	directory, err := installationDirectory(root, values.installationID)
	if err != nil {
		return FilesResult{}, err
	}
	descriptorPath := filepath.Join(directory, descriptorName)
	environmentPath := filepath.Join(directory, environmentName)
	if err := safeio.EnsurePrivateDirectory(filepath.Join(directory, "auth")); err != nil {
		return FilesResult{}, errors.New("authentication directory is unavailable or unsafe")
	}
	result := FilesResult{DescriptorPath: descriptorPath, EnvironmentPath: environmentPath}

	descriptor, environment, err := render(root, descriptorPath, values)
	if err != nil {
		return FilesResult{}, err
	}
	if err := requireCompatibleOrAbsent(descriptorPath, descriptor); err != nil {
		return FilesResult{}, err
	}
	if err := requireCompatibleOrAbsent(environmentPath, environment); err != nil {
		return FilesResult{}, err
	}
	if err := validateOrCreateSecretFiles(values, output); err != nil {
		return FilesResult{}, err
	}

	created := make([]string, 0, 2)
	cleanup := func() {
		for index := len(created) - 1; index >= 0; index-- {
			_ = os.Remove(created[index])
		}
	}
	if err := writeIfAbsent(descriptorPath, descriptor, &created); err != nil {
		cleanup()
		return FilesResult{}, err
	}
	if err := writeIfAbsent(environmentPath, environment, &created); err != nil {
		cleanup()
		return FilesResult{}, err
	}
	result.Created = created
	return result, nil
}

func canonicalProjectRoot(path string) (string, error) {
	if strings.TrimSpace(path) == "" {
		return "", errors.New("setup requires the current ThothII project root")
	}
	if !filepath.IsAbs(path) || filepath.Clean(path) != path {
		return "", errors.New("setup project root must be an absolute canonical path")
	}
	resolved, err := filepath.EvalSymlinks(path)
	if err != nil || resolved != path {
		return "", errors.New("setup project root is unavailable or contains a symlink")
	}
	for _, required := range []string{filepath.Join(path, "compose.yaml"), filepath.Join(path, "deploy")} {
		info, statErr := os.Stat(required)
		if statErr != nil || (filepath.Base(required) == "deploy" && !info.IsDir()) || (filepath.Base(required) != "deploy" && !info.Mode().IsRegular()) {
			return "", errors.New("setup project root is not a ThothII checkout")
		}
	}
	deployInfo, err := os.Lstat(filepath.Join(path, "deploy"))
	if err != nil || deployInfo.Mode()&os.ModeSymlink != 0 {
		return "", errors.New("setup project deployment directory is unavailable or contains a symlink")
	}
	return path, nil
}

func collectAnswers(request Request, input io.Reader, output io.Writer, root string) (answers, error) {
	value := answersFromRequest(request)
	value.installationID = firstNonEmpty(request.InstallationID, os.Getenv("THT_SETUP_INSTALLATION_ID"), "local")
	value.profile = firstNonEmpty(request.Profile, os.Getenv("THT_SETUP_PROFILE"), "local")
	if request.NonInteractive {
		return requireNonInteractiveAnswers(value)
	}
	scanner := bufio.NewScanner(input)
	var err error
	if value.installationID, err = prompt(scanner, output, "Installation ID", value.installationID); err != nil {
		return answers{}, err
	}
	if value.profile, err = prompt(scanner, output, "Deployment profile (local or server)", value.profile); err != nil {
		return answers{}, err
	}
	if value.dwhRESTURL, err = prompt(scanner, output, "DWH API endpoint (optional)", value.dwhRESTURL); err != nil {
		return answers{}, err
	}
	if value.llmURL, err = prompt(scanner, output, "LLM API endpoint (optional)", value.llmURL); err != nil {
		return answers{}, err
	}
	if value.workspaceRemote, err = prompt(scanner, output, "Workspace repository URL", firstNonEmpty(value.workspaceRemote, "https://git.example.invalid/thothii-workspaces.git")); err != nil {
		return answers{}, err
	}
	if value.workspaceBranch, err = prompt(scanner, output, "Workspace repository branch", firstNonEmpty(value.workspaceBranch, "main")); err != nil {
		return answers{}, err
	}
	if value.workspaceAccess, err = prompt(scanner, output, "Workspace repository access (https or ssh)", firstNonEmpty(value.workspaceAccess, accessForRemote(value.workspaceRemote))); err != nil {
		return answers{}, err
	}
	directory := filepath.Join(root, "deploy", value.installationID, "secrets")
	if value.secretsFile, err = prompt(scanner, output, "Secret file location", firstNonEmpty(value.secretsFile, filepath.Join(directory, "thothii.secrets"))); err != nil {
		return answers{}, err
	}
	if value.piAuthFile, err = prompt(scanner, output, "Pi credentials file location", firstNonEmpty(value.piAuthFile, filepath.Join(directory, "pi-auth.json"))); err != nil {
		return answers{}, err
	}
	if value.workspaceAccess == "ssh" {
		if value.gitSSHKeyFile, err = prompt(scanner, output, "Workspace Git SSH key location", firstNonEmpty(value.gitSSHKeyFile, filepath.Join(directory, "workspace-git-key"))); err != nil {
			return answers{}, err
		}
		if value.gitKnownHostsFile, err = prompt(scanner, output, "Workspace Git known-hosts location", firstNonEmpty(value.gitKnownHostsFile, filepath.Join(directory, "workspace-git-known-hosts"))); err != nil {
			return answers{}, err
		}
	} else {
		if value.gitCredentialsFile, err = prompt(scanner, output, "Workspace Git credentials file location", firstNonEmpty(value.gitCredentialsFile, filepath.Join(directory, "workspace-git-credentials"))); err != nil {
			return answers{}, err
		}
		if value.gitCAFile, err = prompt(scanner, output, "Workspace Git CA file location", firstNonEmpty(value.gitCAFile, filepath.Join(directory, "workspace-git-ca.pem"))); err != nil {
			return answers{}, err
		}
	}
	missing, missingErr := missingSecretFiles(value)
	if missingErr != nil {
		return answers{}, missingErr
	}
	if len(missing) > 0 {
		answer, promptErr := prompt(scanner, output, "Create blank secret-file templates for the missing locations? Type yes to confirm", "no")
		if promptErr != nil {
			return answers{}, promptErr
		}
		value.createSecretTemplates = strings.EqualFold(answer, "yes")
	}
	return value, nil
}

func answersFromRequest(request Request) answers {
	answer := request.Answers
	return answers{
		workspaceRemote:       firstNonEmpty(answer.WorkspaceRemote, os.Getenv("THT_SETUP_WORKSPACE_REMOTE")),
		workspaceBranch:       firstNonEmpty(answer.WorkspaceBranch, os.Getenv("THT_SETUP_WORKSPACE_BRANCH")),
		workspaceAccess:       firstNonEmpty(answer.WorkspaceAccess, os.Getenv("THT_SETUP_WORKSPACE_ACCESS")),
		dwhRESTURL:            firstNonEmpty(answer.DWHRESTURL, os.Getenv("THT_SETUP_DWH_REST_URL")),
		llmURL:                firstNonEmpty(answer.LLMURL, os.Getenv("THT_SETUP_LLM_URL")),
		secretsFile:           firstNonEmpty(answer.SecretsFile, os.Getenv("THT_SETUP_SECRETS_FILE")),
		piAuthFile:            firstNonEmpty(answer.PiAuthFile, os.Getenv("THT_SETUP_PI_AUTH_FILE")),
		gitCredentialsFile:    firstNonEmpty(answer.GitCredentialsFile, os.Getenv("THT_SETUP_GIT_CREDENTIALS_FILE")),
		gitCAFile:             firstNonEmpty(answer.GitCAFile, os.Getenv("THT_SETUP_GIT_CA_FILE")),
		gitSSHKeyFile:         firstNonEmpty(answer.GitSSHKeyFile, os.Getenv("THT_SETUP_GIT_SSH_KEY_FILE")),
		gitKnownHostsFile:     firstNonEmpty(answer.GitKnownHostsFile, os.Getenv("THT_SETUP_GIT_KNOWN_HOSTS_FILE")),
		createSecretTemplates: answer.CreateSecretTemplates,
	}
}

func requireNonInteractiveAnswers(value answers) (answers, error) {
	required := []struct{ name, value string }{
		{"THT_SETUP_WORKSPACE_REMOTE", value.workspaceRemote}, {"THT_SETUP_WORKSPACE_BRANCH", value.workspaceBranch},
		{"THT_SETUP_WORKSPACE_ACCESS", value.workspaceAccess}, {"THT_SETUP_SECRETS_FILE", value.secretsFile}, {"THT_SETUP_PI_AUTH_FILE", value.piAuthFile},
	}
	if value.workspaceAccess == "ssh" {
		required = append(required, struct{ name, value string }{"THT_SETUP_GIT_SSH_KEY_FILE", value.gitSSHKeyFile}, struct{ name, value string }{"THT_SETUP_GIT_KNOWN_HOSTS_FILE", value.gitKnownHostsFile})
	} else if value.workspaceAccess == "https" {
		required = append(required, struct{ name, value string }{"THT_SETUP_GIT_CREDENTIALS_FILE", value.gitCredentialsFile}, struct{ name, value string }{"THT_SETUP_GIT_CA_FILE", value.gitCAFile})
	}
	for _, requiredValue := range required {
		if strings.TrimSpace(requiredValue.value) == "" {
			return answers{}, fmt.Errorf("non-interactive setup requires %s or its matching setup flag", requiredValue.name)
		}
	}
	return value, nil
}

func prompt(scanner *bufio.Scanner, output io.Writer, question, defaultValue string) (string, error) {
	fmt.Fprintf(output, "%s [%s]: ", question, defaultValue)
	if !scanner.Scan() {
		return "", fmt.Errorf("setup input ended while waiting for %s", strings.ToLower(question))
	}
	value := strings.TrimSpace(scanner.Text())
	if value == "" {
		return defaultValue, nil
	}
	return value, nil
}

func validateAnswers(value answers) error {
	if !installationIDPattern.MatchString(value.installationID) {
		return errors.New("installation ID must contain only letters, numbers, dashes, and underscores")
	}
	if value.profile != "local" && value.profile != "server" {
		return errors.New("deployment profile must be local or server")
	}
	if value.workspaceAccess != "ssh" && value.workspaceAccess != "https" {
		return errors.New("workspace repository access must be ssh or https")
	}
	for name, endpoint := range map[string]string{"DWH API": value.dwhRESTURL, "LLM API": value.llmURL} {
		if err := validateServiceEndpoint(name, endpoint); err != nil {
			return err
		}
	}
	for name, path := range map[string]string{
		"secret file location": value.secretsFile, "Pi credentials file location": value.piAuthFile,
		"workspace Git credentials file location": value.gitCredentialsFile, "workspace Git CA file location": value.gitCAFile,
		"workspace Git SSH key location": value.gitSSHKeyFile, "workspace Git known-hosts location": value.gitKnownHostsFile,
	} {
		if path == "" && ((value.workspaceAccess == "ssh" && (name == "workspace Git credentials file location" || name == "workspace Git CA file location")) || (value.workspaceAccess == "https" && (name == "workspace Git SSH key location" || name == "workspace Git known-hosts location"))) {
			continue
		}
		if err := safeio.ValidateCanonicalPath(path); err != nil {
			return fmt.Errorf("%s must be an absolute canonical path", name)
		}
	}
	return nil
}

func installationDirectory(root, id string) (string, error) {
	directory := filepath.Join(root, "deploy", id)
	if err := safeio.ValidateCanonicalPath(directory); err != nil {
		return "", errors.New("installation directory is unsafe")
	}
	if info, err := os.Lstat(directory); err == nil {
		if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
			return "", fmt.Errorf("installation directory %s is unavailable or unsafe", directory)
		}
		return directory, nil
	} else if !errors.Is(err, os.ErrNotExist) {
		return "", fmt.Errorf("installation directory %s could not be inspected", directory)
	}
	if err := os.Mkdir(directory, 0o700); err != nil {
		return "", fmt.Errorf("create installation directory %s: %w", directory, err)
	}
	return directory, nil
}

func render(root, descriptorPath string, value answers) ([]byte, []byte, error) {
	descriptor := generatedDescriptor{Profile: value.profile, ProjectDirectory: root, EnvFile: filepath.Join(filepath.Dir(descriptorPath), environmentName)}
	descriptor.Workspace.Remote, descriptor.Workspace.Branch, descriptor.Workspace.Access = value.workspaceRemote, value.workspaceBranch, value.workspaceAccess
	descriptor.Authentication.ConfigDirectory = filepath.Join(filepath.Dir(descriptorPath), "auth")
	descriptor.Overrides = []string{filepath.Join(root, "deploy", "compose.git-"+value.workspaceAccess+".yaml")}
	descriptorBytes, err := yaml.Marshal(descriptor)
	if err != nil {
		return nil, nil, err
	}
	lines := []string{
		"# Generated by tht setup. This file contains locations, never secret values.",
		"THT_WORKSPACE_GIT_REMOTE=" + dotenvValue(value.workspaceRemote),
		"THT_WORKSPACE_GIT_BRANCH=" + dotenvValue(value.workspaceBranch),
		"THT_WORKSPACE_INSTALLATION_ID=" + dotenvValue(value.installationID),
		"THT_AUTH_CONFIG_ROOT=" + dotenvValue(descriptor.Authentication.ConfigDirectory),
		"THT_SECRETS_FILE=" + dotenvValue(value.secretsFile),
		"PI_AUTH_FILE=" + dotenvValue(value.piAuthFile),
		"THOTH_HTTP_PORT=8080", "THOTH_CORE_HTTP_PORT=8787", "MAX_PI_PROCESSES=4",
	}
	if value.workspaceAccess == "ssh" {
		lines = append(lines, "THT_WORKSPACE_GIT_SSH_KEY_FILE="+dotenvValue(value.gitSSHKeyFile), "THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE="+dotenvValue(value.gitKnownHostsFile))
	} else {
		lines = append(lines, "THT_WORKSPACE_GIT_CREDENTIALS_FILE="+dotenvValue(value.gitCredentialsFile), "THT_WORKSPACE_GIT_CA_FILE="+dotenvValue(value.gitCAFile))
	}
	if value.dwhRESTURL != "" {
		lines = append(lines, "THT_DWH_REST_URL="+dotenvValue(value.dwhRESTURL))
	}
	if value.llmURL != "" {
		lines = append(lines, "THT_LLM_URL="+dotenvValue(value.llmURL))
	}
	if value.profile == "server" {
		installationDirectory := filepath.Dir(descriptorPath)
		lines = append(lines,
			"THT_DATA_ROOT="+dotenvValue(filepath.Join(installationDirectory, "data")),
			"THT_PI_STATE_ROOT="+dotenvValue(filepath.Join(installationDirectory, "pi-state")),
			"THT_WORKSPACE_REGISTRY_ROOT="+dotenvValue(filepath.Join(installationDirectory, "workspace-registry")),
			"THT_BACKUP_ROOT="+dotenvValue(filepath.Join(installationDirectory, "backups")),
		)
	}
	return descriptorBytes, []byte(strings.Join(lines, "\n") + "\n"), nil
}

func dotenvValue(value string) string { return strconv.Quote(value) }

func requireCompatibleOrAbsent(path string, expected []byte) error {
	info, err := os.Lstat(path)
	if errors.Is(err, os.ErrNotExist) {
		return nil
	}
	if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
		return fmt.Errorf("configuration file %s is unsafe; choose a different installation ID or remove the unsafe file", path)
	}
	actual, err := os.ReadFile(path)
	if err != nil || !bytes.Equal(actual, expected) {
		return fmt.Errorf("configuration file %s already exists with different content; choose a different installation ID or move that file before running setup", path)
	}
	return nil
}

func writeIfAbsent(path string, contents []byte, created *[]string) error {
	if _, err := os.Lstat(path); err == nil {
		if err := requireCompatibleOrAbsent(path, contents); err != nil {
			return err
		}
		return nil
	} else if !errors.Is(err, os.ErrNotExist) {
		return fmt.Errorf("inspect configuration file %s: %w", path, err)
	}
	if err := atomicWriteNewFile(path, contents, 0o600); err != nil {
		return fmt.Errorf("write configuration file %s: %w", path, err)
	}
	*created = append(*created, path)
	return nil
}

func validateServiceEndpoint(name, endpoint string) error {
	if endpoint == "" {
		return nil
	}
	parsed, err := url.Parse(endpoint)
	if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" ||
		parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" {
		return fmt.Errorf("%s endpoint must be an http(s) URL without user information, password, query, or fragment", name)
	}
	return nil
}

func missingSecretFiles(value answers) ([]string, error) {
	paths := configuredSecretPaths(value)
	missing := make([]string, 0, len(paths))
	for _, path := range paths {
		exists, err := inspectExistingSecretFile(path)
		if err != nil {
			return nil, err
		}
		if !exists {
			missing = append(missing, path)
		}
	}
	sort.Strings(missing)
	return missing, nil
}

func validateOrCreateSecretFiles(value answers, output io.Writer) error {
	missing, err := missingSecretFiles(value)
	if err != nil {
		return err
	}
	if len(missing) == 0 {
		return nil
	}
	if !value.createSecretTemplates {
		return fmt.Errorf("secret files are missing: %s; create them yourself or explicitly confirm blank secret-file templates", strings.Join(missing, ", "))
	}
	for _, path := range missing {
		if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
			return fmt.Errorf("create secret-template directory: %w", err)
		}
		if err := atomicWriteNewFile(path, secretTemplate(path), 0o600); err != nil {
			return fmt.Errorf("create secret-file template %s: %w", path, err)
		}
		fmt.Fprintf(output, "Created blank secret-file template: %s\n", path)
	}
	return nil
}

func inspectExistingSecretFile(path string) (bool, error) {
	before, err := os.Lstat(path)
	if errors.Is(err, os.ErrNotExist) {
		return false, nil
	}
	if err != nil {
		return false, fmt.Errorf("secret file %s could not be inspected; choose a readable regular file", path)
	}
	if !before.Mode().IsRegular() || before.Mode()&os.ModeSymlink != 0 {
		return false, fmt.Errorf("secret file %s must be a readable regular file, not a directory, symlink, or special file", path)
	}
	if _, err := safeio.ReadCanonicalRegular(path, maxSecretBytes); err != nil {
		return false, fmt.Errorf("secret file %s must be a canonical readable regular file", path)
	}
	return true, nil
}

func configuredSecretPaths(value answers) []string {
	paths := []string{value.secretsFile, value.piAuthFile}
	if value.workspaceAccess == "ssh" {
		paths = append(paths, value.gitSSHKeyFile, value.gitKnownHostsFile)
	} else {
		paths = append(paths, value.gitCredentialsFile, value.gitCAFile)
	}
	return paths
}

func secretTemplate(path string) []byte {
	if strings.HasSuffix(path, ".json") {
		return []byte("{}\n")
	}
	return []byte("# Add the required credential value to this protected local file.\n")
}

func writeNewFileAtomically(path string, contents []byte, mode os.FileMode) error {
	if err := safeio.ValidateCanonicalPath(path); err != nil {
		return err
	}
	directory := filepath.Dir(path)
	if info, err := os.Lstat(directory); err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
		return safeio.ErrUnsafeFile
	}
	resolvedDirectory, err := filepath.EvalSymlinks(directory)
	if err != nil || resolvedDirectory != directory {
		return safeio.ErrUnsafeFile
	}
	temporary, err := os.CreateTemp(directory, ".tht-setup-*")
	if err != nil {
		return err
	}
	temporaryPath := temporary.Name()
	defer os.Remove(temporaryPath)
	if err := temporary.Chmod(mode); err != nil {
		temporary.Close()
		return err
	}
	if _, err := temporary.Write(contents); err != nil {
		temporary.Close()
		return err
	}
	if err := temporary.Sync(); err != nil {
		temporary.Close()
		return err
	}
	if err := temporary.Close(); err != nil {
		return err
	}
	if err := os.Link(temporaryPath, path); err != nil {
		return err
	}
	directoryFile, err := os.Open(directory)
	if err == nil {
		_ = directoryFile.Sync()
		_ = directoryFile.Close()
	}
	return nil
}

func accessForRemote(remote string) string {
	if strings.HasPrefix(remote, "git@") || strings.HasPrefix(remote, "ssh://") {
		return "ssh"
	}
	return "https"
}

func firstNonEmpty(values ...string) string {
	for _, value := range values {
		if strings.TrimSpace(value) != "" {
			return strings.TrimSpace(value)
		}
	}
	return ""
}