feat(cli): prepare and validate application documents offline
This commit is contained in:
@@ -41,6 +41,40 @@ THT_WORKSPACE_TEST_CLI=/absolute/path/dist/workspace-tools/darwin-arm64/tht \
|
||||
npx vitest run test/workspace-documents-cli.test.ts
|
||||
```
|
||||
|
||||
## Application document preparation (issue #44)
|
||||
|
||||
Before installation, use `tht installation prepare --directory NEW_PATH`, then
|
||||
`tht installation credentials --directory PATH` to explicitly create technical
|
||||
credentials in the default protected layout. The latter preserves existing files.
|
||||
Edit the documents and external credentials, then repeat:
|
||||
|
||||
```sh
|
||||
tht --installation /absolute/path/thothii-installation.yaml \
|
||||
installation validate --workspaces /absolute/path/workspaces --json
|
||||
```
|
||||
|
||||
`database-bootstrap.yaml` beside the descriptor is a schemaVersion-1 bootstrap
|
||||
input, not a runtime Catalog. Its database entries use the exact Catalog API
|
||||
configuration schema plus private `secretFiles` and optional `evidenceSecretFiles`
|
||||
references. `--bootstrap` selects another document. The helper validates workspace
|
||||
membership, uniqueness, supported transports and required credential references;
|
||||
Go checks protected files, the canonical Installation Model Catalog, environment
|
||||
and local administrator. No process contacts a service or creates projections.
|
||||
Only missing standard release Compose assets are deferred by `config.LoadPrepared`;
|
||||
normal runtime `config.Load` retains all existing checks. Custom overrides must exist.
|
||||
|
||||
The native integration test is opt-in because it requires the built pair:
|
||||
|
||||
```sh
|
||||
THT_INSTALLATION_TEST_CLI=/absolute/path/dist/workspace-tools/darwin-arm64/tht \
|
||||
npx vitest run test/installation-documents-cli.test.ts
|
||||
```
|
||||
|
||||
Run it from `backend/`, following the bundle build above. It supplies prepared
|
||||
fixtures and removes host tools from the subprocess PATH. Platform acceptance and
|
||||
real external credentials remain separate gates. See the IT/EN guides for the
|
||||
complete parameter collection procedure, default `admin` account and local-auth scope.
|
||||
|
||||
## Shell configuration
|
||||
|
||||
The schema-v2 `thothii-installation.yaml` accepts this optional section:
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/preparation"
|
||||
)
|
||||
|
||||
func installationDocumentsCommand(ctx context.Context, installationPath string, args []string, stdout io.Writer) int {
|
||||
report := preparation.NewReport()
|
||||
status := 0
|
||||
options := map[string]string{}
|
||||
valid := len(args) > 0
|
||||
for index := 1; index < len(args); index++ {
|
||||
key := args[index]
|
||||
if _, duplicate := options[key]; duplicate {
|
||||
valid = false
|
||||
break
|
||||
}
|
||||
if key == "--json" {
|
||||
options[key] = "true"
|
||||
continue
|
||||
}
|
||||
if (key != "--directory" && key != "--workspaces" && key != "--bootstrap") || index+1 == len(args) {
|
||||
valid = false
|
||||
break
|
||||
}
|
||||
options[key] = args[index+1]
|
||||
index++
|
||||
}
|
||||
if !valid || (args[0] == "validate" && (installationPath == "" || options["--workspaces"] == "" || options["--directory"] != "")) || (args[0] != "validate" && (options["--directory"] == "" || options["--workspaces"] != "" || options["--bootstrap"] != "")) {
|
||||
report.Add("CLI", "$", "usage", "Use installation prepare|credentials --directory PATH [--json], or tht --installation ABSOLUTE_PATH installation validate --workspaces PATH [--bootstrap PATH] [--json].")
|
||||
status = 2
|
||||
} else if args[0] == "validate" {
|
||||
installation, checkedReport := preparation.Validate(installationPath)
|
||||
report = checkedReport
|
||||
if report.OK {
|
||||
bootstrap := options["--bootstrap"]
|
||||
if bootstrap == "" {
|
||||
bootstrap = filepath.Join(filepath.Dir(installationPath), "database-bootstrap.yaml")
|
||||
}
|
||||
bootstrap, _ = filepath.Abs(bootstrap)
|
||||
workspace, _ := filepath.Abs(options["--workspaces"])
|
||||
if resolved, err := filepath.EvalSymlinks(workspace); err == nil {
|
||||
workspace = resolved
|
||||
}
|
||||
outsideWorkspace := func(path, document, field string) bool {
|
||||
relative, err := filepath.Rel(workspace, path)
|
||||
if err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
||||
report.Add(document, field, "installation_file_in_workspace", "Keep installation documents and credential files outside the shared workspace repository.")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
for _, path := range []string{installationPath, installation.EnvFile, installation.AuthenticationDirectory(), bootstrap} {
|
||||
outsideWorkspace(path, "thothii-installation.yaml", "local-files")
|
||||
}
|
||||
files, _ := installation.SecretFiles()
|
||||
for _, path := range files {
|
||||
outsideWorkspace(path, "operator.env", "protected-file-reference")
|
||||
}
|
||||
if preparation.CheckYAML(bootstrap, "database-bootstrap.yaml", &report) {
|
||||
var output, discarded bytes.Buffer
|
||||
code := workspaceDocumentsCommand(ctx, []string{"bootstrap", "--directory", workspace, "--bootstrap", bootstrap, "--json"}, &output, &discarded)
|
||||
var checked struct {
|
||||
OK bool `json:"ok"`
|
||||
Issues []preparation.Issue `json:"issues"`
|
||||
Warnings []string `json:"warnings"`
|
||||
SecretFiles []struct {
|
||||
Field string `json:"field"`
|
||||
Path string `json:"path"`
|
||||
} `json:"secret_files"`
|
||||
}
|
||||
if json.Unmarshal(output.Bytes(), &checked) != nil {
|
||||
report.Add("CLI", "$", "validator_unavailable", "Reinstall the matching tht and workspace helper pair.")
|
||||
} else if code != 0 || !checked.OK {
|
||||
report.Issues = append(report.Issues, checked.Issues...)
|
||||
if len(checked.Issues) == 0 {
|
||||
report.Add("database-bootstrap.yaml", "$", "bootstrap_invalid", "Correct workspace and binding documents, then validate again.")
|
||||
}
|
||||
} else {
|
||||
report.Warnings = append(report.Warnings, checked.Warnings...)
|
||||
for _, file := range checked.SecretFiles {
|
||||
if outsideWorkspace(file.Path, "database-bootstrap.yaml", file.Field) {
|
||||
preparation.CheckSecret(file.Path, "database-bootstrap.yaml", file.Field, false, &report)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
directory, err := filepath.Abs(options["--directory"])
|
||||
if err == nil {
|
||||
if args[0] == "prepare" {
|
||||
err = preparation.Prepare(directory)
|
||||
} else {
|
||||
err = preparation.Credentials(ctx, directory)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
report.Add("preparation", "$", "preparation_refused", err.Error())
|
||||
}
|
||||
}
|
||||
report.OK = len(report.Issues) == 0
|
||||
if !report.OK && status == 0 {
|
||||
status = 1
|
||||
}
|
||||
if slices.Contains(args, "--json") {
|
||||
_ = json.NewEncoder(stdout).Encode(report)
|
||||
} else {
|
||||
if report.OK {
|
||||
fmt.Fprintln(stdout, "Document operation completed. Local validation does not establish runtime readiness.")
|
||||
}
|
||||
for _, issue := range report.Issues {
|
||||
fmt.Fprintf(stdout, "%s [%s] %s: %s\n", issue.Document, issue.Field, issue.Code, issue.Correction)
|
||||
}
|
||||
for _, warning := range report.Warnings {
|
||||
fmt.Fprintln(stdout, warning)
|
||||
}
|
||||
}
|
||||
return status
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestInstallationPrepareDocumentsBeforeRuntime(t *testing.T) {
|
||||
t.Setenv("PATH", "")
|
||||
root, err := filepath.EvalSymlinks(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
destination := filepath.Join(root, "installation")
|
||||
var stdout, stderr bytes.Buffer
|
||||
status := run(context.Background(), []string{"installation", "prepare", "--directory", destination, "--json"}, &stdout, &stderr)
|
||||
if status != 0 {
|
||||
t.Fatalf("status=%d stdout=%s stderr=%s", status, &stdout, &stderr)
|
||||
}
|
||||
var report map[string]any
|
||||
if json.Unmarshal(stdout.Bytes(), &report) != nil || report["ok"] != true {
|
||||
t.Fatalf("report=%s", &stdout)
|
||||
}
|
||||
for _, name := range []string{"thothii-installation.yaml", "operator.env", "database-bootstrap.yaml", "README.md"} {
|
||||
if _, err := os.Stat(filepath.Join(destination, name)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
before, _ := os.ReadFile(filepath.Join(destination, "thothii-installation.yaml"))
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
if run(context.Background(), []string{"installation", "prepare", "--directory", destination, "--json"}, &stdout, &stderr) == 0 {
|
||||
t.Fatal("overwrote existing documents")
|
||||
}
|
||||
after, _ := os.ReadFile(filepath.Join(destination, "thothii-installation.yaml"))
|
||||
if !bytes.Equal(before, after) {
|
||||
t.Fatal("existing document changed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstallationCredentialsAreExplicitPrivateAndNeverReplaced(t *testing.T) {
|
||||
t.Setenv("PATH", "")
|
||||
root, _ := filepath.EvalSymlinks(t.TempDir())
|
||||
destination := filepath.Join(root, "installation")
|
||||
var stdout, stderr bytes.Buffer
|
||||
if run(context.Background(), []string{"installation", "prepare", "--directory", destination}, &stdout, &stderr) != 0 {
|
||||
t.Fatal(&stdout, &stderr)
|
||||
}
|
||||
secret := filepath.Join(destination, "secrets", "catalog-runtime-password")
|
||||
if _, err := os.Stat(secret); !os.IsNotExist(err) {
|
||||
t.Fatal("prepare generated a secret implicitly")
|
||||
}
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
if run(context.Background(), []string{"installation", "credentials", "--directory", destination, "--json"}, &stdout, &stderr) != 0 {
|
||||
t.Fatal(&stdout, &stderr)
|
||||
}
|
||||
before, err := os.ReadFile(secret)
|
||||
if err != nil || len(bytes.TrimSpace(before)) < 32 {
|
||||
t.Fatal("missing strong technical credential", err)
|
||||
}
|
||||
if bytes.Contains(stdout.Bytes(), bytes.TrimSpace(before)) || bytes.Contains(stderr.Bytes(), bytes.TrimSpace(before)) {
|
||||
t.Fatal("secret leaked")
|
||||
}
|
||||
info, _ := os.Stat(secret)
|
||||
if runtime.GOOS != "windows" && info.Mode().Perm()&0o077 != 0 {
|
||||
t.Fatal("credential is not private")
|
||||
}
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
if run(context.Background(), []string{"installation", "credentials", "--directory", destination, "--json"}, &stdout, &stderr) != 0 {
|
||||
t.Fatal(&stdout, &stderr)
|
||||
}
|
||||
after, _ := os.ReadFile(secret)
|
||||
if !bytes.Equal(before, after) {
|
||||
t.Fatal("credential was replaced")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstallationValidateRejectsWrongEnvironmentFieldType(t *testing.T) {
|
||||
root, _ := filepath.EvalSymlinks(t.TempDir())
|
||||
path := filepath.Join(root, "thothii-installation.yaml")
|
||||
if err := os.WriteFile(path, []byte("schemaVersion: 2\nenvFile: []\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
status := run(context.Background(), []string{"--installation", path, "installation", "validate", "--workspaces", root, "--json"}, &stdout, &stderr)
|
||||
if status != 1 {
|
||||
t.Fatalf("invalid field accepted: %d %s", status, &stdout)
|
||||
}
|
||||
}
|
||||
@@ -43,6 +43,12 @@ Commands:
|
||||
setup [--complete|--configure-only] [--installation-id ID] [--profile local|server]
|
||||
[--shell-mode full|embedded] [--shell-default-locale BCP47-TAG] [--shell-adapter omics-portal]
|
||||
Create, validate, and optionally complete the local installation.
|
||||
installation prepare --directory NEW_PATH [--json]
|
||||
Create commented installation, environment and database bootstrap templates.
|
||||
installation credentials --directory PATH [--json]
|
||||
Explicitly generate protected technical credentials before setup.
|
||||
installation validate --workspaces PATH [--bootstrap PATH] [--json]
|
||||
Check prepared application documents; requires --installation.
|
||||
installation migrate --output PATH --session-default PROVIDER/MODEL
|
||||
--embedding-id PROVIDER/MODEL --embedding-dimensions N
|
||||
Create a review-only schema-v2 candidate from all three legacy model sources.
|
||||
@@ -138,6 +144,9 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
||||
return versionCommand(commandArgs, stdout, stderr)
|
||||
}
|
||||
if command == "installation" {
|
||||
if len(commandArgs) > 0 && (commandArgs[0] == "prepare" || commandArgs[0] == "credentials" || commandArgs[0] == "validate") {
|
||||
return installationDocumentsCommand(ctx, installationPath, commandArgs, stdout)
|
||||
}
|
||||
if len(commandArgs) > 0 && commandArgs[0] == "generate" {
|
||||
return installationGenerationCommand(installationPath, commandArgs[1:], stdout, stderr)
|
||||
}
|
||||
|
||||
@@ -98,6 +98,16 @@ type Installation struct {
|
||||
|
||||
// Load reads and validates an installation descriptor at an absolute path.
|
||||
func Load(path string) (Installation, error) {
|
||||
return load(path, false)
|
||||
}
|
||||
|
||||
// LoadPrepared applies the same authored configuration rules before release assets exist.
|
||||
// Only standard distribution-owned Compose assets are deferred, never custom overrides.
|
||||
func LoadPrepared(path string) (Installation, error) {
|
||||
return load(path, true)
|
||||
}
|
||||
|
||||
func load(path string, prepared bool) (Installation, error) {
|
||||
if !filepath.IsAbs(path) {
|
||||
return Installation{}, fmt.Errorf("installation path must be absolute")
|
||||
}
|
||||
@@ -143,6 +153,11 @@ func Load(path string) (Installation, error) {
|
||||
if err := requireDirectory(raw.ProjectDirectory, "projectDirectory"); err != nil {
|
||||
return Installation{}, err
|
||||
}
|
||||
if prepared {
|
||||
if err := requireCanonicalDirectory(raw.ProjectDirectory); err != nil {
|
||||
return Installation{}, errors.New("projectDirectory must be canonical and accessible")
|
||||
}
|
||||
}
|
||||
for _, legacyProjection := range []string{
|
||||
filepath.Join(raw.ProjectDirectory, "deploy", "pi", "models.json"),
|
||||
filepath.Join(raw.ProjectDirectory, "deploy", "pi", "settings.json"),
|
||||
@@ -195,7 +210,8 @@ func Load(path string) (Installation, error) {
|
||||
return Installation{}, errors.New("authentication.configDirectory must match THT_AUTH_CONFIG_ROOT")
|
||||
}
|
||||
for _, override := range raw.Overrides {
|
||||
if err := requireRegularFile(override, "override"); err != nil {
|
||||
standardAsset := override == filepath.Join(installation.ProjectDirectory, "deploy", "compose.git-https.yaml") || override == filepath.Join(installation.ProjectDirectory, "deploy", "compose.git-ssh.yaml")
|
||||
if err := requireRegularFile(override, "override"); err != nil && !(prepared && standardAsset && errors.Is(statPathError(override), os.ErrNotExist)) {
|
||||
return Installation{}, err
|
||||
}
|
||||
installation.Overrides = append(installation.Overrides, filepath.Clean(override))
|
||||
@@ -209,6 +225,9 @@ func Load(path string) (Installation, error) {
|
||||
}
|
||||
}
|
||||
for _, composeFile := range installation.ComposeFiles()[:2] {
|
||||
if prepared && errors.Is(statPathError(composeFile), os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
if err := requireRegularFile(composeFile, "Compose file"); err != nil {
|
||||
return Installation{}, err
|
||||
}
|
||||
@@ -226,6 +245,8 @@ func Load(path string) (Installation, error) {
|
||||
return installation, nil
|
||||
}
|
||||
|
||||
func statPathError(path string) error { _, err := os.Lstat(path); return err }
|
||||
|
||||
var metadataSecretBundleKeyPattern = regexp.MustCompile(`^[A-Z][A-Z0-9_]{0,127}$`)
|
||||
var metadataAPIKeyEnvironments = map[string]struct{}{
|
||||
"THT_MODEL_API_KEY": {},
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package preparation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/authconfig"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
|
||||
)
|
||||
|
||||
// Credentials creates installation-owned secrets only. External credentials are supplied by
|
||||
// the operator. Existing files are checked and retained so interrupted preparation can resume.
|
||||
func Credentials(ctx context.Context, directory string) error {
|
||||
if err := safeio.ValidatePrivateDirectory(directory); err != nil {
|
||||
return fmt.Errorf("use an existing private preparation directory")
|
||||
}
|
||||
if _, err := safeio.ReadCanonicalPrivateRegular(filepath.Join(directory, "thothii-installation.yaml"), 1<<20); err != nil {
|
||||
return fmt.Errorf("prepare installation documents first")
|
||||
}
|
||||
secrets := filepath.Join(directory, "secrets")
|
||||
if err := safeio.EnsurePrivateDirectory(secrets); err != nil {
|
||||
return fmt.Errorf("secrets directory must be private and operator-owned")
|
||||
}
|
||||
for _, name := range []string{"catalog-runtime-password", "catalog-migrator-password", "admin-password"} {
|
||||
path := filepath.Join(secrets, name)
|
||||
if _, err := os.Lstat(path); err == nil {
|
||||
value, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10)
|
||||
if err != nil || len(strings.TrimSpace(string(value))) < 32 {
|
||||
return fmt.Errorf("existing technical credentials must be private, readable and at least 32 characters; no file was replaced")
|
||||
}
|
||||
continue
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("cannot inspect technical credentials")
|
||||
}
|
||||
value := make([]byte, 32)
|
||||
if _, err := rand.Read(value); err != nil {
|
||||
return fmt.Errorf("cannot generate random credentials")
|
||||
}
|
||||
if err := safeio.WriteCanonicalNewPrivateFile(path, []byte(hex.EncodeToString(value)+"\n"), 0o600); err != nil {
|
||||
return fmt.Errorf("cannot create credential; existing files are retained")
|
||||
}
|
||||
}
|
||||
for name, contents := range map[string]string{"secrets.env": "# Supply the provider credential; never commit this file.\nOPENAI_API_KEY=CHANGE_ME\n", "pi-auth.json": "{}\n"} {
|
||||
path := filepath.Join(secrets, name)
|
||||
if _, err := os.Lstat(path); errors.Is(err, os.ErrNotExist) {
|
||||
if err := safeio.WriteCanonicalNewPrivateFile(path, []byte(contents), 0o600); err != nil {
|
||||
return fmt.Errorf("cannot create external credential template")
|
||||
}
|
||||
} else if _, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10); err != nil {
|
||||
return fmt.Errorf("existing credential template is not private or readable")
|
||||
}
|
||||
}
|
||||
authDirectory := filepath.Join(directory, "auth")
|
||||
if _, err := os.Lstat(filepath.Join(authDirectory, "auth.yaml")); err == nil {
|
||||
if _, _, err := authconfig.Load(authDirectory); err != nil {
|
||||
return fmt.Errorf("existing authentication documents are invalid; repair them explicitly")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
installation := config.Installation{Authentication: config.Authentication{ConfigDirectory: authDirectory}}
|
||||
if authconfig.Run(ctx, installation, []string{"configure", "--mode", "local", "--public-url", "http://localhost:8080", "--admin-user", "admin", "--password-file", filepath.Join(secrets, "admin-password")}, strings.NewReader(""), io.Discard, io.Discard) != 0 {
|
||||
return fmt.Errorf("cannot prepare local administrator; inspect protected auth files and retry without replacing existing credentials")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
// Package preparation owns installation-local documents before any runtime exists.
|
||||
package preparation
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
|
||||
)
|
||||
|
||||
func Prepare(directory string) error {
|
||||
if err := safeio.ValidateCanonicalPath(directory); err != nil {
|
||||
return fmt.Errorf("choose an absolute canonical destination")
|
||||
}
|
||||
exists, err := safeio.PreflightPrivateDirectory(directory)
|
||||
if err != nil || exists {
|
||||
return fmt.Errorf("choose a new private directory with an existing parent; existing documents are never replaced")
|
||||
}
|
||||
if err := safeio.EnsurePrivateDirectory(directory); err != nil {
|
||||
return fmt.Errorf("cannot create private preparation directory")
|
||||
}
|
||||
root := func(name string) string { return strconv.Quote(filepath.Join(directory, name)) }
|
||||
installation := fmt.Sprintf(`# Installation schema v2; replace CHANGE_ME before validation.
|
||||
# Paths refer to local preparation files, never to workspace Git.
|
||||
schemaVersion: 2
|
||||
profile: local
|
||||
projectDirectory: %s
|
||||
envFile: %s
|
||||
shell: {mode: full, defaultLocale: en}
|
||||
workspaceRepository:
|
||||
remote: https://CHANGE_ME/workspaces.git
|
||||
branch: main
|
||||
access: https
|
||||
authentication:
|
||||
configDirectory: %s
|
||||
modelCatalog:
|
||||
defaults: {interaction: openai/gpt-4.1-mini}
|
||||
embedding: {id: 'ollama/qwen3-embedding:0.6b', dimensions: 1024}
|
||||
providers:
|
||||
openai:
|
||||
authentication: {mode: secret_env, apiKeyEnv: OPENAI_API_KEY}
|
||||
session: {mode: pi_builtin}
|
||||
models:
|
||||
gpt-4.1-mini: {session: {}}
|
||||
# Metadata generation is optional: omitted here. Configure its eligibility in this catalog.
|
||||
# This Compose asset will come from the release; no application checkout is required here.
|
||||
overrides: [%s]
|
||||
`, strconv.Quote(directory), root("operator.env"), root("auth"), root("deploy/compose.git-https.yaml"))
|
||||
environment := "# Non-secret paths and parameters; no interpolation or duplicate keys.\n"
|
||||
for _, entry := range [][2]string{
|
||||
{"COMPOSE_PROJECT_NAME", "thothii-local"}, {"THT_WORKSPACE_INSTALLATION_ID", "local"},
|
||||
{"THT_WORKSPACE_GIT_REMOTE", "https://CHANGE_ME/workspaces.git"}, {"THT_WORKSPACE_GIT_BRANCH", "main"},
|
||||
{"THT_INSTALLATION_CONFIG_SOURCE", filepath.Join(directory, "thothii-installation.yaml")},
|
||||
{"THT_AUTH_CONFIG_ROOT", filepath.Join(directory, "auth")},
|
||||
{"THT_SECRETS_FILE", filepath.Join(directory, "secrets", "secrets.env")},
|
||||
{"PI_AUTH_FILE", filepath.Join(directory, "secrets", "pi-auth.json")},
|
||||
{"THT_WORKSPACE_GIT_CREDENTIALS_FILE", filepath.Join(directory, "secrets", "git-credentials")},
|
||||
{"THT_WORKSPACE_GIT_CA_FILE", filepath.Join(directory, "secrets", "git-ca.pem")},
|
||||
{"THT_CATALOG_RUNTIME_PASSWORD_SOURCE", filepath.Join(directory, "secrets", "catalog-runtime-password")},
|
||||
{"THT_CATALOG_MIGRATOR_PASSWORD_SOURCE", filepath.Join(directory, "secrets", "catalog-migrator-password")},
|
||||
{"THOTH_HTTP_PORT", "8080"}, {"THOTH_CORE_HTTP_PORT", "8787"}, {"MAX_PI_PROCESSES", "4"},
|
||||
} {
|
||||
environment += entry[0] + "=" + strconv.Quote(entry[1]) + "\n"
|
||||
}
|
||||
bootstrap := fmt.Sprintf(`# Bootstrap input only; PostgreSQL Metadata Catalog remains the runtime authority.
|
||||
schemaVersion: 1
|
||||
databases:
|
||||
- workspaceId: CHANGE_ME
|
||||
engine: postgres
|
||||
databaseName: CHANGE_ME
|
||||
schema: public
|
||||
binding:
|
||||
transport: postgres_direct
|
||||
host: CHANGE_ME
|
||||
port: 5432
|
||||
username: CHANGE_ME
|
||||
secretFiles:
|
||||
password: %s
|
||||
`, root("secrets/database-password"))
|
||||
documents := map[string]string{
|
||||
".gitignore": "*\n",
|
||||
"thothii-installation.yaml": installation, "operator.env": environment,
|
||||
"database-bootstrap.yaml": bootstrap,
|
||||
"README.md": "# Preparation / Preparazione\n\nReplace every CHANGE_ME / Sostituire ogni CHANGE_ME. Keep all files outside workspace Git / Tenere tutti i file fuori dal Git dei workspace.\n\n1. Edit installation models, workspace remote and operator.env consistently. / Modificare modelli, remoto e operator.env in modo coerente.\n2. Add one database bootstrap entry for every workspace; use read-only DWH credentials in protected files. / Una voce database per workspace, credenziali DWH in sola lettura in file protetti.\n3. Run tht installation credentials --directory PATH before setup; fill provider/database secrets yourself. / Generare credenziali tecniche prima del setup; compilare i segreti esterni.\n4. Repeat tht --installation PATH/thothii-installation.yaml installation validate --workspaces WORKSPACES. / Correggere e ripetere.\n\nNo services, network calls or database writes / Nessun servizio, chiamata di rete o scrittura database.\n",
|
||||
}
|
||||
for _, name := range []string{".gitignore", "thothii-installation.yaml", "operator.env", "database-bootstrap.yaml", "README.md"} {
|
||||
if err := safeio.WriteCanonicalNewPrivateFile(filepath.Join(directory, name), []byte(strings.TrimSpace(documents[name])+"\n"), 0o600); err != nil {
|
||||
return fmt.Errorf("cannot create preparation files; inspect the new directory and retry in a new destination")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
package preparation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/authconfig"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type Issue struct {
|
||||
Document string `json:"document"`
|
||||
Field string `json:"field"`
|
||||
Code string `json:"code"`
|
||||
Correction string `json:"correction"`
|
||||
}
|
||||
type Report struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
Scope string `json:"scope"`
|
||||
OK bool `json:"ok"`
|
||||
Issues []Issue `json:"issues"`
|
||||
Warnings []string `json:"warnings"`
|
||||
DeferredChecks []string `json:"deferred_checks"`
|
||||
}
|
||||
|
||||
func NewReport() Report {
|
||||
return Report{SchemaVersion: 1, Scope: "application-documents", Issues: []Issue{}, Warnings: []string{}, DeferredChecks: []string{"release-assets", "external-connectivity", "catalog-import", "runtime-readiness"}}
|
||||
}
|
||||
func (r *Report) Add(document, field, code, correction string) {
|
||||
r.OK = false
|
||||
r.Issues = append(r.Issues, Issue{document, field, code, correction})
|
||||
}
|
||||
func placeholder(value string) bool {
|
||||
upper := strings.ToUpper(value)
|
||||
return strings.Contains(upper, "CHANGE_ME") || strings.Contains(upper, "REPLACE_ME") || strings.Contains(upper, "YOUR_API_KEY") || strings.Contains(upper, "<PASSWORD>")
|
||||
}
|
||||
|
||||
// CheckYAML refuses unresolved placeholders and ambiguous authored YAML without returning values.
|
||||
func CheckYAML(path, document string, report *Report) bool {
|
||||
content, err := safeio.ReadCanonicalPrivateRegular(path, 1<<20)
|
||||
if err != nil {
|
||||
report.Add(document, "$", "file_unavailable", "Use a readable, private regular document at a canonical absolute path (no links).")
|
||||
return false
|
||||
}
|
||||
decoder := yaml.NewDecoder(bytes.NewReader(content))
|
||||
var node, extra yaml.Node
|
||||
if err := decoder.Decode(&node); err != nil || decoder.Decode(&extra) != io.EOF {
|
||||
report.Add(document, "$", "yaml_invalid", "Keep one well-formed YAML document.")
|
||||
return false
|
||||
}
|
||||
var walk func(*yaml.Node) bool
|
||||
walk = func(n *yaml.Node) bool {
|
||||
if n.Kind == yaml.AliasNode || n.Tag == "!!merge" {
|
||||
return false
|
||||
}
|
||||
if n.Kind == yaml.ScalarNode && placeholder(n.Value) {
|
||||
return false
|
||||
}
|
||||
if n.Kind == yaml.MappingNode {
|
||||
seen := map[string]bool{}
|
||||
for index := 0; index < len(n.Content); index += 2 {
|
||||
key := n.Content[index]
|
||||
if key.Kind != yaml.ScalarNode || seen[key.Value] {
|
||||
return false
|
||||
}
|
||||
seen[key.Value] = true
|
||||
}
|
||||
}
|
||||
for _, child := range n.Content {
|
||||
if !walk(child) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
if !walk(&node) {
|
||||
report.Add(document, "$", "incomplete_or_ambiguous", "Replace CHANGE_ME/REPLACE_ME placeholders, remove duplicate keys, aliases and YAML merge keys.")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
var environmentKey = regexp.MustCompile(`^[A-Z][A-Z0-9_]*$`)
|
||||
|
||||
func checkEnvironment(path string, report *Report) bool {
|
||||
contents, err := safeio.ReadCanonicalPrivateRegular(path, 1<<20)
|
||||
if err != nil {
|
||||
report.Add("operator.env", "$", "environment_unavailable", "Provide a private readable environment file.")
|
||||
return false
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, raw := range strings.Split(string(contents), "\n") {
|
||||
line := strings.TrimSpace(raw)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
key, value, found := strings.Cut(line, "=")
|
||||
if !found || !environmentKey.MatchString(key) || seen[key] || placeholder(value) || strings.Contains(value, "$") {
|
||||
report.Add("operator.env", "$", "environment_invalid", "Use one literal KEY=value per line, unique uppercase keys, and replace placeholders; shell interpolation is not accepted.")
|
||||
return false
|
||||
}
|
||||
if strings.HasSuffix(key, "_PASSWORD") || strings.HasSuffix(key, "_API_KEY") {
|
||||
report.Add("operator.env", "$", "inline_secret", "Move credential values to protected files and keep only file references in operator.env.")
|
||||
return false
|
||||
}
|
||||
seen[key] = true
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func CheckSecret(path, document, field string, allowEmpty bool, report *Report) bool {
|
||||
contents, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10)
|
||||
if err != nil || (!allowEmpty && len(bytes.TrimSpace(contents)) == 0) || placeholder(string(contents)) {
|
||||
report.Add(document, field, "secret_unavailable", "Supply a non-placeholder private readable regular secret file (owner-only permissions, no links); do not put its contents in YAML or logs.")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func Validate(path string) (config.Installation, Report) {
|
||||
report := NewReport()
|
||||
if !CheckYAML(path, "thothii-installation.yaml", &report) {
|
||||
return config.Installation{}, report
|
||||
}
|
||||
// Read only the location before the canonical loader checks every field.
|
||||
contents, _ := safeio.ReadCanonicalPrivateRegular(path, 1<<20)
|
||||
var location struct {
|
||||
EnvFile string `yaml:"envFile"`
|
||||
}
|
||||
if yaml.Unmarshal(contents, &location) != nil {
|
||||
report.Add("thothii-installation.yaml", "envFile", "schema_invalid", "Set envFile to one absolute path string, then correct the remaining descriptor fields.")
|
||||
return config.Installation{}, report
|
||||
}
|
||||
if !checkEnvironment(location.EnvFile, &report) {
|
||||
return config.Installation{}, report
|
||||
}
|
||||
installation, err := config.LoadPrepared(path)
|
||||
if err != nil {
|
||||
field, correction := "$", "Check installation schema v2, absolute paths, workspace remote/branch and matching environment values. Custom overrides and Git credential/CA files must exist."
|
||||
if strings.Contains(err.Error(), "modelCatalog") {
|
||||
field, correction = "modelCatalog", "Check interaction default eligibility, provider endpoints/authentication, embedding id/dimensions, and the private provider key bundle."
|
||||
}
|
||||
if strings.Contains(err.Error(), "authentication") {
|
||||
field, correction = "authentication", "Match the authentication directory to operator.env and prepare its protected files."
|
||||
}
|
||||
report.Add("thothii-installation.yaml", field, "configuration_invalid", correction)
|
||||
return config.Installation{}, report
|
||||
}
|
||||
value := func(key string) string { result, _ := installation.EnvironmentValue(key); return result }
|
||||
for _, key := range []string{"COMPOSE_PROJECT_NAME", "THT_WORKSPACE_INSTALLATION_ID", "THT_INSTALLATION_CONFIG_SOURCE", "THT_SECRETS_FILE", "PI_AUTH_FILE", "THT_CATALOG_RUNTIME_PASSWORD_SOURCE", "THT_CATALOG_MIGRATOR_PASSWORD_SOURCE"} {
|
||||
if value(key) == "" {
|
||||
report.Add("operator.env", key, "required", "Supply this installation parameter or protected file reference before setup.")
|
||||
}
|
||||
}
|
||||
if value("THT_INSTALLATION_CONFIG_SOURCE") != path {
|
||||
report.Add("operator.env", "THT_INSTALLATION_CONFIG_SOURCE", "path_mismatch", "Point to the exact installation descriptor being validated.")
|
||||
}
|
||||
for _, key := range []string{"THOTH_HTTP_PORT", "THOTH_CORE_HTTP_PORT", "MAX_PI_PROCESSES"} {
|
||||
number, err := strconv.Atoi(value(key))
|
||||
if err != nil || number < 1 || number > 65535 {
|
||||
report.Add("operator.env", key, "invalid_number", "Supply a positive integer; HTTP ports must be within 1..65535.")
|
||||
}
|
||||
}
|
||||
if value("THOTH_HTTP_PORT") == value("THOTH_CORE_HTTP_PORT") {
|
||||
report.Add("operator.env", "THOTH_CORE_HTTP_PORT", "port_collision", "Choose different frontend and core host ports.")
|
||||
}
|
||||
files, err := installation.SecretFiles()
|
||||
if err != nil {
|
||||
report.Add("operator.env", "$", "secret_references_invalid", "Use canonical absolute paths for all _FILE and _SOURCE references.")
|
||||
}
|
||||
for _, file := range files {
|
||||
allowEmpty := file == value("THT_WORKSPACE_GIT_CREDENTIALS_FILE")
|
||||
CheckSecret(file, "operator.env", "protected-file-reference", allowEmpty, &report)
|
||||
}
|
||||
auth, users, err := authconfig.Load(installation.AuthenticationDirectory())
|
||||
if err != nil {
|
||||
report.Add("auth/auth.yaml", "$", "authentication_invalid", "Prepare and validate local authentication documents before setup, including the initial administrator.")
|
||||
} else if auth.Mode == "local" {
|
||||
admin := false
|
||||
for _, user := range users.Users {
|
||||
for _, role := range user.Roles {
|
||||
if user.Enabled && role == authconfig.RoleAdmin {
|
||||
admin = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !admin {
|
||||
report.Add("auth/users.yaml", "users", "administrator_missing", "Enable at least one administrator before setup.")
|
||||
}
|
||||
} else {
|
||||
report.Add("auth/auth.yaml", "mode", "unsupported_bootstrap", "This preparation increment supports local administrative authentication; use the documented existing OIDC preparation path until its offline bootstrap validation is available.")
|
||||
}
|
||||
checkPiCredentials(value("PI_AUTH_FILE"), installation.ModelCatalog, &report)
|
||||
report.OK = len(report.Issues) == 0
|
||||
return installation, report
|
||||
}
|
||||
|
||||
// Pi stores api_key or oauth records. Require literal prepared material here; model
|
||||
// environment references belong to the catalog's secret_env path, not host process state.
|
||||
func checkPiCredentials(path string, catalog config.ModelCatalog, report *Report) {
|
||||
contents, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10)
|
||||
var credentials map[string]map[string]any
|
||||
invalid := err != nil || json.Unmarshal(contents, &credentials) != nil || credentials == nil
|
||||
literal := func(value any) bool {
|
||||
text, ok := value.(string)
|
||||
return ok && strings.TrimSpace(text) != "" && !strings.HasPrefix(text, "!") && !strings.Contains(text, "$") && !placeholder(text)
|
||||
}
|
||||
var declarative func(any) bool
|
||||
declarative = func(value any) bool {
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
return !strings.HasPrefix(v, "!")
|
||||
case []any:
|
||||
for _, item := range v {
|
||||
if !declarative(item) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
case map[string]any:
|
||||
for _, item := range v {
|
||||
if !declarative(item) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for provider, record := range credentials {
|
||||
name := strings.ToLower(strings.TrimSpace(provider))
|
||||
if name == "" || provider != name || seen[name] || !declarative(record) {
|
||||
invalid = true
|
||||
}
|
||||
seen[name] = true
|
||||
switch record["type"] {
|
||||
case "api_key":
|
||||
if !literal(record["key"]) {
|
||||
invalid = true
|
||||
}
|
||||
if env, present := record["env"]; present {
|
||||
values, ok := env.(map[string]any)
|
||||
if !ok {
|
||||
invalid = true
|
||||
}
|
||||
for _, value := range values {
|
||||
if _, ok := value.(string); !ok {
|
||||
invalid = true
|
||||
}
|
||||
}
|
||||
}
|
||||
case "oauth":
|
||||
expires, ok := record["expires"].(float64)
|
||||
if !literal(record["access"]) || !literal(record["refresh"]) || !ok || expires <= 0 {
|
||||
invalid = true
|
||||
}
|
||||
default:
|
||||
invalid = true
|
||||
}
|
||||
}
|
||||
for id, provider := range catalog.Providers {
|
||||
if provider.Authentication.Mode == "pi_auth" && !seen[id] {
|
||||
invalid = true
|
||||
}
|
||||
}
|
||||
if invalid {
|
||||
report.Add("pi-auth.json", "providers", "provider_auth_invalid", "Provide a JSON object with literal api_key/type+key or oauth/type+access+refresh+expires records for selected Pi providers; use catalog secret_env for environment-based keys. Commands and unresolved references are not accepted.")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user