feat(cli): prepare and validate application documents offline
This commit is contained in:
Codex
@@ -0,0 +1,274 @@
|
||||
package preparation
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/authconfig"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type Issue struct {
|
||||
Document string `json:"document"`
|
||||
Field string `json:"field"`
|
||||
Code string `json:"code"`
|
||||
Correction string `json:"correction"`
|
||||
}
|
||||
type Report struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
Scope string `json:"scope"`
|
||||
OK bool `json:"ok"`
|
||||
Issues []Issue `json:"issues"`
|
||||
Warnings []string `json:"warnings"`
|
||||
DeferredChecks []string `json:"deferred_checks"`
|
||||
}
|
||||
|
||||
func NewReport() Report {
|
||||
return Report{SchemaVersion: 1, Scope: "application-documents", Issues: []Issue{}, Warnings: []string{}, DeferredChecks: []string{"release-assets", "external-connectivity", "catalog-import", "runtime-readiness"}}
|
||||
}
|
||||
func (r *Report) Add(document, field, code, correction string) {
|
||||
r.OK = false
|
||||
r.Issues = append(r.Issues, Issue{document, field, code, correction})
|
||||
}
|
||||
func placeholder(value string) bool {
|
||||
upper := strings.ToUpper(value)
|
||||
return strings.Contains(upper, "CHANGE_ME") || strings.Contains(upper, "REPLACE_ME") || strings.Contains(upper, "YOUR_API_KEY") || strings.Contains(upper, "<PASSWORD>")
|
||||
}
|
||||
|
||||
// CheckYAML refuses unresolved placeholders and ambiguous authored YAML without returning values.
|
||||
func CheckYAML(path, document string, report *Report) bool {
|
||||
content, err := safeio.ReadCanonicalPrivateRegular(path, 1<<20)
|
||||
if err != nil {
|
||||
report.Add(document, "$", "file_unavailable", "Use a readable, private regular document at a canonical absolute path (no links).")
|
||||
return false
|
||||
}
|
||||
decoder := yaml.NewDecoder(bytes.NewReader(content))
|
||||
var node, extra yaml.Node
|
||||
if err := decoder.Decode(&node); err != nil || decoder.Decode(&extra) != io.EOF {
|
||||
report.Add(document, "$", "yaml_invalid", "Keep one well-formed YAML document.")
|
||||
return false
|
||||
}
|
||||
var walk func(*yaml.Node) bool
|
||||
walk = func(n *yaml.Node) bool {
|
||||
if n.Kind == yaml.AliasNode || n.Tag == "!!merge" {
|
||||
return false
|
||||
}
|
||||
if n.Kind == yaml.ScalarNode && placeholder(n.Value) {
|
||||
return false
|
||||
}
|
||||
if n.Kind == yaml.MappingNode {
|
||||
seen := map[string]bool{}
|
||||
for index := 0; index < len(n.Content); index += 2 {
|
||||
key := n.Content[index]
|
||||
if key.Kind != yaml.ScalarNode || seen[key.Value] {
|
||||
return false
|
||||
}
|
||||
seen[key.Value] = true
|
||||
}
|
||||
}
|
||||
for _, child := range n.Content {
|
||||
if !walk(child) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
if !walk(&node) {
|
||||
report.Add(document, "$", "incomplete_or_ambiguous", "Replace CHANGE_ME/REPLACE_ME placeholders, remove duplicate keys, aliases and YAML merge keys.")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
var environmentKey = regexp.MustCompile(`^[A-Z][A-Z0-9_]*$`)
|
||||
|
||||
func checkEnvironment(path string, report *Report) bool {
|
||||
contents, err := safeio.ReadCanonicalPrivateRegular(path, 1<<20)
|
||||
if err != nil {
|
||||
report.Add("operator.env", "$", "environment_unavailable", "Provide a private readable environment file.")
|
||||
return false
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, raw := range strings.Split(string(contents), "\n") {
|
||||
line := strings.TrimSpace(raw)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
key, value, found := strings.Cut(line, "=")
|
||||
if !found || !environmentKey.MatchString(key) || seen[key] || placeholder(value) || strings.Contains(value, "$") {
|
||||
report.Add("operator.env", "$", "environment_invalid", "Use one literal KEY=value per line, unique uppercase keys, and replace placeholders; shell interpolation is not accepted.")
|
||||
return false
|
||||
}
|
||||
if strings.HasSuffix(key, "_PASSWORD") || strings.HasSuffix(key, "_API_KEY") {
|
||||
report.Add("operator.env", "$", "inline_secret", "Move credential values to protected files and keep only file references in operator.env.")
|
||||
return false
|
||||
}
|
||||
seen[key] = true
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func CheckSecret(path, document, field string, allowEmpty bool, report *Report) bool {
|
||||
contents, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10)
|
||||
if err != nil || (!allowEmpty && len(bytes.TrimSpace(contents)) == 0) || placeholder(string(contents)) {
|
||||
report.Add(document, field, "secret_unavailable", "Supply a non-placeholder private readable regular secret file (owner-only permissions, no links); do not put its contents in YAML or logs.")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func Validate(path string) (config.Installation, Report) {
|
||||
report := NewReport()
|
||||
if !CheckYAML(path, "thothii-installation.yaml", &report) {
|
||||
return config.Installation{}, report
|
||||
}
|
||||
// Read only the location before the canonical loader checks every field.
|
||||
contents, _ := safeio.ReadCanonicalPrivateRegular(path, 1<<20)
|
||||
var location struct {
|
||||
EnvFile string `yaml:"envFile"`
|
||||
}
|
||||
if yaml.Unmarshal(contents, &location) != nil {
|
||||
report.Add("thothii-installation.yaml", "envFile", "schema_invalid", "Set envFile to one absolute path string, then correct the remaining descriptor fields.")
|
||||
return config.Installation{}, report
|
||||
}
|
||||
if !checkEnvironment(location.EnvFile, &report) {
|
||||
return config.Installation{}, report
|
||||
}
|
||||
installation, err := config.LoadPrepared(path)
|
||||
if err != nil {
|
||||
field, correction := "$", "Check installation schema v2, absolute paths, workspace remote/branch and matching environment values. Custom overrides and Git credential/CA files must exist."
|
||||
if strings.Contains(err.Error(), "modelCatalog") {
|
||||
field, correction = "modelCatalog", "Check interaction default eligibility, provider endpoints/authentication, embedding id/dimensions, and the private provider key bundle."
|
||||
}
|
||||
if strings.Contains(err.Error(), "authentication") {
|
||||
field, correction = "authentication", "Match the authentication directory to operator.env and prepare its protected files."
|
||||
}
|
||||
report.Add("thothii-installation.yaml", field, "configuration_invalid", correction)
|
||||
return config.Installation{}, report
|
||||
}
|
||||
value := func(key string) string { result, _ := installation.EnvironmentValue(key); return result }
|
||||
for _, key := range []string{"COMPOSE_PROJECT_NAME", "THT_WORKSPACE_INSTALLATION_ID", "THT_INSTALLATION_CONFIG_SOURCE", "THT_SECRETS_FILE", "PI_AUTH_FILE", "THT_CATALOG_RUNTIME_PASSWORD_SOURCE", "THT_CATALOG_MIGRATOR_PASSWORD_SOURCE"} {
|
||||
if value(key) == "" {
|
||||
report.Add("operator.env", key, "required", "Supply this installation parameter or protected file reference before setup.")
|
||||
}
|
||||
}
|
||||
if value("THT_INSTALLATION_CONFIG_SOURCE") != path {
|
||||
report.Add("operator.env", "THT_INSTALLATION_CONFIG_SOURCE", "path_mismatch", "Point to the exact installation descriptor being validated.")
|
||||
}
|
||||
for _, key := range []string{"THOTH_HTTP_PORT", "THOTH_CORE_HTTP_PORT", "MAX_PI_PROCESSES"} {
|
||||
number, err := strconv.Atoi(value(key))
|
||||
if err != nil || number < 1 || number > 65535 {
|
||||
report.Add("operator.env", key, "invalid_number", "Supply a positive integer; HTTP ports must be within 1..65535.")
|
||||
}
|
||||
}
|
||||
if value("THOTH_HTTP_PORT") == value("THOTH_CORE_HTTP_PORT") {
|
||||
report.Add("operator.env", "THOTH_CORE_HTTP_PORT", "port_collision", "Choose different frontend and core host ports.")
|
||||
}
|
||||
files, err := installation.SecretFiles()
|
||||
if err != nil {
|
||||
report.Add("operator.env", "$", "secret_references_invalid", "Use canonical absolute paths for all _FILE and _SOURCE references.")
|
||||
}
|
||||
for _, file := range files {
|
||||
allowEmpty := file == value("THT_WORKSPACE_GIT_CREDENTIALS_FILE")
|
||||
CheckSecret(file, "operator.env", "protected-file-reference", allowEmpty, &report)
|
||||
}
|
||||
auth, users, err := authconfig.Load(installation.AuthenticationDirectory())
|
||||
if err != nil {
|
||||
report.Add("auth/auth.yaml", "$", "authentication_invalid", "Prepare and validate local authentication documents before setup, including the initial administrator.")
|
||||
} else if auth.Mode == "local" {
|
||||
admin := false
|
||||
for _, user := range users.Users {
|
||||
for _, role := range user.Roles {
|
||||
if user.Enabled && role == authconfig.RoleAdmin {
|
||||
admin = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !admin {
|
||||
report.Add("auth/users.yaml", "users", "administrator_missing", "Enable at least one administrator before setup.")
|
||||
}
|
||||
} else {
|
||||
report.Add("auth/auth.yaml", "mode", "unsupported_bootstrap", "This preparation increment supports local administrative authentication; use the documented existing OIDC preparation path until its offline bootstrap validation is available.")
|
||||
}
|
||||
checkPiCredentials(value("PI_AUTH_FILE"), installation.ModelCatalog, &report)
|
||||
report.OK = len(report.Issues) == 0
|
||||
return installation, report
|
||||
}
|
||||
|
||||
// Pi stores api_key or oauth records. Require literal prepared material here; model
|
||||
// environment references belong to the catalog's secret_env path, not host process state.
|
||||
func checkPiCredentials(path string, catalog config.ModelCatalog, report *Report) {
|
||||
contents, err := safeio.ReadCanonicalPrivateRegular(path, 64<<10)
|
||||
var credentials map[string]map[string]any
|
||||
invalid := err != nil || json.Unmarshal(contents, &credentials) != nil || credentials == nil
|
||||
literal := func(value any) bool {
|
||||
text, ok := value.(string)
|
||||
return ok && strings.TrimSpace(text) != "" && !strings.HasPrefix(text, "!") && !strings.Contains(text, "$") && !placeholder(text)
|
||||
}
|
||||
var declarative func(any) bool
|
||||
declarative = func(value any) bool {
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
return !strings.HasPrefix(v, "!")
|
||||
case []any:
|
||||
for _, item := range v {
|
||||
if !declarative(item) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
case map[string]any:
|
||||
for _, item := range v {
|
||||
if !declarative(item) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for provider, record := range credentials {
|
||||
name := strings.ToLower(strings.TrimSpace(provider))
|
||||
if name == "" || provider != name || seen[name] || !declarative(record) {
|
||||
invalid = true
|
||||
}
|
||||
seen[name] = true
|
||||
switch record["type"] {
|
||||
case "api_key":
|
||||
if !literal(record["key"]) {
|
||||
invalid = true
|
||||
}
|
||||
if env, present := record["env"]; present {
|
||||
values, ok := env.(map[string]any)
|
||||
if !ok {
|
||||
invalid = true
|
||||
}
|
||||
for _, value := range values {
|
||||
if _, ok := value.(string); !ok {
|
||||
invalid = true
|
||||
}
|
||||
}
|
||||
}
|
||||
case "oauth":
|
||||
expires, ok := record["expires"].(float64)
|
||||
if !literal(record["access"]) || !literal(record["refresh"]) || !ok || expires <= 0 {
|
||||
invalid = true
|
||||
}
|
||||
default:
|
||||
invalid = true
|
||||
}
|
||||
}
|
||||
for id, provider := range catalog.Providers {
|
||||
if provider.Authentication.Mode == "pi_auth" && !seen[id] {
|
||||
invalid = true
|
||||
}
|
||||
}
|
||||
if invalid {
|
||||
report.Add("pi-auth.json", "providers", "provider_auth_invalid", "Provide a JSON object with literal api_key/type+key or oauth/type+access+refresh+expires records for selected Pi providers; use catalog secret_env for environment-based keys. Commands and unresolved references are not accepted.")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user