mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: add AI catalog description generation

Browse Source
This commit is contained in:
Codex
2026-08-29 16:42:56 +02:00
parent b0afba81ca
commit 376dd5a09d
76 changed files with 14860 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/tht/internal/config/installation.go
+282 -2
View File
@@ -14,6 +14,7 @@ import (
"sort"
"strings"
"sync"
"unicode"
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
"github.com/compose-spec/compose-go/v2/dotenv"
@@ -25,6 +26,10 @@ const installationFileName = "thothii-installation.yaml"
const maxEnvironmentFileBytes = 1 << 20
const maxMetadataSecretBundleBytes = 64 << 10
const maxMetadataGenerationModels = 64
const maxSecretSources = 32
var dotenvParseMu sync.Mutex
@@ -35,9 +40,34 @@ type descriptor struct {
EnvFile string `yaml:"envFile"`
WorkspaceRepository workspaceRepositoryDescriptor `yaml:"workspaceRepository"`
Authentication authenticationDescriptor `yaml:"authentication"`
MetadataGeneration metadataGenerationDescriptor `yaml:"metadataGeneration"`
Overrides []string `yaml:"overrides"`
}
type metadataGenerationDescriptor struct {
Default string `yaml:"default"`
Models []metadataGenerationModelDescriptor `yaml:"models"`
}
type metadataGenerationModelDescriptor struct {
ID string `yaml:"id"`
Label string `yaml:"label"`
LiteLLM liteLLMDescriptor `yaml:"litellm"`
APIKeyEnv string `yaml:"apiKeyEnv"`
}
type liteLLMDescriptor struct {
Provider string `yaml:"provider"`
Model string `yaml:"model"`
DisableThinking bool `yaml:"disableThinking"`
Endpoint *metadataEndpointDescriptor `yaml:"endpoint"`
}
type metadataEndpointDescriptor struct {
BaseURL string `yaml:"baseUrl"`
APIVersion string `yaml:"apiVersion"`
}
type authenticationDescriptor struct {
ConfigDirectory string `yaml:"configDirectory"`
RuntimeProjection *runtimeProjectionDescriptor `yaml:"runtimeProjection"`
@@ -75,6 +105,36 @@ type Authentication struct {
RuntimeProjection *RuntimeProjection
}
// MetadataGenerationEndpoint contains optional provider endpoint settings for one LiteLLM model.
type MetadataGenerationEndpoint struct {
BaseURL string
APIVersion string
}
// MetadataGenerationLiteLLM identifies the provider/model pair used by the internal completion helper.
type MetadataGenerationLiteLLM struct {
Provider string
Model string
DisableThinking bool
Endpoint *MetadataGenerationEndpoint
}
// MetadataGenerationModel is one selectable installation-owned metadata-generation model.
// APIKeyEnv is an optional reference only; credential values never enter Installation.
// An empty value is allowed only for a model with an explicit keyless endpoint.
type MetadataGenerationModel struct {
ID string
Label string
LiteLLM MetadataGenerationLiteLLM
APIKeyEnv string
}
// MetadataGeneration is the installation-owned model list and its default selection.
type MetadataGeneration struct {
Default string
Models []MetadataGenerationModel
}
// Installation is a validated local Compose installation. It intentionally contains paths, not
// environment values or secret content.
type Installation struct {
@@ -84,6 +144,7 @@ type Installation struct {
EnvFile string
WorkspaceRepository WorkspaceRepository
Authentication Authentication
MetadataGeneration MetadataGeneration
Overrides []string
}
@@ -137,6 +198,29 @@ func Load(path string) (Installation, error) {
GID: raw.Authentication.RuntimeProjection.GID,
}
}
metadataGeneration := MetadataGeneration{
Default: raw.MetadataGeneration.Default,
Models: make([]MetadataGenerationModel, 0, len(raw.MetadataGeneration.Models)),
}
for _, rawModel := range raw.MetadataGeneration.Models {
model := MetadataGenerationModel{
ID: rawModel.ID,
Label: rawModel.Label,
LiteLLM: MetadataGenerationLiteLLM{
Provider: rawModel.LiteLLM.Provider,
Model: rawModel.LiteLLM.Model,
DisableThinking: rawModel.LiteLLM.DisableThinking,
},
APIKeyEnv: rawModel.APIKeyEnv,
}
if rawModel.LiteLLM.Endpoint != nil {
model.LiteLLM.Endpoint = &MetadataGenerationEndpoint{
BaseURL: rawModel.LiteLLM.Endpoint.BaseURL,
APIVersion: rawModel.LiteLLM.Endpoint.APIVersion,
}
}
metadataGeneration.Models = append(metadataGeneration.Models, model)
}
installation := Installation{
Path: path,
Profile: raw.Profile,
@@ -147,13 +231,17 @@ func Load(path string) (Installation, error) {
Branch: raw.WorkspaceRepository.Branch,
Access: raw.WorkspaceRepository.Access,
},
Authentication: authentication,
Overrides: make([]string, 0, len(raw.Overrides)),
Authentication: authentication,
MetadataGeneration: metadataGeneration,
Overrides: make([]string, 0, len(raw.Overrides)),
}
values, err := installation.environmentValues()
if err != nil {
return Installation{}, errors.New("installation secret declarations could not be read")
}
if err := installation.validateMetadataGeneration(values); err != nil {
return Installation{}, err
}
if values["THT_AUTH_CONFIG_ROOT"] != installation.AuthenticationDirectory() {
return Installation{}, errors.New("authentication.configDirectory must match THT_AUTH_CONFIG_ROOT")
}
@@ -189,6 +277,198 @@ func Load(path string) (Installation, error) {
return installation, nil
}
func (i Installation) validateMetadataGeneration(values map[string]string) error {
if len(i.MetadataGeneration.Models) > maxMetadataGenerationModels {
return fmt.Errorf(
"metadataGeneration.models must contain at most %d entries",
maxMetadataGenerationModels,
)
}
seen := make(map[string]struct{}, len(i.MetadataGeneration.Models))
for index, model := range i.MetadataGeneration.Models {
if err := validateMetadataGenerationModel(index, model); err != nil {
return err
}
if _, exists := seen[model.ID]; exists {
return fmt.Errorf("duplicate metadataGeneration model id %q", model.ID)
}
seen[model.ID] = struct{}{}
}
if len(i.MetadataGeneration.Models) > 0 && i.MetadataGeneration.Default == "" {
return errors.New("metadataGeneration.default is required when models are configured")
}
if i.MetadataGeneration.Default != "" {
if _, exists := seen[i.MetadataGeneration.Default]; !exists {
return fmt.Errorf(
"metadataGeneration.default %q does not identify a configured model",
i.MetadataGeneration.Default,
)
}
}
if len(i.MetadataGeneration.Models) == 0 {
return nil
}
if values["THT_INSTALLATION_CONFIG_SOURCE"] != i.Path {
return errors.New("metadataGeneration requires THT_INSTALLATION_CONFIG_SOURCE to match the installation file")
}
requiresSecrets := false
for _, model := range i.MetadataGeneration.Models {
if model.APIKeyEnv != "" {
requiresSecrets = true
break
}
}
secrets := map[string]string{}
if requiresSecrets {
bundlePath := values["THT_SECRETS_FILE"]
if bundlePath == "" {
return errors.New("metadataGeneration keyed models require THT_SECRETS_FILE")
}
var err error
secrets, err = readMetadataGenerationSecrets(bundlePath)
if err != nil {
return err
}
}
for _, model := range i.MetadataGeneration.Models {
if model.APIKeyEnv == "" {
continue
}
value, exists := secrets[model.APIKeyEnv]
if !exists {
return fmt.Errorf(
"metadataGeneration model %q secret %q is missing from THT_SECRETS_FILE",
model.ID,
model.APIKeyEnv,
)
}
if !usableMetadataGenerationSecret(value) {
return fmt.Errorf(
"metadataGeneration model %q secret %q is unusable",
model.ID,
model.APIKeyEnv,
)
}
}
return nil
}
var metadataModelIDPattern = regexp.MustCompile(`^[a-z][a-z0-9._-]{0,63}$`)
var metadataProviderPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`)
var metadataProviderModelPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$`)
var metadataAPIVersionPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`)
var metadataSecretBundleKeyPattern = regexp.MustCompile(`^[A-Z][A-Z0-9_]{0,127}$`)
var metadataAPIKeyEnvironments = map[string]struct{}{
"THT_METADATA_API_KEY": {},
"ANTHROPIC_API_KEY": {},
"AZURE_API_KEY": {},
"GEMINI_API_KEY": {},
"DEEPSEEK_API_KEY": {},
"OPENAI_API_KEY": {},
"OPENROUTER_API_KEY": {},
"ZAI_API_KEY": {},
}
var metadataSecretBundleKeys = map[string]struct{}{
"THT_MODEL_API_KEY": {},
"THT_DWH_API_KEY": {},
"THT_VEC_API_KEY": {},
"THT_VEC_WRITE_API_KEY": {},
"THT_CA": {},
"THT_SSL_CA": {},
"THT_VECTOR_BOOTSTRAP_PASSWORD": {},
"THT_VECTOR_MIGRATOR_PASSWORD": {},
"THT_VECTOR_READER_PASSWORD": {},
"THT_VECTOR_WRITER_PASSWORD": {},
"PI_PROVIDER_API_KEY": {},
"THT_OIDC_CLIENT_SECRET": {},
"THT_AUTHENTIK_API_TOKEN": {},
"THT_METADATA_API_KEY": {},
"ANTHROPIC_API_KEY": {},
"AZURE_API_KEY": {},
"GEMINI_API_KEY": {},
"DEEPSEEK_API_KEY": {},
"OPENAI_API_KEY": {},
"OPENROUTER_API_KEY": {},
"ZAI_API_KEY": {},
}
func validateMetadataGenerationModel(index int, model MetadataGenerationModel) error {
prefix := fmt.Sprintf("metadataGeneration.models[%d]", index)
if !metadataModelIDPattern.MatchString(model.ID) {
return fmt.Errorf("%s.id is invalid", prefix)
}
if len(model.Label) == 0 || len(model.Label) > 128 || strings.TrimSpace(model.Label) != model.Label ||
strings.IndexFunc(model.Label, unicode.IsControl) >= 0 {
return fmt.Errorf("%s.label is invalid", prefix)
}
if !metadataProviderPattern.MatchString(model.LiteLLM.Provider) {
return fmt.Errorf("%s.litellm.provider is invalid", prefix)
}
if !metadataProviderModelPattern.MatchString(model.LiteLLM.Model) {
return fmt.Errorf("%s.litellm.model is invalid", prefix)
}
if model.APIKeyEnv == "" {
if model.LiteLLM.Endpoint == nil {
return fmt.Errorf("%s.apiKeyEnv is required unless an explicit keyless endpoint is configured", prefix)
}
} else {
if !metadataSecretBundleKeyPattern.MatchString(model.APIKeyEnv) {
return fmt.Errorf("%s.apiKeyEnv is invalid", prefix)
}
if _, allowed := metadataAPIKeyEnvironments[model.APIKeyEnv]; !allowed {
return fmt.Errorf("%s.apiKeyEnv is invalid", prefix)
}
}
if endpoint := model.LiteLLM.Endpoint; endpoint != nil {
parsed, err := url.Parse(endpoint.BaseURL)
if err != nil || strings.TrimSpace(endpoint.BaseURL) != endpoint.BaseURL ||
(parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Hostname() == "" ||
parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" {
return fmt.Errorf("%s.litellm.endpoint.baseUrl is invalid", prefix)
}
if endpoint.APIVersion != "" && !metadataAPIVersionPattern.MatchString(endpoint.APIVersion) {
return fmt.Errorf("%s.litellm.endpoint.apiVersion is invalid", prefix)
}
}
if model.LiteLLM.DisableThinking && model.LiteLLM.Endpoint == nil {
return fmt.Errorf("%s.litellm.disableThinking requires an explicit endpoint", prefix)
}
return nil
}
func readMetadataGenerationSecrets(path string) (map[string]string, error) {
contents, err := safeio.ReadCanonicalPrivateRegular(path, maxMetadataSecretBundleBytes)
if err != nil {
return nil, errors.New("metadataGeneration secrets in THT_SECRETS_FILE are unavailable")
}
values := make(map[string]string)
for _, raw := range strings.Split(string(contents), "\n") {
line := strings.TrimSuffix(raw, "\r")
if strings.TrimSpace(line) == "" || strings.HasPrefix(strings.TrimSpace(line), "#") {
continue
}
key, value, found := strings.Cut(line, "=")
if !found || !metadataSecretBundleKeyPattern.MatchString(key) {
return nil, errors.New("metadataGeneration secrets in THT_SECRETS_FILE are invalid")
}
if _, allowed := metadataSecretBundleKeys[key]; !allowed {
return nil, errors.New("metadataGeneration secrets in THT_SECRETS_FILE are invalid")
}
if _, duplicate := values[key]; duplicate {
return nil, errors.New("metadataGeneration secrets in THT_SECRETS_FILE contain duplicate keys")
}
values[key] = value
}
return values, nil
}
func usableMetadataGenerationSecret(value string) bool {
return len(value) > 0 && len(value) <= 16*1024 && strings.TrimSpace(value) == value &&
strings.IndexFunc(value, func(character rune) bool {
return unicode.IsSpace(character) || unicode.IsControl(character)
}) < 0
}
// AuthenticationDirectory returns the descriptor-owned, non-secret authentication root.
func (i Installation) AuthenticationDirectory() string { return i.Authentication.ConfigDirectory }