273 lines
11 KiB
Go
273 lines
11 KiB
Go
package modelprojection
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
|
)
|
|
|
|
func TestRenderProducesDeterministicCatalogPiAndComposeProjections(t *testing.T) {
|
|
installation := projectionFixture(t)
|
|
|
|
first, err := Render(installation)
|
|
if err != nil {
|
|
t.Fatalf("Render() error = %v", err)
|
|
}
|
|
second, err := Render(installation)
|
|
if err != nil {
|
|
t.Fatalf("Render() second error = %v", err)
|
|
}
|
|
if len(first) != 5 {
|
|
t.Fatalf("artifact count = %d, want 5", len(first))
|
|
}
|
|
for path, contents := range first {
|
|
if !bytes.Equal(contents, second[path]) {
|
|
t.Fatalf("artifact %q is not deterministic", path)
|
|
}
|
|
}
|
|
|
|
catalog := string(first[CatalogFile])
|
|
for _, expected := range []string{`"id": "local/qwen"`, `"defaultInteraction": "zai/glm-5.3"`, `"embedding"`} {
|
|
if !strings.Contains(catalog, expected) {
|
|
t.Fatalf("catalog projection missing %s:\n%s", expected, catalog)
|
|
}
|
|
}
|
|
piModels := string(first[PiModelsFile])
|
|
if !strings.Contains(piModels, `"zai"`) || strings.Contains(piModels, `"deepseek"`) {
|
|
t.Fatalf("Pi models projection must contain only custom providers:\n%s", piModels)
|
|
}
|
|
settings := string(first[PiSettingsFile])
|
|
if strings.Contains(settings, `"deepseek/deepseek-v4-pro"`) || !strings.Contains(settings, `"local/qwen2.5:7b"`) {
|
|
t.Fatalf("Pi settings must enable only shared Core/Admin models:\n%s", settings)
|
|
}
|
|
if !strings.Contains(piModels, `"id": "qwen2.5:7b"`) || !strings.Contains(catalog, `"id": "local/qwen"`) {
|
|
t.Fatalf("Pi projection must map canonical catalog keys to upstream model IDs")
|
|
}
|
|
compose := string(first[ComposeFile])
|
|
if !strings.Contains(compose, `THT_HOST_PLATFORM: "`+runtime.GOOS+`"`) {
|
|
t.Fatal("Compose must project the CLI host OS, not the Docker container OS")
|
|
}
|
|
for _, expected := range []string{"THT_MODEL_CATALOG_FILE", "THT_DEFAULT_SESSION_MODEL", "zai/glm-5.3", "THT_INTERNAL_EMBEDDING_ID", "THT_INTERNAL_EMBEDDING_MODEL", "OLLAMA_MODEL", installation.GeneratedModelCatalogPath()} {
|
|
if !strings.Contains(compose, expected) {
|
|
t.Fatalf("Compose projection missing %q:\n%s", expected, compose)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDeepSeekBuiltinAndMetadataShareCatalogIdentityAndSecretReference(t *testing.T) {
|
|
i := projectionFixture(t)
|
|
p := i.ModelCatalog.Providers["deepseek"]
|
|
p.Authentication = config.ModelAuthentication{Mode: "secret_env", APIKeyEnv: "DEEPSEEK_API_KEY"}
|
|
p.MetadataGeneration = &config.ModelMetadataAdapter{LiteLLMProvider: "deepseek"}
|
|
for _, id := range []string{"deepseek-v4-pro", "deepseek-v4-flash"} {
|
|
p.Models[id] = config.CatalogModel{Session: &config.SessionModel{}, MetadataGeneration: &config.MetadataGenerationModel{}}
|
|
}
|
|
i.ModelCatalog.Providers["deepseek"] = p
|
|
i.ModelCatalog.Defaults.Interaction = "deepseek/deepseek-v4-pro"
|
|
files, err := Render(i)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var catalog runtimeCatalog
|
|
if err := json.Unmarshal(files[CatalogFile], &catalog); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
count := 0
|
|
for _, model := range catalog.Models {
|
|
if model.Provider != "deepseek" {
|
|
continue
|
|
}
|
|
count++
|
|
if model.Authentication.APIKeyEnv != "DEEPSEEK_API_KEY" || model.SessionAdapter == nil || model.MetadataAdapter == nil {
|
|
t.Fatalf("DeepSeek must share auth and both adapters: %#v", model)
|
|
}
|
|
if !bytes.Contains(files[PiSettingsFile], []byte(model.ID)) {
|
|
t.Fatalf("shared model missing from Pi selection: %s", model.ID)
|
|
}
|
|
}
|
|
if count != 2 || bytes.Contains(files[PiModelsFile], []byte("deepseek")) {
|
|
t.Fatal("both DeepSeek models must use native Pi definitions, not duplicated custom definitions")
|
|
}
|
|
}
|
|
|
|
func TestQwenThinkingFormatSurvivesCatalogAndPiProjection(t *testing.T) {
|
|
i := projectionFixture(t)
|
|
model := i.ModelCatalog.Providers["local"].Models["qwen"]
|
|
if err := json.Unmarshal([]byte(`{"reasoning":true,"contextWindow":32768,"maxTokens":8192,"compatibility":{"supportsDeveloperRole":false,"supportsReasoningEffort":false,"supportsStore":false,"maxTokensField":"max_tokens","thinkingFormat":"qwen-chat-template"}}`), model.Session); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
files, err := Render(i)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, path := range []string{CatalogFile, PiModelsFile} {
|
|
if !bytes.Contains(files[path], []byte(`"thinkingFormat": "qwen-chat-template"`)) {
|
|
t.Fatalf("%s dropped Qwen thinking compatibility", path)
|
|
}
|
|
}
|
|
var models piModels
|
|
if err := json.Unmarshal(files[PiModelsFile], &models); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !models.Providers["local"].Models[0].Reasoning {
|
|
t.Fatal("Pi must know the model supports explicit thinking control")
|
|
}
|
|
}
|
|
|
|
func TestProjectionKeepsSingleUseInventoryOutOfSharedPiSelection(t *testing.T) {
|
|
i := projectionFixture(t)
|
|
p := i.ModelCatalog.Providers["zai"]
|
|
p.Models["core-only"] = config.CatalogModel{Session: &config.SessionModel{ContextWindow: 32768, MaxTokens: 8192}}
|
|
p.Models["admin-only"] = config.CatalogModel{MetadataGeneration: &config.MetadataGenerationModel{}}
|
|
i.ModelCatalog.Providers["zai"] = p
|
|
files, err := Render(i)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var catalog runtimeCatalog
|
|
if err := json.Unmarshal(files[CatalogFile], &catalog); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, model := range catalog.Models {
|
|
if model.ID == "zai/core-only" && model.MetadataAdapter != nil {
|
|
t.Fatal("projected an unused metadata adapter")
|
|
}
|
|
if model.ID == "zai/admin-only" && model.SessionAdapter != nil {
|
|
t.Fatal("projected an unused session adapter")
|
|
}
|
|
}
|
|
if bytes.Contains(files[PiSettingsFile], []byte("core-only")) || bytes.Contains(files[PiSettingsFile], []byte("admin-only")) {
|
|
t.Fatal("single-use inventory leaked into global Pi selection")
|
|
}
|
|
}
|
|
|
|
func TestGenerateDoesNotReplaceUnchangedProjection(t *testing.T) {
|
|
installation := projectionFixture(t)
|
|
if err := Generate(installation); err != nil {
|
|
t.Fatalf("Generate() initial error = %v", err)
|
|
}
|
|
paths := []string{
|
|
installation.GeneratedModelCatalogPath(),
|
|
installation.GeneratedPiModelsPath(),
|
|
installation.GeneratedPiSettingsPath(),
|
|
installation.GeneratedFrontendConfigPath(),
|
|
installation.ModelProjectionComposePath(),
|
|
}
|
|
before := make(map[string]os.FileInfo, len(paths))
|
|
for _, path := range paths {
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before[path] = info
|
|
}
|
|
|
|
originalRename := renameProjectionDirectory
|
|
t.Cleanup(func() { renameProjectionDirectory = originalRename })
|
|
renames := 0
|
|
renameProjectionDirectory = func(oldPath, newPath string) error {
|
|
renames++
|
|
return os.Rename(oldPath, newPath)
|
|
}
|
|
if err := Generate(installation); err != nil {
|
|
t.Fatalf("Generate() repeated error = %v", err)
|
|
}
|
|
if renames != 0 {
|
|
t.Fatalf("Generate() replaced an unchanged generation with %d renames", renames)
|
|
}
|
|
for _, path := range paths {
|
|
after, err := os.Stat(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !os.SameFile(before[path], after) {
|
|
t.Fatalf("Generate() replaced unchanged artifact %q", path)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestGenerateRestoresWholePreviousGenerationWhenPublishFails(t *testing.T) {
|
|
installation := projectionFixture(t)
|
|
if err := Generate(installation); err != nil {
|
|
t.Fatalf("Generate() initial error = %v", err)
|
|
}
|
|
before := make(map[string][]byte)
|
|
for _, path := range []string{CatalogFile, PiModelsFile, PiSettingsFile, ComposeFile, FrontendConfigFile} {
|
|
contents, err := os.ReadFile(filepath.Join(installation.GeneratedDirectory(), filepath.FromSlash(path)))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before[path] = contents
|
|
}
|
|
provider := installation.ModelCatalog.Providers["zai"]
|
|
model := provider.Models["glm-5.3"]
|
|
model.Label = "Changed label"
|
|
provider.Models["glm-5.3"] = model
|
|
installation.ModelCatalog.Providers["zai"] = provider
|
|
|
|
originalRename := renameProjectionDirectory
|
|
t.Cleanup(func() { renameProjectionDirectory = originalRename })
|
|
renames := 0
|
|
renameProjectionDirectory = func(oldPath, newPath string) error {
|
|
renames++
|
|
if renames == 2 {
|
|
return errors.New("injected publish failure")
|
|
}
|
|
return os.Rename(oldPath, newPath)
|
|
}
|
|
if err := Generate(installation); err == nil {
|
|
t.Fatal("Generate() error = nil, want injected publish failure")
|
|
}
|
|
for path, expected := range before {
|
|
actual, err := os.ReadFile(filepath.Join(installation.GeneratedDirectory(), filepath.FromSlash(path)))
|
|
if err != nil || !bytes.Equal(actual, expected) {
|
|
t.Fatalf("artifact %q changed after failed generation: err=%v", path, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestGeneratePublishesAllArtifactsAndCheckDetectsDrift(t *testing.T) {
|
|
installation := projectionFixture(t)
|
|
if err := Generate(installation); err != nil {
|
|
t.Fatalf("Generate() error = %v", err)
|
|
}
|
|
for _, path := range []string{installation.GeneratedModelCatalogPath(), installation.GeneratedPiModelsPath(), installation.GeneratedPiSettingsPath(), installation.ModelProjectionComposePath()} {
|
|
if info, err := os.Stat(path); err != nil || !info.Mode().IsRegular() {
|
|
t.Fatalf("generated artifact %q: info=%v err=%v", path, info, err)
|
|
}
|
|
}
|
|
if drift, err := Check(installation); err != nil || len(drift) != 0 {
|
|
t.Fatalf("Check() = %v, %v; want no drift", drift, err)
|
|
}
|
|
if err := os.WriteFile(installation.GeneratedPiSettingsPath(), []byte("{}\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if drift, err := Check(installation); err != nil || len(drift) != 1 || drift[0] != PiSettingsFile {
|
|
t.Fatalf("Check() = %v, %v; want [%s]", drift, err, PiSettingsFile)
|
|
}
|
|
}
|
|
|
|
func projectionFixture(t *testing.T) config.Installation {
|
|
t.Helper()
|
|
root := t.TempDir()
|
|
return config.Installation{
|
|
Path: filepath.Join(root, "thothii-installation.yaml"),
|
|
ModelCatalog: config.ModelCatalog{
|
|
Defaults: config.ModelCatalogDefaults{Interaction: "zai/glm-5.3"},
|
|
Embedding: config.ModelCatalogEmbedding{ID: "ollama/qwen3-embedding:0.6b", Dimensions: 1024},
|
|
Providers: map[string]config.ModelProvider{
|
|
"deepseek": {Authentication: config.ModelAuthentication{Mode: "pi_auth"}, Session: &config.ModelSessionAdapter{Mode: "pi_builtin"}, Models: map[string]config.CatalogModel{"deepseek-v4-pro": {Session: &config.SessionModel{}}}},
|
|
"zai": {Endpoint: &config.ModelEndpoint{BaseURL: "https://api.z.ai/v1"}, Authentication: config.ModelAuthentication{Mode: "none"}, Session: &config.ModelSessionAdapter{Mode: "openai_compatible"}, MetadataGeneration: &config.ModelMetadataAdapter{LiteLLMProvider: "openai"}, Models: map[string]config.CatalogModel{"glm-5.3": {Label: "GLM-5.3", Session: &config.SessionModel{Reasoning: true, ContextWindow: 200000, MaxTokens: 131072}, MetadataGeneration: &config.MetadataGenerationModel{}}}},
|
|
"local": {Endpoint: &config.ModelEndpoint{BaseURL: "http://ollama:11434/v1"}, Authentication: config.ModelAuthentication{Mode: "none"}, Session: &config.ModelSessionAdapter{Mode: "openai_compatible"}, MetadataGeneration: &config.ModelMetadataAdapter{LiteLLMProvider: "openai"}, Models: map[string]config.CatalogModel{"qwen": {UpstreamModel: "qwen2.5:7b", Session: &config.SessionModel{ContextWindow: 32768, MaxTokens: 8192}, MetadataGeneration: &config.MetadataGenerationModel{DisableThinking: true}}}},
|
|
},
|
|
},
|
|
}
|
|
}
|