import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, test, vi } from "vitest"; import { buildApp } from "../src/app.js"; import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js"; import { loadMetadataGenerationModels, MetadataGenerationModelUnavailableError, } from "../src/catalog/metadata-generation-models.js"; import { loadConfig } from "../src/config.js"; import type { WorkspaceRegistry } from "../src/workspaces/registry.js"; const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); function metadataConfiguration( metadataGeneration: string, secrets = "OPENAI_API_KEY=raw-provider-secret\n", ) { const root = mkdtempSync(join(tmpdir(), "thothii-metadata-models-")); roots.push(root); const installationFile = join(root, "thothii-installation.yaml"); const secretsFile = join(root, "thothii.secrets"); writeFileSync(installationFile, metadataGeneration, { mode: 0o600 }); writeFileSync(secretsFile, secrets, { mode: 0o600 }); chmodSync(installationFile, 0o600); chmodSync(secretsFile, 0o600); return { installationFile, secretsFile }; } function appFor(installationFile: string, secretsFile: string) { const config = loadConfig({ NODE_ENV: "test", THT_HARNESS_DIR: "/missing", THT_INSTALLATION_CONFIG_FILE: installationFile, THT_SECRETS_FILE: secretsFile, PI_PROVIDER: "unrelated-pi-provider", PI_MODEL: "unrelated-pi-model", }); return buildApp(config, { thtRunner: {} as never, workspaceRegistry: { list: vi.fn(async () => []) } as unknown as WorkspaceRegistry, workspaceDiagnoser: vi.fn(), catalogRepository: new MemoryCatalogRepository(), }); } test("exposes only safe metadata-generation choices and their configured default", async () => { const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration: default: openai-mini models: - id: openai-mini label: OpenAI Mini litellm: provider: openai model: gpt-4.1-mini endpoint: baseUrl: https://api.openai.example/v1 apiVersion: "2026-08-01" apiKeyEnv: OPENAI_API_KEY `); const app = appFor(installationFile, secretsFile); const response = await app.inject({ method: "GET", url: "/catalog/metadata-generation/models" }); expect(response.statusCode).toBe(200); expect(response.json()).toEqual({ models: [{ id: "openai-mini", label: "OpenAI Mini" }], default: "openai-mini", }); expect(response.body).not.toMatch(/openai\/gpt|gpt-4\.1|api\.openai|OPENAI_API_KEY|raw-provider-secret/); await app.close(); }); test("rejects an unprotected installation descriptor", () => { const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration: default: openai-mini models: - id: openai-mini label: OpenAI Mini litellm: {provider: openai, model: gpt-4.1-mini} apiKeyEnv: OPENAI_API_KEY `); chmodSync(installationFile, 0o644); expect(() => loadMetadataGenerationModels({ installationFile, secretsFile })) .toThrow("metadata-generation installation is unavailable"); }); test("returns an empty safe catalog when no metadata-generation model is configured", async () => { const { installationFile, secretsFile } = metadataConfiguration("profile: local\n"); const app = appFor(installationFile, secretsFile); const response = await app.inject({ method: "GET", url: "/catalog/metadata-generation/models" }); expect(response.statusCode).toBe(200); expect(response.json()).toEqual({ models: [], default: null }); await app.close(); }); test("resolves only a configured selection for the later generation boundary", () => { const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration: default: openai-mini models: - id: openai-mini label: OpenAI Mini litellm: provider: openai model: gpt-4.1-mini endpoint: {baseUrl: https://api.openai.example/v1, apiVersion: "2026-08-01"} apiKeyEnv: OPENAI_API_KEY `); const models = loadMetadataGenerationModels({ installationFile, secretsFile }); expect(models.resolve("openai-mini")).toEqual({ id: "openai-mini", provider: "openai", model: "gpt-4.1-mini", endpoint: { baseUrl: "https://api.openai.example/v1", apiVersion: "2026-08-01" }, apiKeyEnv: "OPENAI_API_KEY", apiKey: "raw-provider-secret", }); expect(() => models.resolve("unknown-model")).toThrow(MetadataGenerationModelUnavailableError); }); test("loads DeepSeek, GLM, and an explicit keyless Qwen endpoint from installation setup", () => { const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration: default: glm-53 models: - id: deepseek-v4-pro label: DeepSeek V4 Pro litellm: {provider: deepseek, model: deepseek-v4-pro} apiKeyEnv: DEEPSEEK_API_KEY - id: glm-53 label: GLM 5.3 litellm: provider: openai model: glm-5.3 endpoint: {baseUrl: https://api.z.ai/api/coding/paas/v4} apiKeyEnv: ZAI_API_KEY - id: qwen-36 label: Qwen 3.6 litellm: provider: openai model: qwen3.6-35b-a3b disableThinking: true endpoint: {baseUrl: https://models.internal.example/v1} `, "DEEPSEEK_API_KEY=deepseek-secret\nZAI_API_KEY=zai-secret\n"); const models = loadMetadataGenerationModels({ installationFile, secretsFile }); expect(models.catalog()).toEqual({ models: [ { id: "deepseek-v4-pro", label: "DeepSeek V4 Pro" }, { id: "glm-53", label: "GLM 5.3" }, { id: "qwen-36", label: "Qwen 3.6" }, ], default: "glm-53", }); expect(models.resolve("deepseek-v4-pro")).toMatchObject({ apiKeyEnv: "DEEPSEEK_API_KEY", apiKey: "deepseek-secret", }); expect(models.resolve("qwen-36")).toEqual({ id: "qwen-36", provider: "openai", model: "qwen3.6-35b-a3b", disableThinking: true, endpoint: { baseUrl: "https://models.internal.example/v1" }, }); }); test("loads an explicit keyless endpoint without a secret bundle", () => { const { installationFile } = metadataConfiguration(`metadataGeneration: default: qwen-36 models: - id: qwen-36 label: Qwen 3.6 litellm: provider: openai model: qwen3.6-35b-a3b disableThinking: true endpoint: {baseUrl: https://models.internal.example/v1} `); expect(loadMetadataGenerationModels({ installationFile }).resolve("qwen-36")).toEqual({ id: "qwen-36", provider: "openai", model: "qwen3.6-35b-a3b", disableThinking: true, endpoint: { baseUrl: "https://models.internal.example/v1" }, }); }); test.each([ ["invalid YAML", "metadataGeneration: [\n", "OPENAI_API_KEY=secret\n", /invalid YAML/], ["duplicate ids", `metadataGeneration: default: openai-mini models: - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY} - {id: openai-mini, label: Two, litellm: {provider: openai, model: gpt-4.1}, apiKeyEnv: OPENAI_API_KEY} `, "OPENAI_API_KEY=secret\n", /model id "openai-mini" is duplicated/], ["missing default", `metadataGeneration: models: - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY} `, "OPENAI_API_KEY=secret\n", /default is required/], ["unknown default", `metadataGeneration: default: absent models: - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY} `, "OPENAI_API_KEY=secret\n", /default "absent" is not configured/], ["malformed settings", `metadataGeneration: default: openai-mini models: - {id: openai-mini, label: One, litellm: {provider: "open ai", model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY} `, "OPENAI_API_KEY=secret\n", /configuration is invalid/], ["malformed endpoint", `metadataGeneration: default: openai-mini models: - id: openai-mini label: One litellm: {provider: openai, model: gpt-4.1-mini, endpoint: {baseUrl: not-a-url}} apiKeyEnv: OPENAI_API_KEY `, "OPENAI_API_KEY=secret\n", /configuration is invalid/], ["keyless hosted model without endpoint", `metadataGeneration: default: openai-mini models: - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}} `, "", /configuration is invalid/], ["disable thinking without endpoint", `metadataGeneration: default: openai-mini models: - id: openai-mini label: One litellm: {provider: openai, model: gpt-4.1-mini, disableThinking: true} apiKeyEnv: OPENAI_API_KEY `, "OPENAI_API_KEY=secret\n", /configuration is invalid/], ["unallowed secret reference", `metadataGeneration: default: openai-mini models: - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: THT_DWH_API_KEY} `, "THT_DWH_API_KEY=secret\n", /configuration is invalid/], ["missing referenced secret", `metadataGeneration: default: openai-mini models: - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY} `, "THT_DWH_API_KEY=secret\n", /secret "OPENAI_API_KEY" is missing/], ["unusable referenced secret", `metadataGeneration: default: openai-mini models: - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY} `, "OPENAI_API_KEY=secret with whitespace\n", /secret "OPENAI_API_KEY" is unusable/], ] as const)("rejects %s metadata-generation configuration", (_name, yaml, secrets, expected) => { const { installationFile, secretsFile } = metadataConfiguration(yaml, secrets); expect(() => loadMetadataGenerationModels({ installationFile, secretsFile })).toThrow(expected); }); test("rejects a missing secret-bundle declaration for configured models", () => { const { installationFile } = metadataConfiguration(`metadataGeneration: default: openai-mini models: - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY} `); expect(() => loadMetadataGenerationModels({ installationFile })) .toThrow("metadata-generation keyed models require THT_SECRETS_FILE"); });