import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, test } from "vitest"; import { loadMetadataGenerationModels, MetadataGenerationModelUnavailableError, } from "../src/catalog/metadata-generation-models.js"; import { loadRuntimeModelCatalog, splitCanonicalModelId } from "../src/models/runtime-model-catalog.js"; const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); function runtimeCatalog(overrides: Record = {}, secrets = "OPENAI_API_KEY=raw-provider-secret\n") { const root = mkdtempSync(join(tmpdir(), "thothii-runtime-models-")); roots.push(root); const catalogFile = join(root, "catalog.json"); const secretsFile = join(root, "thothii.secrets"); const catalog = { schemaVersion: 2, defaultInteraction: "zai/glm-5.3", embedding: { id: "ollama/qwen3-embedding:0.6b", dimensions: 1024 }, models: [ { id: "zai/glm-5.3", provider: "zai", model: "glm-5.3", label: "GLM 5.3", upstreamModel: "glm-5.3", endpoint: { baseUrl: "https://api.z.ai/v1" }, authentication: { mode: "secret_env", apiKeyEnv: "OPENAI_API_KEY" }, sessionAdapter: { mode: "openai_compatible" }, metadataAdapter: { litellmProvider: "openai" }, session: { reasoning: true, contextWindow: 200000, maxTokens: 131072 }, metadataGeneration: { disableThinking: false }, }, { id: "deepseek/deepseek-v4-pro", provider: "deepseek", model: "deepseek-v4-pro", label: "DeepSeek V4 Pro", upstreamModel: "deepseek-v4-pro", authentication: { mode: "pi_auth" }, sessionAdapter: { mode: "pi_builtin" }, session: { reasoning: false }, }, ], ...overrides, }; writeFileSync(catalogFile, JSON.stringify(catalog), { mode: 0o600 }); writeFileSync(secretsFile, secrets, { mode: 0o600 }); chmodSync(catalogFile, 0o600); chmodSync(secretsFile, 0o600); return { catalogFile, secretsFile }; } test("loads session default and safe metadata choices from the normalized runtime catalog", () => { const { catalogFile, secretsFile } = runtimeCatalog(); const runtime = loadRuntimeModelCatalog(catalogFile); const metadata = loadMetadataGenerationModels({ catalogFile, secretsFile }); expect(runtime.defaultInteraction).toBe("zai/glm-5.3"); expect(runtime.hasSession("deepseek/deepseek-v4-pro")).toBe(false); expect(metadata.catalog()).toEqual({ models: [{ id: "zai/glm-5.3", label: "GLM 5.3" }], default: "zai/glm-5.3", }); expect(metadata.resolve("zai/glm-5.3")).toEqual({ id: "zai/glm-5.3", provider: "openai", model: "glm-5.3", endpoint: { baseUrl: "https://api.z.ai/v1" }, apiKeyEnv: "OPENAI_API_KEY", apiKey: "raw-provider-secret", }); expect(() => metadata.resolve("zai/missing")).toThrow(MetadataGenerationModelUnavailableError); }); test("DeepSeek uses one identity and bundle credential for native Pi and LiteLLM", () => { const models = ["deepseek-v4-pro", "deepseek-v4-flash"].map((model) => ({ id: `deepseek/${model}`, provider: "deepseek", model, label: model, upstreamModel: model, authentication: { mode: "secret_env", apiKeyEnv: "DEEPSEEK_API_KEY" }, sessionAdapter: { mode: "pi_builtin" }, metadataAdapter: { litellmProvider: "deepseek" }, session: { reasoning: false }, metadataGeneration: { disableThinking: false }, })); const files = runtimeCatalog({ defaultInteraction: models[0].id, models }, "DEEPSEEK_API_KEY=shared-key\n"); const runtime = loadRuntimeModelCatalog(files.catalogFile); const metadata = loadMetadataGenerationModels(files); expect(runtime.sessionModels().map((model) => model.id)).toEqual(metadata.catalog().models.map((model) => model.id)); expect(metadata.catalog().default).toBe(runtime.defaultInteraction); for (const model of models) { expect(metadata.resolve(model.id)).toMatchObject({ id: model.id, provider: "deepseek", model: model.model, apiKeyEnv: "DEEPSEEK_API_KEY", apiKey: "shared-key", }); } expect(() => metadata.resolve("deepseek-metadata/deepseek-v4-pro")) .toThrow(MetadataGenerationModelUnavailableError); }); test("returns empty catalogs when no runtime projection is configured", () => { expect(loadRuntimeModelCatalog().defaultInteraction).toBeNull(); expect(loadMetadataGenerationModels({}).catalog()).toEqual({ models: [], default: null }); }); test.each([ ["qwen-chat-template", true, true], ["qwen", true, true], ["qwen-typo", true, false], ["qwen-chat-template", false, false], ])("validates Qwen thinking format %s with reasoning=%s", (thinkingFormat, reasoning, valid) => { const { catalogFile } = runtimeCatalog({ defaultInteraction: "local/qwen", models: [{ id: "local/qwen", provider: "local", model: "qwen", label: "Qwen", upstreamModel: "qwen", endpoint: { baseUrl: "http://localhost:8000/v1" }, authentication: { mode: "none" }, sessionAdapter: { mode: "openai_compatible" }, session: { reasoning, contextWindow: 32768, maxTokens: 8192, compatibility: { supportsDeveloperRole: false, supportsReasoningEffort: false, supportsStore: false, maxTokensField: "max_tokens", thinkingFormat, }, }, }], }); if (valid) { expect(loadRuntimeModelCatalog(catalogFile).sessionModels()[0].session?.compatibility) .toMatchObject({ thinkingFormat }); } else { expect(() => loadRuntimeModelCatalog(catalogFile)).toThrow("runtime model catalog is invalid"); } }); test("rejects a drifted default and an unprotected projection", () => { const drifted = runtimeCatalog({ defaultInteraction: "zai/missing" }); expect(() => loadRuntimeModelCatalog(drifted.catalogFile)).toThrow("interaction default is invalid"); const unprotected = runtimeCatalog(); chmodSync(unprotected.catalogFile, 0o666); expect(() => loadRuntimeModelCatalog(unprotected.catalogFile)).toThrow("runtime model catalog is unavailable"); }); test("rejects authentication semantics that cannot come from the installation catalog", () => { const invalid = runtimeCatalog({ defaultInteraction: undefined, models: [{ id: "zai/glm-5.3", provider: "zai", model: "glm-5.3", label: "GLM 5.3", upstreamModel: "glm-5.3", authentication: { mode: "secret_env" }, sessionAdapter: { mode: "pi_builtin" }, session: { reasoning: true }, }], }); expect(() => loadRuntimeModelCatalog(invalid.catalogFile)).toThrow("runtime model catalog is invalid"); }); test("fails closed for missing or unusable provider secrets", () => { const missing = runtimeCatalog({}, "THT_DWH_API_KEY=other\n"); expect(() => loadMetadataGenerationModels(missing)).toThrow('secret "OPENAI_API_KEY" is missing'); const unusable = runtimeCatalog({}, "OPENAI_API_KEY=contains whitespace\n"); expect(() => loadMetadataGenerationModels(unusable)).toThrow('secret "OPENAI_API_KEY" is unusable'); }); test("splits canonical session identities without provider aliases", () => { expect(splitCanonicalModelId("zai/glm-5.3")).toEqual({ provider: "zai", model: "glm-5.3" }); expect(() => splitCanonicalModelId("glm-5.3")).toThrow("model identity is invalid"); }); test("rejects a default supported by only one configured use", () => { const { catalogFile } = runtimeCatalog({ defaultInteraction: "deepseek/deepseek-v4-pro" }); expect(() => loadRuntimeModelCatalog(catalogFile)).toThrow("interaction default is invalid"); }); test("requires regenerated runtime schema v2 rather than interpreting two legacy defaults", () => { const { catalogFile } = runtimeCatalog({ schemaVersion: 1, defaultSession: "zai/glm-5.3" }); expect(() => loadRuntimeModelCatalog(catalogFile)).toThrow("runtime model catalog is invalid"); });