ThothII/backend/test/metadata-generation-models.test.ts

import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, expect, test } from "vitest";
import {
  loadMetadataGenerationModels,
  MetadataGenerationModelUnavailableError,
} from "../src/catalog/metadata-generation-models.js";
import { loadRuntimeModelCatalog, splitCanonicalModelId } from "../src/models/runtime-model-catalog.js";

const roots: string[] = [];

afterEach(() => {
  for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
});

function runtimeCatalog(overrides: Record<string, unknown> = {}, secrets = "OPENAI_API_KEY=raw-provider-secret\n") {
  const root = mkdtempSync(join(tmpdir(), "thothii-runtime-models-"));
  roots.push(root);
  const catalogFile = join(root, "catalog.json");
  const secretsFile = join(root, "thothii.secrets");
  const catalog = {
    schemaVersion: 2,
    defaultInteraction: "zai/glm-5.3",
    embedding: { id: "ollama/qwen3-embedding:0.6b", dimensions: 1024 },
    models: [
      {
        id: "zai/glm-5.3", provider: "zai", model: "glm-5.3", label: "GLM 5.3",
        upstreamModel: "glm-5.3", endpoint: { baseUrl: "https://api.z.ai/v1" },
        authentication: { mode: "secret_env", apiKeyEnv: "OPENAI_API_KEY" },
        sessionAdapter: { mode: "openai_compatible" },
        metadataAdapter: { litellmProvider: "openai" },
        session: { reasoning: true, contextWindow: 200000, maxTokens: 131072 },
        metadataGeneration: { disableThinking: false },
      },
      {
        id: "deepseek/deepseek-v4-pro", provider: "deepseek", model: "deepseek-v4-pro",
        label: "DeepSeek V4 Pro", upstreamModel: "deepseek-v4-pro",
        authentication: { mode: "pi_auth" }, sessionAdapter: { mode: "pi_builtin" },
        session: { reasoning: false },
      },
    ],
    ...overrides,
  };
  writeFileSync(catalogFile, JSON.stringify(catalog), { mode: 0o600 });
  writeFileSync(secretsFile, secrets, { mode: 0o600 });
  chmodSync(catalogFile, 0o600);
  chmodSync(secretsFile, 0o600);
  return { catalogFile, secretsFile };
}

test("loads session default and safe metadata choices from the normalized runtime catalog", () => {
  const { catalogFile, secretsFile } = runtimeCatalog();
  const runtime = loadRuntimeModelCatalog(catalogFile);
  const metadata = loadMetadataGenerationModels({ catalogFile, secretsFile });

  expect(runtime.defaultInteraction).toBe("zai/glm-5.3");
  expect(runtime.hasSession("deepseek/deepseek-v4-pro")).toBe(false);
  expect(metadata.catalog()).toEqual({
    models: [{ id: "zai/glm-5.3", label: "GLM 5.3" }],
    default: "zai/glm-5.3",
  });
  expect(metadata.resolve("zai/glm-5.3")).toEqual({
    id: "zai/glm-5.3", provider: "openai", model: "glm-5.3",
    endpoint: { baseUrl: "https://api.z.ai/v1" },
    apiKeyEnv: "OPENAI_API_KEY", apiKey: "raw-provider-secret",
  });
  expect(() => metadata.resolve("zai/missing")).toThrow(MetadataGenerationModelUnavailableError);
});

test("DeepSeek uses one identity and bundle credential for native Pi and LiteLLM", () => {
  const models = ["deepseek-v4-pro", "deepseek-v4-flash"].map((model) => ({
    id: `deepseek/${model}`, provider: "deepseek", model, label: model, upstreamModel: model,
    authentication: { mode: "secret_env", apiKeyEnv: "DEEPSEEK_API_KEY" },
    sessionAdapter: { mode: "pi_builtin" }, metadataAdapter: { litellmProvider: "deepseek" },
    session: { reasoning: false }, metadataGeneration: { disableThinking: false },
  }));
  const files = runtimeCatalog({ defaultInteraction: models[0].id, models }, "DEEPSEEK_API_KEY=shared-key\n");
  const runtime = loadRuntimeModelCatalog(files.catalogFile);
  const metadata = loadMetadataGenerationModels(files);
  expect(runtime.sessionModels().map((model) => model.id)).toEqual(metadata.catalog().models.map((model) => model.id));
  expect(metadata.catalog().default).toBe(runtime.defaultInteraction);
  for (const model of models) {
    expect(metadata.resolve(model.id)).toMatchObject({
      id: model.id, provider: "deepseek", model: model.model,
      apiKeyEnv: "DEEPSEEK_API_KEY", apiKey: "shared-key",
    });
  }
  expect(() => metadata.resolve("deepseek-metadata/deepseek-v4-pro"))
    .toThrow(MetadataGenerationModelUnavailableError);
});

test("returns empty catalogs when no runtime projection is configured", () => {
  expect(loadRuntimeModelCatalog().defaultInteraction).toBeNull();
  expect(loadMetadataGenerationModels({}).catalog()).toEqual({ models: [], default: null });
});

test.each([
  ["qwen-chat-template", true, true],
  ["qwen", true, true],
  ["qwen-typo", true, false],
  ["qwen-chat-template", false, false],
])("validates Qwen thinking format %s with reasoning=%s", (thinkingFormat, reasoning, valid) => {
  const { catalogFile } = runtimeCatalog({
    defaultInteraction: "local/qwen",
    models: [{
      id: "local/qwen", provider: "local", model: "qwen", label: "Qwen",
      upstreamModel: "qwen", endpoint: { baseUrl: "http://localhost:8000/v1" },
      authentication: { mode: "none" }, sessionAdapter: { mode: "openai_compatible" },
      session: {
        reasoning, contextWindow: 32768, maxTokens: 8192,
        compatibility: {
          supportsDeveloperRole: false, supportsReasoningEffort: false,
          supportsStore: false, maxTokensField: "max_tokens", thinkingFormat,
        },
      },
    }],
  });
  if (valid) {
    expect(loadRuntimeModelCatalog(catalogFile).sessionModels()[0].session?.compatibility)
      .toMatchObject({ thinkingFormat });
  } else {
    expect(() => loadRuntimeModelCatalog(catalogFile)).toThrow("runtime model catalog is invalid");
  }
});

test("rejects a drifted default and an unprotected projection", () => {
  const drifted = runtimeCatalog({ defaultInteraction: "zai/missing" });
  expect(() => loadRuntimeModelCatalog(drifted.catalogFile)).toThrow("interaction default is invalid");

  const unprotected = runtimeCatalog();
  chmodSync(unprotected.catalogFile, 0o666);
  expect(() => loadRuntimeModelCatalog(unprotected.catalogFile)).toThrow("runtime model catalog is unavailable");
});

test("rejects authentication semantics that cannot come from the installation catalog", () => {
  const invalid = runtimeCatalog({
    defaultInteraction: undefined,
    models: [{
      id: "zai/glm-5.3",
      provider: "zai",
      model: "glm-5.3",
      label: "GLM 5.3",
      upstreamModel: "glm-5.3",
      authentication: { mode: "secret_env" },
      sessionAdapter: { mode: "pi_builtin" },
      session: { reasoning: true },
    }],
  });
  expect(() => loadRuntimeModelCatalog(invalid.catalogFile)).toThrow("runtime model catalog is invalid");
});

test("fails closed for missing or unusable provider secrets", () => {
  const missing = runtimeCatalog({}, "THT_DWH_API_KEY=other\n");
  expect(() => loadMetadataGenerationModels(missing)).toThrow('secret "OPENAI_API_KEY" is missing');

  const unusable = runtimeCatalog({}, "OPENAI_API_KEY=contains whitespace\n");
  expect(() => loadMetadataGenerationModels(unusable)).toThrow('secret "OPENAI_API_KEY" is unusable');
});

test("splits canonical session identities without provider aliases", () => {
  expect(splitCanonicalModelId("zai/glm-5.3")).toEqual({ provider: "zai", model: "glm-5.3" });
  expect(() => splitCanonicalModelId("glm-5.3")).toThrow("model identity is invalid");
});

test("rejects a default supported by only one configured use", () => {
  const { catalogFile } = runtimeCatalog({ defaultInteraction: "deepseek/deepseek-v4-pro" });
  expect(() => loadRuntimeModelCatalog(catalogFile)).toThrow("interaction default is invalid");
});

test("requires regenerated runtime schema v2 rather than interpreting two legacy defaults", () => {
  const { catalogFile } = runtimeCatalog({ schemaVersion: 1, defaultSession: "zai/glm-5.3" });
  expect(() => loadRuntimeModelCatalog(catalogFile)).toThrow("runtime model catalog is invalid");
});