ThothII/backend/test/metadata-generation-models.test.ts

import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, expect, test } from "vitest";
import {
  loadMetadataGenerationModels,
  MetadataGenerationModelUnavailableError,
} from "../src/catalog/metadata-generation-models.js";
import { loadRuntimeModelCatalog, splitCanonicalModelId } from "../src/models/runtime-model-catalog.js";

const roots: string[] = [];

afterEach(() => {
  for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
});

function runtimeCatalog(overrides: Record<string, unknown> = {}, secrets = "OPENAI_API_KEY=raw-provider-secret\n") {
  const root = mkdtempSync(join(tmpdir(), "thothii-runtime-models-"));
  roots.push(root);
  const catalogFile = join(root, "catalog.json");
  const secretsFile = join(root, "thothii.secrets");
  const catalog = {
    schemaVersion: 1,
    defaultSession: "zai/glm-5.3",
    defaultMetadataGeneration: "zai/glm-5.3",
    embedding: { id: "ollama/qwen3-embedding:0.6b", dimensions: 1024 },
    models: [
      {
        id: "zai/glm-5.3", provider: "zai", model: "glm-5.3", label: "GLM 5.3",
        upstreamModel: "glm-5.3", endpoint: { baseUrl: "https://api.z.ai/v1" },
        authentication: { mode: "secret_env", apiKeyEnv: "OPENAI_API_KEY" },
        sessionAdapter: { mode: "openai_compatible" },
        metadataAdapter: { litellmProvider: "openai" },
        session: { reasoning: true, contextWindow: 200000, maxTokens: 131072 },
        metadataGeneration: { disableThinking: false },
      },
      {
        id: "deepseek/deepseek-v4-pro", provider: "deepseek", model: "deepseek-v4-pro",
        label: "DeepSeek V4 Pro", upstreamModel: "deepseek-v4-pro",
        authentication: { mode: "pi_auth" }, sessionAdapter: { mode: "pi_builtin" },
        session: { reasoning: false },
      },
    ],
    ...overrides,
  };
  writeFileSync(catalogFile, JSON.stringify(catalog), { mode: 0o600 });
  writeFileSync(secretsFile, secrets, { mode: 0o600 });
  chmodSync(catalogFile, 0o600);
  chmodSync(secretsFile, 0o600);
  return { catalogFile, secretsFile };
}

test("loads session default and safe metadata choices from the normalized runtime catalog", () => {
  const { catalogFile, secretsFile } = runtimeCatalog();
  const runtime = loadRuntimeModelCatalog(catalogFile);
  const metadata = loadMetadataGenerationModels({ catalogFile, secretsFile });

  expect(runtime.defaultSession).toBe("zai/glm-5.3");
  expect(runtime.hasSession("deepseek/deepseek-v4-pro")).toBe(true);
  expect(metadata.catalog()).toEqual({
    models: [{ id: "zai/glm-5.3", label: "GLM 5.3" }],
    default: "zai/glm-5.3",
  });
  expect(metadata.resolve("zai/glm-5.3")).toEqual({
    id: "zai/glm-5.3", provider: "openai", model: "glm-5.3",
    endpoint: { baseUrl: "https://api.z.ai/v1" },
    apiKeyEnv: "OPENAI_API_KEY", apiKey: "raw-provider-secret",
  });
  expect(() => metadata.resolve("zai/missing")).toThrow(MetadataGenerationModelUnavailableError);
});

test("returns empty catalogs when no runtime projection is configured", () => {
  expect(loadRuntimeModelCatalog().defaultSession).toBeNull();
  expect(loadMetadataGenerationModels({}).catalog()).toEqual({ models: [], default: null });
});

test("rejects a drifted default and an unprotected projection", () => {
  const drifted = runtimeCatalog({ defaultSession: "zai/missing" });
  expect(() => loadRuntimeModelCatalog(drifted.catalogFile)).toThrow("session default is invalid");

  const unprotected = runtimeCatalog();
  chmodSync(unprotected.catalogFile, 0o666);
  expect(() => loadRuntimeModelCatalog(unprotected.catalogFile)).toThrow("runtime model catalog is unavailable");
});

test("rejects authentication semantics that cannot come from the installation catalog", () => {
  const invalid = runtimeCatalog({
    defaultMetadataGeneration: undefined,
    models: [{
      id: "zai/glm-5.3",
      provider: "zai",
      model: "glm-5.3",
      label: "GLM 5.3",
      upstreamModel: "glm-5.3",
      authentication: { mode: "secret_env" },
      sessionAdapter: { mode: "pi_builtin" },
      session: { reasoning: true },
    }],
  });
  expect(() => loadRuntimeModelCatalog(invalid.catalogFile)).toThrow("runtime model catalog is invalid");
});

test("fails closed for missing or unusable provider secrets", () => {
  const missing = runtimeCatalog({}, "THT_DWH_API_KEY=other\n");
  expect(() => loadMetadataGenerationModels(missing)).toThrow('secret "OPENAI_API_KEY" is missing');

  const unusable = runtimeCatalog({}, "OPENAI_API_KEY=contains whitespace\n");
  expect(() => loadMetadataGenerationModels(unusable)).toThrow('secret "OPENAI_API_KEY" is unusable');
});

test("splits canonical session identities without provider aliases", () => {
  expect(splitCanonicalModelId("zai/glm-5.3")).toEqual({ provider: "zai", model: "glm-5.3" });
  expect(() => splitCanonicalModelId("glm-5.3")).toThrow("model identity is invalid");
});