ThothII/backend/test/metadata-generation-models.test.ts

import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, expect, test, vi } from "vitest";
import { buildApp } from "../src/app.js";
import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
import {
  loadMetadataGenerationModels,
  MetadataGenerationModelUnavailableError,
} from "../src/catalog/metadata-generation-models.js";
import { loadConfig } from "../src/config.js";
import type { WorkspaceRegistry } from "../src/workspaces/registry.js";

const roots: string[] = [];

afterEach(() => {
  for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
});

function metadataConfiguration(
  metadataGeneration: string,
  secrets = "OPENAI_API_KEY=raw-provider-secret\n",
) {
  const root = mkdtempSync(join(tmpdir(), "thothii-metadata-models-"));
  roots.push(root);
  const installationFile = join(root, "thothii-installation.yaml");
  const secretsFile = join(root, "thothii.secrets");
  writeFileSync(installationFile, metadataGeneration, { mode: 0o600 });
  writeFileSync(secretsFile, secrets, { mode: 0o600 });
  chmodSync(installationFile, 0o600);
  chmodSync(secretsFile, 0o600);
  return { installationFile, secretsFile };
}

function appFor(installationFile: string, secretsFile: string) {
  const config = loadConfig({
    NODE_ENV: "test",
    THT_HARNESS_DIR: "/missing",
    THT_INSTALLATION_CONFIG_FILE: installationFile,
    THT_SECRETS_FILE: secretsFile,
    PI_PROVIDER: "unrelated-pi-provider",
    PI_MODEL: "unrelated-pi-model",
  });
  return buildApp(config, {
    thtRunner: {} as never,
    workspaceRegistry: { list: vi.fn(async () => []) } as unknown as WorkspaceRegistry,
    workspaceDiagnoser: vi.fn(),
    catalogRepository: new MemoryCatalogRepository(),
  });
}

test("exposes only safe metadata-generation choices and their configured default", async () => {
  const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration:
  default: openai-mini
  models:
    - id: openai-mini
      label: OpenAI Mini
      litellm:
        provider: openai
        model: gpt-4.1-mini
        endpoint:
          baseUrl: https://api.openai.example/v1
          apiVersion: "2026-08-01"
      apiKeyEnv: OPENAI_API_KEY
`);
  const app = appFor(installationFile, secretsFile);

  const response = await app.inject({ method: "GET", url: "/catalog/metadata-generation/models" });

  expect(response.statusCode).toBe(200);
  expect(response.json()).toEqual({
    models: [{ id: "openai-mini", label: "OpenAI Mini" }],
    default: "openai-mini",
  });
  expect(response.body).not.toMatch(/openai\/gpt|gpt-4\.1|api\.openai|OPENAI_API_KEY|raw-provider-secret/);
  await app.close();
});

test("rejects an unprotected installation descriptor", () => {
  const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration:
  default: openai-mini
  models:
    - id: openai-mini
      label: OpenAI Mini
      litellm: {provider: openai, model: gpt-4.1-mini}
      apiKeyEnv: OPENAI_API_KEY
`);
  chmodSync(installationFile, 0o644);

  expect(() => loadMetadataGenerationModels({ installationFile, secretsFile }))
    .toThrow("metadata-generation installation is unavailable");
});

test("returns an empty safe catalog when no metadata-generation model is configured", async () => {
  const { installationFile, secretsFile } = metadataConfiguration("profile: local\n");
  const app = appFor(installationFile, secretsFile);

  const response = await app.inject({ method: "GET", url: "/catalog/metadata-generation/models" });

  expect(response.statusCode).toBe(200);
  expect(response.json()).toEqual({ models: [], default: null });
  await app.close();
});

test("resolves only a configured selection for the later generation boundary", () => {
  const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration:
  default: openai-mini
  models:
    - id: openai-mini
      label: OpenAI Mini
      litellm:
        provider: openai
        model: gpt-4.1-mini
        endpoint: {baseUrl: https://api.openai.example/v1, apiVersion: "2026-08-01"}
      apiKeyEnv: OPENAI_API_KEY
`);
  const models = loadMetadataGenerationModels({ installationFile, secretsFile });

  expect(models.resolve("openai-mini")).toEqual({
    id: "openai-mini",
    provider: "openai",
    model: "gpt-4.1-mini",
    endpoint: { baseUrl: "https://api.openai.example/v1", apiVersion: "2026-08-01" },
    apiKeyEnv: "OPENAI_API_KEY",
    apiKey: "raw-provider-secret",
  });
  expect(() => models.resolve("unknown-model")).toThrow(MetadataGenerationModelUnavailableError);
});

test("loads DeepSeek models, GLM, and an explicit keyless Qwen endpoint from installation setup", () => {
  const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration:
  default: glm-53
  models:
    - id: deepseek-v4-pro
      label: DeepSeek V4 Pro
      litellm: {provider: deepseek, model: deepseek-v4-pro}
      apiKeyEnv: DEEPSEEK_API_KEY
    - id: deepseek-v4-flash
      label: DeepSeek V4 Flash
      litellm: {provider: deepseek, model: deepseek-v4-flash}
      apiKeyEnv: DEEPSEEK_API_KEY
    - id: glm-53
      label: GLM 5.3
      litellm:
        provider: openai
        model: glm-5.3
        endpoint: {baseUrl: https://api.z.ai/api/coding/paas/v4}
      apiKeyEnv: ZAI_API_KEY
    - id: qwen-36
      label: Qwen 3.6
      litellm:
        provider: openai
        model: qwen3.6-35b-a3b
        disableThinking: true
        endpoint: {baseUrl: https://models.internal.example/v1}
`, "DEEPSEEK_API_KEY=deepseek-secret\nZAI_API_KEY=zai-secret\n");

  const models = loadMetadataGenerationModels({ installationFile, secretsFile });

  expect(models.catalog()).toEqual({
    models: [
      { id: "deepseek-v4-pro", label: "DeepSeek V4 Pro" },
      { id: "deepseek-v4-flash", label: "DeepSeek V4 Flash" },
      { id: "glm-53", label: "GLM 5.3" },
      { id: "qwen-36", label: "Qwen 3.6" },
    ],
    default: "glm-53",
  });
  expect(models.resolve("deepseek-v4-pro")).toMatchObject({
    apiKeyEnv: "DEEPSEEK_API_KEY",
    apiKey: "deepseek-secret",
  });
  expect(models.resolve("qwen-36")).toEqual({
    id: "qwen-36",
    provider: "openai",
    model: "qwen3.6-35b-a3b",
    disableThinking: true,
    endpoint: { baseUrl: "https://models.internal.example/v1" },
  });
});

test("loads an explicit keyless endpoint without a secret bundle", () => {
  const { installationFile } = metadataConfiguration(`metadataGeneration:
  default: qwen-36
  models:
    - id: qwen-36
      label: Qwen 3.6
      litellm:
        provider: openai
        model: qwen3.6-35b-a3b
        disableThinking: true
        endpoint: {baseUrl: https://models.internal.example/v1}
`);

  expect(loadMetadataGenerationModels({ installationFile }).resolve("qwen-36")).toEqual({
    id: "qwen-36",
    provider: "openai",
    model: "qwen3.6-35b-a3b",
    disableThinking: true,
    endpoint: { baseUrl: "https://models.internal.example/v1" },
  });
});

test.each([
  ["invalid YAML", "metadataGeneration: [\n", "OPENAI_API_KEY=secret\n", /invalid YAML/],
  ["duplicate ids", `metadataGeneration:
  default: openai-mini
  models:
    - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY}
    - {id: openai-mini, label: Two, litellm: {provider: openai, model: gpt-4.1}, apiKeyEnv: OPENAI_API_KEY}
`, "OPENAI_API_KEY=secret\n", /model id "openai-mini" is duplicated/],
  ["missing default", `metadataGeneration:
  models:
    - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY}
`, "OPENAI_API_KEY=secret\n", /default is required/],
  ["unknown default", `metadataGeneration:
  default: absent
  models:
    - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY}
`, "OPENAI_API_KEY=secret\n", /default "absent" is not configured/],
  ["malformed settings", `metadataGeneration:
  default: openai-mini
  models:
    - {id: openai-mini, label: One, litellm: {provider: "open ai", model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY}
`, "OPENAI_API_KEY=secret\n", /configuration is invalid/],
  ["malformed endpoint", `metadataGeneration:
  default: openai-mini
  models:
    - id: openai-mini
      label: One
      litellm: {provider: openai, model: gpt-4.1-mini, endpoint: {baseUrl: not-a-url}}
      apiKeyEnv: OPENAI_API_KEY
`, "OPENAI_API_KEY=secret\n", /configuration is invalid/],
  ["keyless hosted model without endpoint", `metadataGeneration:
  default: openai-mini
  models:
    - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}}
`, "", /configuration is invalid/],
  ["disable thinking without endpoint", `metadataGeneration:
  default: openai-mini
  models:
    - id: openai-mini
      label: One
      litellm: {provider: openai, model: gpt-4.1-mini, disableThinking: true}
      apiKeyEnv: OPENAI_API_KEY
`, "OPENAI_API_KEY=secret\n", /configuration is invalid/],
  ["unallowed secret reference", `metadataGeneration:
  default: openai-mini
  models:
    - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: THT_DWH_API_KEY}
`, "THT_DWH_API_KEY=secret\n", /configuration is invalid/],
  ["missing referenced secret", `metadataGeneration:
  default: openai-mini
  models:
    - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY}
`, "THT_DWH_API_KEY=secret\n", /secret "OPENAI_API_KEY" is missing/],
  ["unusable referenced secret", `metadataGeneration:
  default: openai-mini
  models:
    - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY}
`, "OPENAI_API_KEY=secret with whitespace\n", /secret "OPENAI_API_KEY" is unusable/],
] as const)("rejects %s metadata-generation configuration", (_name, yaml, secrets, expected) => {
  const { installationFile, secretsFile } = metadataConfiguration(yaml, secrets);
  expect(() => loadMetadataGenerationModels({ installationFile, secretsFile })).toThrow(expected);
});

test("rejects a missing secret-bundle declaration for configured models", () => {
  const { installationFile } = metadataConfiguration(`metadataGeneration:
  default: openai-mini
  models:
    - {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY}
`);

  expect(() => loadMetadataGenerationModels({ installationFile }))
    .toThrow("metadata-generation keyed models require THT_SECRETS_FILE");
});