ThothII/backend/test/list-models.test.ts

import { test, expect } from "vitest";
import { spawn } from "node:child_process";
import { mkdtempSync, writeFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import path from "node:path";
import { createPiModelLister } from "../src/pi/list-models.js";
import { loadConfig } from "../src/config.js";

const FAKE = path.resolve("../harness/tests/fake_pi/fake_pi_rpc.mjs");

function scriptWith(models: unknown[]): string {
  const dir = mkdtempSync(join(tmpdir(), "tht-models-"));
  const file = join(dir, "models.json");
  writeFileSync(file, JSON.stringify({ available_models: models }));
  return file;
}

function enabled(...ids: string[]) {
  return () => ({ ids, warnings: [], source: "/test/settings.json" });
}

test("createPiModelLister returns mapped PiModel[] from get_available_models", async () => {
  const script = scriptWith([
    { provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true, extra: "ignored" },
    { provider: "anthropic", id: "claude-opus-4-8", name: "Claude Opus 4.8", reasoning: true },
  ]);
  try {
    const lister = createPiModelLister(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
      loadEnabledModels: enabled("zai/glm-5.2", "anthropic/claude-opus-4-8"),
      spawnFn: () => spawn("node", [FAKE, script]) as any,
    });
    const models = await lister();
    expect(models).toEqual([
      { provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true },
      { provider: "anthropic", id: "claude-opus-4-8", name: "Claude Opus 4.8", reasoning: true },
    ]);
  } finally {
    rmSync(path.dirname(script), { recursive: true, force: true });
  }
});

test("createPiModelLister caches within ttl (spawns once for two calls)", async () => {
  const script = scriptWith([{ provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true }]);
  try {
    let spawns = 0;
    const lister = createPiModelLister(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
      loadEnabledModels: enabled("zai/glm-5.2"),
      spawnFn: () => { spawns++; return spawn("node", [FAKE, script]) as any; },
      ttlMs: 10_000,
      nowMs: () => 1000,
    });
    await lister();
    await lister();
    expect(spawns).toBe(1);
  } finally {
    rmSync(path.dirname(script), { recursive: true, force: true });
  }
});

test("production model-list spawn preserves PATH and passes the portable data root", async () => {
  const script = scriptWith([]);
  const calls: any[][] = [];
  const previousPath = process.env.PATH;
  process.env.PATH = "/usr/local/bin:/usr/bin";
  try {
    const lister = createPiModelLister(loadConfig({
      THT_HARNESS_DIR: "/app/harness",
      PI_BIN: "/usr/local/bin/pi",
      THT_DATA_ROOT: "/data",
    }), {
      loadEnabledModels: enabled("test/unavailable"),
      spawnFn: (...args: any[]) => {
        calls.push(args);
        return spawn("node", [FAKE, script]) as any;
      },
    });
    await lister();
    expect(calls[0][0]).toBe("/usr/local/bin/pi");
    expect(calls[0][1]).toEqual(["--mode", "rpc"]);
    expect(calls[0][2]).toMatchObject({
      cwd: "/app/harness",
      env: expect.objectContaining({ PATH: "/usr/local/bin:/usr/bin", THT_DATA_ROOT: "/data" }),
    });
  } finally {
    if (previousPath === undefined) delete process.env.PATH;
    else process.env.PATH = previousPath;
    rmSync(path.dirname(script), { recursive: true, force: true });
  }
});

test("model-list spawn scrubs ambient provider credentials and generic secret metadata", async () => {
  const script = scriptWith([]);
  const calls: any[][] = [];
  const previousDataRoot = process.env.THT_DATA_ROOT;
  const previousCredential = process.env.PI_PROVIDER_API_KEY;
  process.env.THT_DATA_ROOT = "/ambient-must-not-leak";
  process.env.PI_PROVIDER_API_KEY = "must-not-leak";
  process.env.OPENAI_API_KEY = "must-not-leak";
  process.env.AWS_SECRET_ACCESS_KEY = "must-not-leak";
  process.env.CLOUDFLARE_ACCOUNT_ID = "must-not-leak";
  try {
    const lister = createPiModelLister(loadConfig({ PI_BIN: "/usr/local/bin/pi" }), {
      loadEnabledModels: enabled("test/unavailable"),
      spawnFn: (...args: any[]) => {
        calls.push(args);
        return spawn("node", [FAKE, script]) as any;
      },
    });
    await lister();
    expect(calls[0][2].env).not.toHaveProperty("THT_DATA_ROOT");
    expect(calls[0][2].env).not.toHaveProperty("PI_PROVIDER_API_KEY");
    expect(calls[0][2].env).not.toHaveProperty("OPENAI_API_KEY");
    expect(calls[0][2].env).not.toHaveProperty("AWS_SECRET_ACCESS_KEY");
    expect(calls[0][2].env).not.toHaveProperty("CLOUDFLARE_ACCOUNT_ID");
  } finally {
    if (previousDataRoot === undefined) delete process.env.THT_DATA_ROOT;
    else process.env.THT_DATA_ROOT = previousDataRoot;
    if (previousCredential === undefined) delete process.env.PI_PROVIDER_API_KEY;
    else process.env.PI_PROVIDER_API_KEY = previousCredential;
    delete process.env.OPENAI_API_KEY;
    delete process.env.AWS_SECRET_ACCESS_KEY;
    delete process.env.CLOUDFLARE_ACCOUNT_ID;
    rmSync(path.dirname(script), { recursive: true, force: true });
  }
});

test("model listing does not require PI_PROVIDER or read the generic credential", async () => {
  const script = scriptWith([
    { provider: "zai", id: "glm-5.2", name: "GLM-5.2", reasoning: true },
  ]);
  const calls: any[][] = [];
  try {
    const lister = createPiModelLister(loadConfig({
      PI_BIN: "/usr/local/bin/pi",
      THT_MODEL_API_KEY_FILE: "/missing-and-must-not-be-read",
    }), {
      loadEnabledModels: enabled("zai/glm-5.2"),
      spawnFn: (...args: any[]) => {
        calls.push(args);
        return spawn("node", [FAKE, script]) as any;
      },
    });
    await expect(lister()).resolves.toHaveLength(1);
    expect(calls[0][2].env).not.toHaveProperty("ZAI_API_KEY");
    expect(calls[0][2].env).not.toHaveProperty("THT_MODEL_API_KEY_FILE");
  } finally {
    rmSync(path.dirname(script), { recursive: true, force: true });
  }
});

test("returns only enabled available models in enabledModels order", async () => {
  const script = scriptWith([
    { provider: "zai", id: "glm-5v-turbo", name: "GLM-5V-Turbo", reasoning: true },
    { provider: "local-qwen", id: "qwen3.6-35b-a3b", name: "Qwen3.6 Local", reasoning: false },
    { provider: "zai", id: "glm-5.2", name: "GLM-5.2", reasoning: true },
    { provider: "deepseek", id: "deepseek-v4-flash", name: "DeepSeek V4 Flash", reasoning: true },
  ]);
  try {
    const lister = createPiModelLister(loadConfig({}), {
      loadEnabledModels: enabled(
        "zai/glm-5.2",
        "deepseek/deepseek-v4-flash",
        "local-qwen/qwen3.6-35b-a3b",
      ),
      spawnFn: () => spawn("node", [FAKE, script]) as any,
    });
    expect((await lister()).map((m) => `${m.provider}/${m.id}`)).toEqual([
      "zai/glm-5.2",
      "deepseek/deepseek-v4-flash",
      "local-qwen/qwen3.6-35b-a3b",
    ]);
  } finally {
    rmSync(path.dirname(script), { recursive: true, force: true });
  }
});

test("empty enabled model scope fails closed without spawning Pi", async () => {
  let spawns = 0;
  const warnings: string[] = [];
  const lister = createPiModelLister(loadConfig({}), {
    loadEnabledModels: () => ({ ids: [], warnings: ["scope invalid"] }),
    warn: (message) => warnings.push(message),
    spawnFn: () => { spawns += 1; throw new Error("must not spawn"); },
  });
  await expect(lister()).resolves.toEqual([]);
  expect(spawns).toBe(0);
  expect(warnings).toEqual(["scope invalid"]);
});

test("warns and returns empty when enabled identifiers are unavailable", async () => {
  const script = scriptWith([
    { provider: "zai", id: "glm-5v-turbo", name: "GLM-5V-Turbo", reasoning: true },
  ]);
  const warnings: string[] = [];
  try {
    const lister = createPiModelLister(loadConfig({}), {
      loadEnabledModels: enabled("zai/glm-5.2"),
      warn: (message) => warnings.push(message),
      spawnFn: () => spawn("node", [FAKE, script]) as any,
    });
    await expect(lister()).resolves.toEqual([]);
    expect(warnings).toEqual(["No Pi-enabled models are currently available"]);
  } finally {
    rmSync(path.dirname(script), { recursive: true, force: true });
  }
});