ThothII/backend/test/list-models.test.ts

import { test, expect } from "vitest";
import { spawn } from "node:child_process";
import { mkdtempSync, writeFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import path from "node:path";
import { createPiModelLister } from "../src/pi/list-models.js";
import { loadConfig } from "../src/config.js";
import type { RuntimeModel, RuntimeModelCatalog } from "../src/models/runtime-model-catalog.js";

const FAKE = path.resolve("../harness/tests/fake_pi/fake_pi_rpc.mjs");

function scriptWith(models: unknown[]): string {
  const dir = mkdtempSync(join(tmpdir(), "tht-models-"));
  const file = join(dir, "models.json");
  writeFileSync(file, JSON.stringify({ available_models: models }));
  return file;
}

function enabled(...ids: string[]) {
  return () => ({ ids, warnings: [], source: "/test/settings.json" });
}

const noManagedModels = { readModelsStore: () => undefined };
const MANAGED_CONFIG_ERROR = "Pi provider/model configuration is invalid";

test("createPiModelLister returns mapped PiModel[] from get_available_models", async () => {
  const script = scriptWith([
    { provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true, extra: "ignored" },
    { provider: "anthropic", id: "claude-opus-4-8", name: "Claude Opus 4.8", reasoning: true },
  ]);
  try {
    const lister = createPiModelLister(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
      ...noManagedModels,
      loadEnabledModels: enabled("zai/glm-5.2", "anthropic/claude-opus-4-8"),
      spawnFn: () => spawn("node", [FAKE, script]) as any,
    });
    const models = await lister();
    expect(models).toEqual([
      { provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true },
      { provider: "anthropic", id: "claude-opus-4-8", name: "Claude Opus 4.8", reasoning: true },
    ]);
  } finally {
    rmSync(path.dirname(script), { recursive: true, force: true });
  }
});

test("catalog listing translates upstream Pi IDs back to canonical model keys", async () => {
  const script = scriptWith([
    { provider: "local", id: "qwen2.5:7b", name: "Upstream label", reasoning: false },
  ]);
  const model: RuntimeModel = {
    id: "local/qwen", provider: "local", model: "qwen", label: "Catalog Qwen",
    upstreamModel: "qwen2.5:7b", endpoint: { baseUrl: "http://ollama:11434/v1" },
    authentication: { mode: "none" }, sessionAdapter: { mode: "openai_compatible" },
    session: { reasoning: true, contextWindow: 32768, maxTokens: 8192 },
  };
  const modelCatalog: RuntimeModelCatalog = {
    defaultInteraction: model.id, embedding: null,
    sessionModels: () => [model], metadataModels: () => [], hasSession: (id) => id === model.id,
  };
  try {
    const lister = createPiModelLister(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
      ...noManagedModels,
      modelCatalog,
      loadEnabledModels: enabled("local/qwen2.5:7b"),
      spawnFn: () => spawn("node", [FAKE, script]) as any,
    });

    await expect(lister()).resolves.toEqual([{
      provider: "local", id: "qwen", name: "Catalog Qwen", reasoning: true,
    }]);
  } finally {
    rmSync(path.dirname(script), { recursive: true, force: true });
  }
});

test("catalog listing supplies the shared DeepSeek key without requiring Pi auth", async () => {
  const script = scriptWith([{ provider: "deepseek", id: "deepseek-v4-pro", name: "DeepSeek V4 Pro" }]);
  const secret = join(path.dirname(script), "thothii.secrets");
  writeFileSync(secret, "DEEPSEEK_API_KEY=shared-key\nOPENAI_API_KEY=unrelated-key\n", { mode: 0o600 });
  const model: RuntimeModel = {
    id: "deepseek/deepseek-v4-pro", provider: "deepseek", model: "deepseek-v4-pro",
    label: "DeepSeek V4 Pro", upstreamModel: "deepseek-v4-pro",
    authentication: { mode: "secret_env", apiKeyEnv: "DEEPSEEK_API_KEY" },
    sessionAdapter: { mode: "pi_builtin" }, session: { reasoning: false },
  };
  const modelCatalog: RuntimeModelCatalog = {
    defaultInteraction: model.id, embedding: null,
    sessionModels: () => [model], metadataModels: () => [], hasSession: (id) => id === model.id,
  };
  try {
    const lister = createPiModelLister(loadConfig({ THT_SECRETS_FILE: secret }), {
      ...noManagedModels, modelCatalog, loadEnabledModels: enabled(model.id),
      spawnFn: (_command, _args, options) => {
        expect(options.env.DEEPSEEK_API_KEY).toBe("shared-key");
        expect(options.env).not.toHaveProperty("OPENAI_API_KEY");
        expect(options.env).not.toHaveProperty("THT_SECRETS_FILE");
        return spawn("node", [FAKE, script], { env: options.env }) as any;
      },
    });
    await expect(lister()).resolves.toEqual([{
      provider: model.provider, id: model.model, name: model.label, reasoning: false,
    }]);
  } finally {
    rmSync(path.dirname(script), { recursive: true, force: true });
  }
});

test("createPiModelLister caches within ttl (spawns once for two calls)", async () => {
  const script = scriptWith([{ provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true }]);
  try {
    let spawns = 0;
    const lister = createPiModelLister(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
      ...noManagedModels,
      loadEnabledModels: enabled("zai/glm-5.2"),
      spawnFn: () => { spawns++; return spawn("node", [FAKE, script]) as any; },
      ttlMs: 10_000,
      nowMs: () => 1000,
    });
    await lister();
    await lister();
    expect(spawns).toBe(1);
  } finally {
    rmSync(path.dirname(script), { recursive: true, force: true });
  }
});

test("production model-list spawn preserves PATH and passes the portable data root", async () => {
  const script = scriptWith([]);
  const calls: any[][] = [];
  const previousPath = process.env.PATH;
  process.env.PATH = "/usr/local/bin:/usr/bin";
  try {
    const lister = createPiModelLister(loadConfig({
      THT_HARNESS_DIR: "/app/harness",
      PI_BIN: "/usr/local/bin/pi",
      THT_DATA_ROOT: "/data",
    }), {
      ...noManagedModels,
      loadEnabledModels: enabled("test/unavailable"),
      spawnFn: (...args: any[]) => {
        calls.push(args);
        return spawn("node", [FAKE, script]) as any;
      },
    });
    await lister();
    expect(calls[0][0]).toBe("/usr/local/bin/pi");
    expect(calls[0][1]).toEqual(["--mode", "rpc"]);
    expect(calls[0][2]).toMatchObject({
      cwd: "/app/harness",
      env: expect.objectContaining({ PATH: "/usr/local/bin:/usr/bin", THT_DATA_ROOT: "/data" }),
    });
  } finally {
    if (previousPath === undefined) delete process.env.PATH;
    else process.env.PATH = previousPath;
    rmSync(path.dirname(script), { recursive: true, force: true });
  }
});

test("model-list spawn scrubs ambient provider credentials and generic secret metadata", async () => {
  const script = scriptWith([]);
  const calls: any[][] = [];
  const previousDataRoot = process.env.THT_DATA_ROOT;
  const previousCredential = process.env.PI_PROVIDER_API_KEY;
  process.env.THT_DATA_ROOT = "/ambient-must-not-leak";
  process.env.PI_PROVIDER_API_KEY = "must-not-leak";
  process.env.OPENAI_API_KEY = "must-not-leak";
  process.env.AWS_SECRET_ACCESS_KEY = "must-not-leak";
  process.env.CLOUDFLARE_ACCOUNT_ID = "must-not-leak";
  try {
    const lister = createPiModelLister(loadConfig({ PI_BIN: "/usr/local/bin/pi" }), {
      ...noManagedModels,
      loadEnabledModels: enabled("test/unavailable"),
      spawnFn: (...args: any[]) => {
        calls.push(args);
        return spawn("node", [FAKE, script]) as any;
      },
    });
    await lister();
    expect(calls[0][2].env).not.toHaveProperty("THT_DATA_ROOT");
    expect(calls[0][2].env).not.toHaveProperty("PI_PROVIDER_API_KEY");
    expect(calls[0][2].env).not.toHaveProperty("OPENAI_API_KEY");
    expect(calls[0][2].env).not.toHaveProperty("AWS_SECRET_ACCESS_KEY");
    expect(calls[0][2].env).not.toHaveProperty("CLOUDFLARE_ACCOUNT_ID");
  } finally {
    if (previousDataRoot === undefined) delete process.env.THT_DATA_ROOT;
    else process.env.THT_DATA_ROOT = previousDataRoot;
    if (previousCredential === undefined) delete process.env.PI_PROVIDER_API_KEY;
    else process.env.PI_PROVIDER_API_KEY = previousCredential;
    delete process.env.OPENAI_API_KEY;
    delete process.env.AWS_SECRET_ACCESS_KEY;
    delete process.env.CLOUDFLARE_ACCOUNT_ID;
    rmSync(path.dirname(script), { recursive: true, force: true });
  }
});

test("model listing does not require PI_PROVIDER or read the generic credential", async () => {
  const script = scriptWith([
    { provider: "zai", id: "glm-5.2", name: "GLM-5.2", reasoning: true },
  ]);
  const calls: any[][] = [];
  try {
    const lister = createPiModelLister(loadConfig({
      PI_BIN: "/usr/local/bin/pi",
      THT_MODEL_API_KEY_FILE: "/missing-and-must-not-be-read",
    }), {
      ...noManagedModels,
      loadEnabledModels: enabled("zai/glm-5.2"),
      spawnFn: (...args: any[]) => {
        calls.push(args);
        return spawn("node", [FAKE, script]) as any;
      },
    });
    await expect(lister()).resolves.toHaveLength(1);
    expect(calls[0][2].env).not.toHaveProperty("ZAI_API_KEY");
    expect(calls[0][2].env).not.toHaveProperty("THT_MODEL_API_KEY_FILE");
  } finally {
    rmSync(path.dirname(script), { recursive: true, force: true });
  }
});

test("returns only enabled available models in enabledModels order", async () => {
  const script = scriptWith([
    { provider: "zai", id: "glm-5v-turbo", name: "GLM-5V-Turbo", reasoning: true },
    { provider: "local-qwen", id: "qwen3.6-35b-a3b", name: "Qwen3.6 Local", reasoning: false },
    { provider: "zai", id: "glm-5.2", name: "GLM-5.2", reasoning: true },
    { provider: "deepseek", id: "deepseek-v4-flash", name: "DeepSeek V4 Flash", reasoning: true },
  ]);
  try {
    const lister = createPiModelLister(loadConfig({}), {
      ...noManagedModels,
      loadEnabledModels: enabled(
        "zai/glm-5.2",
        "deepseek/deepseek-v4-flash",
        "local-qwen/qwen3.6-35b-a3b",
      ),
      spawnFn: () => spawn("node", [FAKE, script]) as any,
    });
    expect((await lister()).map((m) => `${m.provider}/${m.id}`)).toEqual([
      "zai/glm-5.2",
      "deepseek/deepseek-v4-flash",
      "local-qwen/qwen3.6-35b-a3b",
    ]);
  } finally {
    rmSync(path.dirname(script), { recursive: true, force: true });
  }
});

test("empty enabled model scope fails closed without spawning Pi", async () => {
  let spawns = 0;
  const warnings: string[] = [];
  const lister = createPiModelLister(loadConfig({}), {
    ...noManagedModels,
    loadEnabledModels: () => ({ ids: [], warnings: ["scope invalid"] }),
    warn: (message) => warnings.push(message),
    spawnFn: () => { spawns += 1; throw new Error("must not spawn"); },
  });
  await expect(lister()).resolves.toEqual([]);
  expect(spawns).toBe(0);
  expect(warnings).toEqual(["scope invalid"]);
});

test("warns and returns empty when enabled identifiers are unavailable", async () => {
  const script = scriptWith([
    { provider: "zai", id: "glm-5v-turbo", name: "GLM-5V-Turbo", reasoning: true },
  ]);
  const warnings: string[] = [];
  try {
    const lister = createPiModelLister(loadConfig({}), {
      ...noManagedModels,
      loadEnabledModels: enabled("zai/glm-5.2"),
      warn: (message) => warnings.push(message),
      spawnFn: () => spawn("node", [FAKE, script]) as any,
    });
    await expect(lister()).resolves.toEqual([]);
    expect(warnings).toEqual(["No Pi-enabled models are currently available"]);
  } finally {
    rmSync(path.dirname(script), { recursive: true, force: true });
  }
});

const executableModelsConfigCases: Array<[string, unknown]> = [
  ["nested provider headers", {
    providers: {
      selected: {
        headers: { Authorization: "!sensitive-header-command /private/header-path" },
      },
    },
  }],
  ["provider apiKey", {
    providers: {
      selected: { apiKey: "!sensitive-api-key-command /private/key-path" },
    },
  }],
  ["selected model objects", {
    providers: {
      selected: {
        models: [{ id: "model", name: "!sensitive-model-command /private/model-path" }],
      },
    },
  }],
  ["selected model overrides", {
    providers: {
      selected: {
        modelOverrides: {
          model: { headers: { "X-Override": "!sensitive-override-command /private/override-path" } },
        },
      },
    },
  }],
  ["nested arrays", {
    providers: {
      selected: {
        compat: { nested: ["literal", { value: "!sensitive-array-command /private/array-path" }] },
      },
    },
  }],
];

// Catches Task 8 model discovery delegating raw managed models.json values to Pi. Pi 0.80.3
// executes leading-! values at request time, so the complete managed store must be rejected before
// it can become an authoritative source of API choices.
test.each(executableModelsConfigCases)(
  "managed models ingestion rejects executable strings in %s",
  async (_name, modelsConfig) => {
    let spawns = 0;
    const lister = createPiModelLister(loadConfig({}), {
      loadEnabledModels: enabled("selected/model"),
      readModelsStore: () => JSON.stringify(modelsConfig),
      spawnFn: () => {
        spawns += 1;
        throw new Error("unsafe model-list spawn");
      },
    });

    let caught: unknown;
    try {
      await lister();
    } catch (error) {
      caught = error;
    }
    expect(caught).toBeInstanceOf(Error);
    expect((caught as Error).message).toBe(MANAGED_CONFIG_ERROR);
    expect(String(caught)).not.toMatch(/sensitive|private|command|path/i);
    expect(spawns).toBe(0);
  },
);

// Selection-time smoke filtering intentionally ignores unrelated providers, but the full
// installation-owned models.json is invalid at the model-choice ingestion boundary.
test("managed models ingestion rejects an executable string in an unrelated provider", async () => {
  let spawns = 0;
  const lister = createPiModelLister(loadConfig({}), {
    loadEnabledModels: enabled("selected/model"),
    readModelsStore: () => JSON.stringify({
      providers: {
        selected: { models: [{ id: "model" }] },
        unrelated: { apiKey: "!sensitive-unrelated-command /private/unrelated-path" },
      },
    }),
    spawnFn: () => {
      spawns += 1;
      throw new Error("unsafe model-list spawn");
    },
  });

  await expect(lister()).rejects.toThrow(MANAGED_CONFIG_ERROR);
  expect(spawns).toBe(0);
});

test("managed models ingestion revalidates the store before serving a cached choice", async () => {
  const script = scriptWith([
    { provider: "selected", id: "model", name: "Selected model", reasoning: true },
  ]);
  let managedModels = JSON.stringify({
    providers: { selected: { models: [{ id: "model" }] } },
  });
  let spawns = 0;
  try {
    const lister = createPiModelLister(loadConfig({}), {
      loadEnabledModels: enabled("selected/model"),
      readModelsStore: () => managedModels,
      spawnFn: () => {
        spawns += 1;
        return spawn("node", [FAKE, script]) as any;
      },
      ttlMs: 10_000,
    });

    await expect(lister()).resolves.toHaveLength(1);
    managedModels = JSON.stringify({
      providers: { selected: { headers: { Authorization: "!new-unsafe-value" } } },
    });

    await expect(lister()).rejects.toThrow(MANAGED_CONFIG_ERROR);
    expect(spawns).toBe(1);
  } finally {
    rmSync(path.dirname(script), { recursive: true, force: true });
  }
});