ThothII/backend/test/pi-management.test.ts

import { mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expect, test, vi } from "vitest";
import { loadConfig } from "../src/config.js";
import {
  PiManagementError,
  createPiManagement,
  type PiExecFile,
} from "../src/pi/management.js";

function configFor(settingsFile = join(mkdtempSync(join(tmpdir(), "tht-pi-management-")), "settings.json")) {
  return loadConfig({
    THT_HARNESS_DIR: "../harness",
    SETTINGS_FILE: settingsFile,
    PI_BIN: "/usr/local/bin/pi",
    PI_MANAGEMENT_TIMEOUT_MS: "750",
  });
}

const supportedModels = [
  { provider: "zai", id: "glm-5.2", name: "GLM 5.2", reasoning: true },
  { provider: "deepseek", id: "deepseek-v4", name: "DeepSeek V4", reasoning: true },
];

function successfulExec(calls: Array<{ command: string; args: string[]; timeout: number }>): PiExecFile {
  return async (command, args, options) => {
    calls.push({ command, args, timeout: options.timeout });
    return { stdout: "pi 0.80.3\n", stderr: "" };
  };
}

// Catches a Pi executable that emits unexpected text or is invoked through a shell, which could
// turn a version display into a command-injection or information-disclosure surface.
test("status parses only a Pi version from a fixed execFile argument array", async () => {
  const calls: Array<{ command: string; args: string[]; timeout: number }> = [];
  const service = createPiManagement(configFor(), {
    execute: successfulExec(calls),
    listModels: async () => supportedModels,
    readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }),
    credentialStatus: () => "missing",
    now: () => new Date("2026-08-05T10:00:00.000Z"),
  });

  await expect(service.status()).resolves.toEqual({
    version: "0.80.3",
    ready: true,
    credentials: "missing",
    config: { provider: "zai", model: "glm-5.2", reasoning: "medium" },
    checkedAt: "2026-08-05T10:00:00.000Z",
  });
  expect(calls).toHaveLength(1);
  expect(calls[0]).toMatchObject({ command: "/usr/local/bin/pi", args: ["--version"] });
  expect(calls[0].timeout).toBeGreaterThan(0);
  expect(calls[0].timeout).toBeLessThanOrEqual(750);
});

// Catches credential presence being inferred from smoke success/failure or exposing any
// credential material instead of the installation's explicit sanitized presence state.
test.each(["present", "missing"] as const)(
  "status reports configured-provider credentials only as %s",
  async (credentials) => {
    const checkedProviders: Array<string | undefined> = [];
    const service = createPiManagement(configFor(), {
      execute: successfulExec([]),
      listModels: async () => supportedModels,
      readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }),
      credentialStatus: (provider) => {
        checkedProviders.push(provider);
        return credentials;
      },
      now: () => new Date("2026-08-05T10:00:00.000Z"),
    });

    const status = await service.status();
    expect(status).toEqual({
      version: "0.80.3",
      ready: true,
      credentials,
      config: { provider: "zai", model: "glm-5.2", reasoning: "medium" },
      checkedAt: "2026-08-05T10:00:00.000Z",
    });
    expect(checkedProviders).toEqual(["zai"]);
    expect(JSON.stringify(status)).not.toMatch(/api.?key|token|password|secret/i);
  },
);

// Catches an options response that leaks provider metadata or lets callers choose model IDs that
// Pi did not explicitly enable for this installation.
test("options expose only closed provider, model, and reasoning choices", async () => {
  const service = createPiManagement(configFor(), {
    execute: successfulExec([]),
    listModels: async () => supportedModels,
    now: () => new Date("2026-08-05T10:00:00.000Z"),
  });

  await expect(service.options()).resolves.toEqual({
    providers: ["zai", "deepseek"],
    models: [
      { provider: "zai", id: "glm-5.2" },
      { provider: "deepseek", id: "deepseek-v4" },
    ],
    reasoning: ["low", "medium", "high"],
    checkedAt: "2026-08-05T10:00:00.000Z",
  });
});

// Catches raw managed models.json validation details being collapsed into an ambiguous model-list
// failure or escaping through the Pi Management options API.
test("options report invalid managed model configuration with a stable sanitized error", async () => {
  const service = createPiManagement(configFor(), {
    execute: successfulExec([]),
    listModels: async () => {
      throw Object.assign(
        new Error("!sensitive-command /private/models.json raw-secret"),
        { code: "PI_MANAGED_CONFIG_INVALID" },
      );
    },
  });

  let caught: unknown;
  try {
    await service.options();
  } catch (error) {
    caught = error;
  }
  expect(caught).toMatchObject<PiManagementError>({
    code: "pi_management_unavailable",
    message: "Pi provider/model configuration is invalid",
  });
  expect(String(caught)).not.toMatch(/sensitive|private|models\.json|secret/i);
});

// Catches configuration writes that accept whitespace, unknown choices, or extra free-form fields
// before reaching the durable installation settings file.
test("config rejects invalid free-form values before writing settings", async () => {
  const directory = mkdtempSync(join(tmpdir(), "tht-pi-management-invalid-"));
  try {
    let writes = 0;
    const service = createPiManagement(configFor(join(directory, "settings.json")), {
      execute: successfulExec([]),
      listModels: async () => supportedModels,
      readSettings: () => ({}),
      saveSettings: () => { writes += 1; return {}; },
    });

    await expect(service.configure({
      provider: "zai ", model: "glm-5.2", reasoning: "medium", unexpected: "value",
    } as any)).rejects.toMatchObject<PiManagementError>({ code: "pi_management_invalid_config" });
    expect(writes).toBe(0);
  } finally {
    rmSync(directory, { recursive: true, force: true });
  }
});

// Catches a non-atomic implementation that can leave partial settings or temporary files after a
// normal installation-default update.
test("config validates closed choices and atomically persists non-secret defaults", async () => {
  const directory = mkdtempSync(join(tmpdir(), "tht-pi-management-write-"));
  const settingsFile = join(directory, "settings.json");
  try {
    const service = createPiManagement(configFor(settingsFile), {
      execute: successfulExec([]),
      listModels: async () => supportedModels,
      now: () => new Date("2026-08-05T10:00:00.000Z"),
    });

    await expect(service.configure({
      provider: "zai", model: "glm-5.2", reasoning: "high",
    })).resolves.toEqual({
      provider: "zai", model: "glm-5.2", reasoning: "high", updatedAt: "2026-08-05T10:00:00.000Z",
    });
    expect(JSON.parse(readFileSync(settingsFile, "utf8"))).toEqual({
      provider: "zai", model: "glm-5.2", thinking: "high",
    });
    expect(readdirSync(directory)).toEqual(["settings.json"]);
  } finally {
    rmSync(directory, { recursive: true, force: true });
  }
});

// Catches a hung Pi smoke check that leaves an operator waiting indefinitely or returns raw child
// diagnostics containing provider credentials.
test("smoke uses the configured timeout and reports a sanitized timeout", async () => {
  const calls: Array<{ command: string; args: string[]; timeout: number }> = [];
  const service = createPiManagement(configFor(), {
    execute: async (command, args, options) => {
      calls.push({ command, args, timeout: options.timeout });
      throw Object.assign(new Error("provider token=raw-provider-token"), { code: "ETIMEDOUT" });
    },
    listModels: async () => supportedModels,
    readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }),
    now: () => new Date("2026-08-05T10:00:00.000Z"),
  });

  await expect(service.test()).resolves.toEqual({
    ready: false,
    message: "Pi smoke check timed out",
    checkedAt: "2026-08-05T10:00:00.000Z",
  });
  expect(calls).toHaveLength(1);
  expect(calls[0]).toMatchObject({ command: "/usr/local/bin/pi", args: ["--version"] });
  expect(calls[0]!.timeout).toBeGreaterThan(0);
  expect(calls[0]!.timeout).toBeLessThanOrEqual(750);
});

// Catches a smoke endpoint that validates only the Pi binary/model catalogue and never makes a
// request through the configured provider and model.
test("smoke exercises the configured provider and model", async () => {
  const providerChecks: unknown[] = [];
  const service = createPiManagement(configFor(), {
    execute: successfulExec([]),
    listModels: async () => supportedModels,
    readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }),
    smokeProvider: async (request) => { providerChecks.push(request); },
    now: () => new Date("2026-08-05T10:00:00.000Z"),
  });

  await expect(service.test()).resolves.toEqual({
    ready: true,
    checkedAt: "2026-08-05T10:00:00.000Z",
  });
  expect(providerChecks).toEqual([{
    provider: "zai", model: "glm-5.2", reasoning: "medium", timeoutMs: expect.any(Number),
  }]);
});

// Catches expired provider credentials being treated as ready or raw provider diagnostics being
// reflected through the management API.
test("smoke fails closed and sanitizes configured-provider authentication errors", async () => {
  const service = createPiManagement(configFor(), {
    execute: successfulExec([]),
    listModels: async () => supportedModels,
    readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }),
    smokeProvider: async () => {
      throw new Error('401 {"token":"raw-expired-token","output":"raw-provider-output"}');
    },
    now: () => new Date("2026-08-05T10:00:00.000Z"),
  });

  const result = await service.test();
  expect(result).toEqual({
    ready: false,
    message: "Pi provider smoke check failed",
    checkedAt: "2026-08-05T10:00:00.000Z",
  });
  expect(JSON.stringify(result)).not.toMatch(/raw-expired-token|raw-provider-output/);
});

// Catches selected auth/models validation failures being downgraded to a generic provider error
// or exposing the rejected command, path, or secret through POST /pi-management/test.
test("smoke reports invalid managed provider configuration with a stable sanitized error", async () => {
  const service = createPiManagement(configFor(), {
    execute: successfulExec([]),
    listModels: async () => supportedModels,
    readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }),
    smokeProvider: async () => {
      throw Object.assign(
        new Error("!sensitive-command /private/models.json raw-secret"),
        { code: "PI_MANAGED_CONFIG_INVALID" },
      );
    },
    now: () => new Date("2026-08-05T10:00:00.000Z"),
  });

  const result = await service.test();
  expect(result).toEqual({
    ready: false,
    message: "Pi provider/model configuration is invalid",
    checkedAt: "2026-08-05T10:00:00.000Z",
  });
  expect(JSON.stringify(result)).not.toMatch(/sensitive|private|models\.json|secret/i);
});

// Catches separate per-phase timeouts that allow a later provider turn to exceed the one
// end-to-end Pi Management smoke budget.
test("smoke applies one deadline across version and a hung provider turn", async () => {
  vi.useFakeTimers();
  vi.setSystemTime(new Date("2026-08-05T10:00:00.000Z"));
  try {
    const providerTimeouts: number[] = [];
    const service = createPiManagement(configFor(), {
      execute: async () => await new Promise((resolve) => setTimeout(
        () => resolve({ stdout: "pi 0.80.3\n", stderr: "" }),
        500,
      )),
      listModels: async () => supportedModels,
      readSettings: () => ({ provider: "zai", model: "glm-5.2", thinking: "medium" }),
      smokeProvider: async ({ timeoutMs }) => {
        providerTimeouts.push(timeoutMs);
        await new Promise(() => {});
      },
      now: () => new Date("2026-08-05T10:00:00.000Z"),
    });

    let settled = false;
    const pending = service.test().finally(() => { settled = true; });
    await vi.advanceTimersByTimeAsync(500);
    expect(providerTimeouts).toEqual([250]);
    await vi.advanceTimersByTimeAsync(249);
    expect(settled).toBe(false);
    await vi.advanceTimersByTimeAsync(1);
    await expect(pending).resolves.toEqual({
      ready: false,
      message: "Pi smoke check timed out",
      checkedAt: "2026-08-05T10:00:00.000Z",
    });
  } finally {
    vi.useRealTimers();
  }
});

// Catches an unbounded diagnostics endpoint or one that returns bearer tokens and connection
// passwords captured in Pi output.
test("logs keep only the latest 200 redacted lines", async () => {
  const source = Array.from({ length: 205 }, (_, index) => `line-${index + 1}`);
  source[203] = "Authorization: Bearer raw-bearer-token";
  source[204] = "database_url=postgres://thoth:raw-db-password@example.invalid/db";
  source[202] = '{"token":"raw-json-secret","password":"raw-json-password"}';
  source[201] = "THT_MODEL_API_KEY=raw-env-secret";
  const service = createPiManagement(configFor(), {
    execute: successfulExec([]),
    listModels: async () => supportedModels,
    readLogs: () => source.join("\n"),
    now: () => new Date("2026-08-05T10:00:00.000Z"),
  });

  const logs = await service.logs();
  expect(logs.checkedAt).toBe("2026-08-05T10:00:00.000Z");
  expect(logs.lines).toHaveLength(200);
  expect(logs.lines[0]).toBe("line-6");
  expect(logs.lines.join("\n")).not.toContain("raw-bearer-token");
  expect(logs.lines.join("\n")).not.toContain("raw-db-password");
  expect(logs.lines.join("\n")).not.toContain("raw-json-secret");
  expect(logs.lines.join("\n")).not.toContain("raw-json-password");
  expect(logs.lines.join("\n")).not.toContain("raw-env-secret");
  expect(logs.lines.join("\n")).toContain("[REDACTED]");
});