ThothII/backend/test/workspaces-diagnostics.test.ts

import { expect, test, vi } from "vitest";
import { EventEmitter } from "node:events";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
  createConcreteDiagnosticAdapters,
  createProductionWorkspaceDiagnoser,
  createWorkspaceDiagnoser,
  type DiagnosticAdapters,
} from "../src/workspaces/diagnostics.js";
import { resolveRuntimeBindings } from "../src/workspaces/bindings.js";
import type { RuntimeBindings } from "../src/workspaces/runtime-renderer.js";
import { parseWorkspaceYaml } from "../src/workspaces/schema.js";

const workspace = parseWorkspaceYaml(`workspace:
  schema_version: 2
  id: psd-clinical
  name: Policlinico San Donato
  language: it
dwh:
  engine: postgres
  database: warehouse
  schema: datawarehouse
  timeout_ms: 8000
  supported_transports: [postgres_direct, rest_api, ssh_tunnel]
semantic_index:
  vector_store:
    engine: pgvector
    database: postgres
    schema: vectors
    collection: clinical_documents
    dimensions: 768
    distance: cosine
    timeout_ms: 8000
    supported_transports: [pgvector_direct, rest_api, ssh_tunnel]
  embedding:
    provider: ollama_compatible
    model: nomic-embed-text-v2-moe
    dimensions: 768
    timeout_ms: 8000
llm_policy:
  allowed: [zai/glm-5.2]
diagnostics:
  dwh_rest:
    method: POST
    path: /rpc/ping
    auth: bearer
    response: { database: database, schema: schema }
  vector_rest:
    metadata:
      method: GET
      path: /vector/metadata
      auth: bearer
      response: { collection: collection, dimensions: dimensions, distance: distance }
    reversible_probe:
      method: POST
      path: /vector/diagnostic-probe
      auth: bearer
      response: { operation: operation }
`);

const writerWorkspace = parseWorkspaceYaml(`workspace:
  schema_version: 2
  id: psd-clinical
  name: Policlinico San Donato
  language: it
dwh:
  engine: postgres
  database: warehouse
  schema: datawarehouse
  timeout_ms: 8000
  supported_transports: [postgres_direct, rest_api, ssh_tunnel]
semantic_index:
  vector_store:
    engine: pgvector
    database: postgres
    schema: vectors
    collection: clinical_documents
    dimensions: 768
    distance: cosine
    timeout_ms: 8000
    supported_transports: [pgvector_direct, rest_api, ssh_tunnel]
  vector_writer: {}
  embedding:
    provider: ollama_compatible
    model: nomic-embed-text-v2-moe
    dimensions: 768
    timeout_ms: 8000
llm_policy:
  allowed: [zai/glm-5.2]
diagnostics:
  dwh_rest:
    method: POST
    path: /rpc/ping
    auth: bearer
    response: { database: database, schema: schema }
  vector_rest:
    metadata:
      method: GET
      path: /vector/metadata
      auth: bearer
      response: { collection: collection, dimensions: dimensions, distance: distance }
    reversible_probe:
      method: POST
      path: /vector/diagnostic-probe
      auth: bearer
      response: { operation: operation }
`);

const bindings: RuntimeBindings = {
  dwh: {
    transport: "postgres_direct",
    missing: [],
    values: {
      THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.example.test",
      THT_WS_PSD_CLINICAL_DWH_PORT: "5432",
      THT_WS_PSD_CLINICAL_DWH_USER: "reader",
      THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: "/run/secrets/dwh-password",
      THT_WS_PSD_CLINICAL_DWH_TLS_CA_FILE: "/run/secrets/dwh-ca",
    },
  },
  vector: {
    transport: "pgvector_direct",
    missing: [],
    values: {
      THT_WS_PSD_CLINICAL_VECTOR_HOST: "vector.example.test",
      THT_WS_PSD_CLINICAL_VECTOR_PORT: "5432",
      THT_WS_PSD_CLINICAL_VECTOR_USER: "vector-reader",
      THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_FILE: "/run/secrets/vector-password",
      THT_WS_PSD_CLINICAL_VECTOR_TLS_CA_FILE: "/run/secrets/vector-ca",
    },
  },
  vectorWriter: { transport: "rest_api", missing: [], values: {} },
  embedding: {
    transport: "rest_api",
    missing: [],
    values: {
      THT_WS_PSD_CLINICAL_EMBEDDING_BASE_URL: "https://embedding.example.test",
      THT_WS_PSD_CLINICAL_EMBEDDING_API_KEY_FILE: "/run/secrets/embedding-key",
      THT_WS_PSD_CLINICAL_EMBEDDING_TLS_CA_FILE: "/run/secrets/embedding-ca",
    },
  },
};

const writerBindings: RuntimeBindings = {
  ...bindings,
  vector: {
    transport: "rest_api",
    missing: [],
    values: {
      THT_WS_PSD_CLINICAL_VECTOR_BASE_URL: "https://vector.example.test",
      THT_WS_PSD_CLINICAL_VECTOR_API_KEY_FILE: "/run/secrets/vector-reader-key",
    },
  },
  vectorWriter: {
    transport: "rest_api",
    missing: [],
    values: { THT_WS_PSD_CLINICAL_VECTOR_WRITER_API_KEY_FILE: "/run/secrets/vector-writer-key" },
  },
};

function successfulAdapters(overrides: Partial<DiagnosticAdapters> = {}): DiagnosticAdapters {
  return {
    probeConnector: vi.fn(async (request) => ({
      resolved: true,
      tlsVerified: true,
      authenticated: true,
      resource: request.resource,
    })),
    withSshTunnel: vi.fn(async (_request, probe) => probe({ host: "127.0.0.1", port: 45678 })),
    inspectVector: vi.fn(async () => ({
      collection: "clinical_documents",
      dimensions: 768,
      distance: "cosine",
    })),
    probeEmbedding: vi.fn(async () => ({ available: true, dimensions: 768 })),
    writeDiagnosticRecord: vi.fn(async () => undefined),
    removeDiagnosticRecord: vi.fn(async () => undefined),
    ...overrides,
  };
}

function diagnose(adapters = successfulAdapters()) {
  return createWorkspaceDiagnoser(adapters, { timeoutMs: 5000 });
}

test("reports the missing vector collection dimensions as semantic-index incompatibility", async () => {
  const result = await diagnose(successfulAdapters({
    inspectVector: vi.fn(async () => ({
      collection: "clinical_documents",
      dimensions: undefined,
      distance: "cosine",
    })),
  }))(workspace, bindings, { writeProbe: false });

  expect(result.diagnostics).toContainEqual(expect.objectContaining({
    code: "semantic_index_incompatible",
  }));
  expect(result.activatable).toBe(false);
});

test("refuses an SSH tunnel when known-hosts is missing", async () => {
  const sshBindingsWithoutKnownHosts: RuntimeBindings = {
    ...bindings,
    dwh: {
      transport: "ssh_tunnel",
      missing: ["THT_WS_PSD_CLINICAL_DWH_SSH_KNOWN_HOSTS_FILE"],
      values: {
        THT_WS_PSD_CLINICAL_DWH_USER: "reader",
        THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: "/run/secrets/dwh-password",
        THT_WS_PSD_CLINICAL_DWH_SSH_HOST: "bastion.example.test",
        THT_WS_PSD_CLINICAL_DWH_SSH_PORT: "22",
        THT_WS_PSD_CLINICAL_DWH_SSH_USER: "tunnel",
        THT_WS_PSD_CLINICAL_DWH_SSH_PRIVATE_KEY_FILE: "/run/secrets/ssh-key",
        THT_WS_PSD_CLINICAL_DWH_SSH_TARGET_HOST: "dwh.internal",
        THT_WS_PSD_CLINICAL_DWH_SSH_TARGET_PORT: "5432",
      },
    },
  };
  const adapters = successfulAdapters();

  const result = await diagnose(adapters)(workspace, sshBindingsWithoutKnownHosts, { writeProbe: false });

  expect(result.activatable).toBe(false);
  expect(result.diagnostics[0]).toMatchObject({
    code: "binding_missing",
    field: expect.stringContaining("SSH_KNOWN_HOSTS_FILE"),
  });
  expect(adapters.withSshTunnel).not.toHaveBeenCalled();
});

test("checks direct and REST resolution, TLS, authentication, and resource metadata without exposing failures", async () => {
  const adapters = successfulAdapters({
    probeConnector: vi.fn(async (request) => ({
      resolved: true,
      tlsVerified: true,
      authenticated: true,
      resource: request.role === "dwh"
        ? { database: "warehouse", schema: "wrong_schema" }
        : request.resource,
    })),
  });
  const restBindings: RuntimeBindings = {
    ...bindings,
    dwh: {
      transport: "rest_api",
      missing: [],
      values: {
        THT_WS_PSD_CLINICAL_DWH_BASE_URL: "https://dwh.example.test",
        THT_WS_PSD_CLINICAL_DWH_API_KEY_FILE: "/run/secrets/dwh-api-key",
        THT_WS_PSD_CLINICAL_DWH_TLS_CA_FILE: "/run/secrets/dwh-ca",
      },
    },
  };

  const result = await diagnose(adapters)(workspace, restBindings, { writeProbe: false });

  expect(adapters.probeConnector).toHaveBeenCalledWith(expect.objectContaining({
    transport: "rest_api",
    timeoutMs: 5000,
    tlsCaFile: "/run/secrets/dwh-ca",
    credentialFile: "/run/secrets/dwh-api-key",
    resource: { database: "warehouse", schema: "datawarehouse" },
  }));
  expect(result).toMatchObject({ activatable: false });
  expect(JSON.stringify(result)).not.toContain("wrong_schema");
  expect(JSON.stringify(result)).not.toContain("/run/secrets/dwh-api-key");
});

test("uses a loopback-only SSH tunnel for the bounded connector probe", async () => {
  const adapters = successfulAdapters();
  const sshBindings: RuntimeBindings = {
    ...bindings,
    dwh: {
      transport: "ssh_tunnel",
      missing: [],
      values: {
        THT_WS_PSD_CLINICAL_DWH_USER: "reader",
        THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: "/run/secrets/dwh-password",
        THT_WS_PSD_CLINICAL_DWH_TLS_CA_FILE: "/run/secrets/dwh-ca",
        THT_WS_PSD_CLINICAL_DWH_SSH_HOST: "bastion.example.test",
        THT_WS_PSD_CLINICAL_DWH_SSH_PORT: "22",
        THT_WS_PSD_CLINICAL_DWH_SSH_USER: "tunnel",
        THT_WS_PSD_CLINICAL_DWH_SSH_PRIVATE_KEY_FILE: "/run/secrets/ssh-key",
        THT_WS_PSD_CLINICAL_DWH_SSH_KNOWN_HOSTS_FILE: "/run/secrets/known-hosts",
        THT_WS_PSD_CLINICAL_DWH_SSH_TARGET_HOST: "dwh.internal",
        THT_WS_PSD_CLINICAL_DWH_SSH_TARGET_PORT: "5432",
      },
    },
  };

  const result = await diagnose(adapters)(workspace, sshBindings, { writeProbe: false });

  expect(result.activatable).toBe(true);
  expect(adapters.withSshTunnel).toHaveBeenCalledWith(expect.objectContaining({
    localHost: "127.0.0.1",
    localPort: 0,
    knownHostsFile: "/run/secrets/known-hosts",
    timeoutMs: 5000,
  }), expect.any(Function));
  expect(adapters.probeConnector).toHaveBeenCalledWith(expect.objectContaining({
    host: "127.0.0.1",
    port: 45678,
  }));
});

test("passes the declared vector database and schema to direct diagnostics", async () => {
  const adapters = successfulAdapters();

  await diagnose(adapters)(workspace, bindings, { writeProbe: false });

  expect(adapters.probeConnector).toHaveBeenCalledWith(expect.objectContaining({
    role: "vector",
    resource: { database: "postgres", schema: "vectors", collection: "clinical_documents" },
  }));
});

test("uses strict known-host SSH arguments and always closes the temporary tunnel", async () => {
  const directory = await mkdtemp(join(tmpdir(), "thothii-diagnostic-"));
  const privateKeyFile = join(directory, "ssh-key");
  await writeFile(privateKeyFile, "test-key\n", { mode: 0o600 });
  const close = vi.fn(async () => undefined);
  const start = vi.fn(async () => ({ tunnel: { host: "127.0.0.1" as const, port: 45432 }, close }));

  try {
    const adapter = createConcreteDiagnosticAdapters({ sshProcess: { start } });
    await adapter.withSshTunnel({
      sshHost: "bastion.example.test", sshPort: 22, sshUser: "tunnel", privateKeyFile,
      knownHostsFile: "/run/secrets/known-hosts", targetHost: "vector.internal", targetPort: 5432,
      localHost: "127.0.0.1", localPort: 0, timeoutMs: 5000, signal: new AbortController().signal,
    }, async () => undefined);

    expect(start).toHaveBeenCalledWith(expect.any(Object), expect.arrayContaining([
      "StrictHostKeyChecking=yes", "UserKnownHostsFile=/run/secrets/known-hosts", "-i", privateKeyFile,
    ]));
    expect(close).toHaveBeenCalledOnce();
  } finally {
    await rm(directory, { recursive: true, force: true });
  }
});

test("provides a default bounded SSH factory through injected spawn and loopback allocation", async () => {
  const directory = await mkdtemp(join(tmpdir(), "thothii-diagnostic-"));
  const privateKeyFile = join(directory, "ssh-key");
  await writeFile(privateKeyFile, "test-key\n", { mode: 0o600 });
  const kill = vi.fn(() => true);
  const sshSpawn = vi.fn(() => ({ kill }));
  try {
    const adapter = createConcreteDiagnosticAdapters({ sshSpawn, reserveLoopbackPort: async () => 45432, sshForwardConfirmed: async () => undefined } as any);
    await adapter.withSshTunnel({ sshHost: "bastion.example.test", sshPort: 22, sshUser: "tunnel", privateKeyFile, knownHostsFile: "/run/secrets/known-hosts", targetHost: "dwh.internal", targetPort: 5432, localHost: "127.0.0.1", localPort: 0, timeoutMs: 5000, signal: new AbortController().signal }, async () => undefined);
    expect(sshSpawn).toHaveBeenCalledWith(expect.arrayContaining([
      "StrictHostKeyChecking=yes", "UserKnownHostsFile=/run/secrets/known-hosts", "-L", "127.0.0.1:45432:dwh.internal:5432",
    ]));
    expect(kill).toHaveBeenCalledWith("SIGTERM");
  } finally {
    await rm(directory, { recursive: true, force: true });
  }
});

test("waits for SSH readiness before probing and includes ExitOnForwardFailure", async () => {
  const directory = await mkdtemp(join(tmpdir(), "thothii-diagnostic-"));
  const privateKeyFile = join(directory, "ssh-key");
  await writeFile(privateKeyFile, "test-key\n", { mode: 0o600 });
  let releaseReady: (() => void) | undefined;
  const ready = new Promise<void>((resolve) => { releaseReady = resolve; });
  const probe = vi.fn(async () => undefined);
  const sshSpawn = vi.fn(() => ({ kill: vi.fn(() => true) }));
  try {
    const adapter = createConcreteDiagnosticAdapters({ sshSpawn, reserveLoopbackPort: async () => 45432, sshForwardConfirmed: async () => await ready } as any);
    const running = adapter.withSshTunnel({ sshHost: "bastion.example.test", sshPort: 22, sshUser: "tunnel", privateKeyFile, knownHostsFile: "/run/secrets/known-hosts", targetHost: "dwh.internal", targetPort: 5432, localHost: "127.0.0.1", localPort: 0, timeoutMs: 5000, signal: new AbortController().signal }, probe);
    await Promise.resolve();
    expect(probe).not.toHaveBeenCalled();
    releaseReady?.();
    await running;
    expect(sshSpawn).toHaveBeenCalledWith(expect.arrayContaining(["ExitOnForwardFailure=yes"]));
    expect(probe).toHaveBeenCalledOnce();
  } finally {
    await rm(directory, { recursive: true, force: true });
  }
});

test("rejects unrelated listener readiness until the SSH child confirms its own forward", async () => {
  const directory = await mkdtemp(join(tmpdir(), "thothii-diagnostic-"));
  const privateKeyFile = join(directory, "ssh-key");
  await writeFile(privateKeyFile, "test-key\n", { mode: 0o600 });
  const child = Object.assign(new EventEmitter(), { kill: vi.fn(() => true) });
  const probe = vi.fn(async () => undefined);
  try {
    const adapter = createConcreteDiagnosticAdapters({ sshSpawn: vi.fn(() => child), reserveLoopbackPort: async () => 45432, waitForSshReady: async () => undefined } as any);
    await expect(adapter.withSshTunnel({ sshHost: "bastion.example.test", sshPort: 22, sshUser: "tunnel", privateKeyFile, knownHostsFile: "/run/secrets/known-hosts", targetHost: "dwh.internal", targetPort: 5432, localHost: "127.0.0.1", localPort: 0, timeoutMs: 20, signal: new AbortController().signal }, probe)).rejects.toThrow("SSH tunnel readiness failed");
    expect(probe).not.toHaveBeenCalled();
  } finally {
    await rm(directory, { recursive: true, force: true });
  }
});

test("permits the probe only after this SSH child confirms its forwarded port", async () => {
  const directory = await mkdtemp(join(tmpdir(), "thothii-diagnostic-"));
  const privateKeyFile = join(directory, "ssh-key");
  await writeFile(privateKeyFile, "test-key\n", { mode: 0o600 });
  const stderr = new EventEmitter();
  const child = Object.assign(new EventEmitter(), { kill: vi.fn(() => true), stderr });
  child.kill.mockImplementation(() => { child.emit("exit", 0); return true; });
  const probe = vi.fn(async () => undefined);
  try {
    const adapter = createConcreteDiagnosticAdapters({ sshSpawn: vi.fn(() => child), reserveLoopbackPort: async () => 45432 } as any);
    const running = adapter.withSshTunnel({ sshHost: "bastion.example.test", sshPort: 22, sshUser: "tunnel", privateKeyFile, knownHostsFile: "/run/secrets/known-hosts", targetHost: "dwh.internal", targetPort: 5432, localHost: "127.0.0.1", localPort: 0, timeoutMs: 500, signal: new AbortController().signal }, probe);
    for (let attempt = 0; attempt < 20 && stderr.listenerCount("data") === 0; attempt += 1) {
      await new Promise((resolve) => setTimeout(resolve, 1));
    }
    expect(stderr.listenerCount("data")).toBeGreaterThan(0);
    expect(probe).not.toHaveBeenCalled();
    stderr.emit("data", "debug1: Local forwarding listening on 127.0.0.1 port 45432.\n");
    await running;
    expect(probe).toHaveBeenCalledOnce();
  } finally {
    await rm(directory, { recursive: true, force: true });
  }
});

test("accepts an SSH forward confirmation split across stderr chunks", async () => {
  const directory = await mkdtemp(join(tmpdir(), "thothii-diagnostic-"));
  const privateKeyFile = join(directory, "ssh-key");
  await writeFile(privateKeyFile, "test-key\n", { mode: 0o600 });
  const stderr = new EventEmitter();
  const child = Object.assign(new EventEmitter(), { kill: vi.fn(() => true), stderr });
  child.kill.mockImplementation(() => { child.emit("exit", 0); return true; });
  const probe = vi.fn(async () => undefined);
  try {
    const adapter = createConcreteDiagnosticAdapters({ sshSpawn: vi.fn(() => child), reserveLoopbackPort: async () => 45432 } as any);
    const running = adapter.withSshTunnel({ sshHost: "bastion.example.test", sshPort: 22, sshUser: "tunnel", privateKeyFile, knownHostsFile: "/run/secrets/known-hosts", targetHost: "dwh.internal", targetPort: 5432, localHost: "127.0.0.1", localPort: 0, timeoutMs: 500, signal: new AbortController().signal }, probe);
    for (let attempt = 0; attempt < 20 && stderr.listenerCount("data") === 0; attempt += 1) await new Promise((resolve) => setTimeout(resolve, 1));
    stderr.emit("data", "debug1: Local forwarding listening on 127.0.0.1 ");
    stderr.emit("data", "port 45432.\n");
    await running;
    expect(probe).toHaveBeenCalledOnce();
  } finally {
    await rm(directory, { recursive: true, force: true });
  }
});

test("requires a matching embedding model vector and removes its unique write probe", async () => {
  const adapters = successfulAdapters();

  const result = await diagnose(adapters)(writerWorkspace, writerBindings, { writeProbe: true });

  expect(result.activatable).toBe(true);
  expect(adapters.probeEmbedding).toHaveBeenCalledWith(expect.objectContaining({
    model: "nomic-embed-text-v2-moe",
    timeoutMs: 5000,
  }));
  expect(adapters.writeDiagnosticRecord).toHaveBeenCalledWith(expect.objectContaining({
    collection: "clinical_documents",
    id: expect.stringMatching(/^diagnostic:/),
    dimensions: 768,
  }));
  expect(adapters.removeDiagnosticRecord).toHaveBeenCalledWith(expect.objectContaining({
    collection: "clinical_documents",
    id: expect.stringMatching(/^diagnostic:/),
  }));
});

test("passes the resolver's distinct vector-writer binding to the diagnoser", async () => {
  const directory = await mkdtemp(join(tmpdir(), "thothii-diagnostic-bindings-"));
  const readerKey = join(directory, "reader-key");
  const writerKey = join(directory, "writer-key");
  const dwhKey = join(directory, "dwh-key");
  await Promise.all([
    writeFile(readerKey, "reader\n", { mode: 0o600 }),
    writeFile(writerKey, "writer\n", { mode: 0o600 }),
    writeFile(dwhKey, "dwh\n", { mode: 0o600 }),
  ]);
  const adapters = successfulAdapters();
  try {
    const resolved = resolveRuntimeBindings(writerWorkspace, {
      THT_WS_PSD_CLINICAL_DWH_TRANSPORT: "rest_api",
      THT_WS_PSD_CLINICAL_DWH_BASE_URL: "https://dwh.example.test",
      THT_WS_PSD_CLINICAL_DWH_API_KEY_FILE: dwhKey,
      THT_WS_PSD_CLINICAL_VECTOR_TRANSPORT: "rest_api",
      THT_WS_PSD_CLINICAL_VECTOR_BASE_URL: "https://vector.example.test",
      THT_WS_PSD_CLINICAL_VECTOR_API_KEY_FILE: readerKey,
      THT_WS_PSD_CLINICAL_VECTOR_WRITER_API_KEY_FILE: writerKey,
      THT_WS_PSD_CLINICAL_EMBEDDING_BASE_URL: "https://embedding.example.test",
    }, [directory]);

    await diagnose(adapters)(writerWorkspace, resolved, { writeProbe: true });

    expect(resolved.vectorWriter.values).toEqual({
      THT_WS_PSD_CLINICAL_VECTOR_WRITER_API_KEY_FILE: writerKey,
    });
    expect(adapters.writeDiagnosticRecord).toHaveBeenCalledWith(expect.objectContaining({ credentialFile: writerKey }));
  } finally {
    await rm(directory, { recursive: true, force: true });
  }
});

test("keeps a reader-only workspace activatable without a vector write probe", async () => {
  const adapters = successfulAdapters();

  const result = await diagnose(adapters)(workspace, bindings, { writeProbe: true });

  expect(result.activatable).toBe(true);
  expect(adapters.writeDiagnosticRecord).not.toHaveBeenCalled();
  expect(adapters.removeDiagnosticRecord).not.toHaveBeenCalled();
});

test("does not substitute the reader credential for a declared vector writer", async () => {
  const adapters = successfulAdapters();

  const result = await diagnose(adapters)(writerWorkspace, bindings, { writeProbe: true });

  expect(result.activatable).toBe(true);
  expect(adapters.writeDiagnosticRecord).not.toHaveBeenCalled();
});

test("rejects a writer credential that aliases the reader credential", async () => {
  const directory = await mkdtemp(join(tmpdir(), "thothii-diagnostic-"));
  const readerKey = join(directory, "reader-key");
  const writerAlias = join(directory, "writer-key");
  await writeFile(readerKey, "same-secret\n", { mode: 0o600 });
  await (await import("node:fs/promises")).symlink(readerKey, writerAlias);
  const adapters = successfulAdapters();
  const aliasedBindings: RuntimeBindings = {
    ...writerBindings,
    vector: { ...writerBindings.vector, values: { ...writerBindings.vector.values, THT_WS_PSD_CLINICAL_VECTOR_API_KEY_FILE: readerKey } },
    vectorWriter: { ...writerBindings.vectorWriter, values: { THT_WS_PSD_CLINICAL_VECTOR_WRITER_API_KEY_FILE: writerAlias } },
  };

  try {
    const result = await diagnose(adapters)(writerWorkspace, aliasedBindings, { writeProbe: true });
    expect(result.activatable).toBe(false);
    expect(adapters.writeDiagnosticRecord).not.toHaveBeenCalled();
  } finally {
    await rm(directory, { recursive: true, force: true });
  }
});

test("uses the declared POST DWH ping endpoint without exposing its local credential", async () => {
  const directory = await mkdtemp(join(tmpdir(), "thothii-diagnostic-"));
  const credentialFile = join(directory, "dwh-api-key");
  await writeFile(credentialFile, "local-secret\n", { mode: 0o600 });
  const fetchSpy = vi.fn(async () => new Response(JSON.stringify({ database: "warehouse", schema: "datawarehouse" }), {
    status: 200,
    headers: { "content-type": "application/json" },
  }));
  vi.stubGlobal("fetch", fetchSpy);

  try {
    const result = await createConcreteDiagnosticAdapters().probeConnector({
      role: "dwh",
      transport: "rest_api",
      baseUrl: "https://dwh.example.test",
      credentialFile,
      resource: { database: "warehouse", schema: "datawarehouse" },
      diagnostic: {
        method: "POST", path: "/rpc/ping", auth: "bearer",
        response: { database: "database", schema: "schema" },
      },
      timeoutMs: 5000,
      signal: new AbortController().signal,
    });

    expect(fetchSpy).toHaveBeenCalledWith("https://dwh.example.test/rpc/ping", expect.objectContaining({
      method: "POST",
      redirect: "error",
    }));
    expect(result).toMatchObject({ authenticated: true, resource: { database: "warehouse", schema: "datawarehouse" } });
    expect(JSON.stringify(result)).not.toContain("local-secret");
  } finally {
    vi.unstubAllGlobals();
    await rm(directory, { recursive: true, force: true });
  }
});

test("requires an authenticated TLS database query before direct diagnostics succeed", async () => {
  const directory = await mkdtemp(join(tmpdir(), "thothii-diagnostic-"));
  const passwordFile = join(directory, "password");
  const caFile = join(directory, "ca.pem");
  await Promise.all([writeFile(passwordFile, "password\n", { mode: 0o600 }), writeFile(caFile, "test-ca\n")]);
  const query = vi.fn(async () => ({ rows: [{ database: "warehouse", schema: "datawarehouse" }] }));
  const end = vi.fn(async () => undefined);
  const connect = vi.fn(async () => ({ query, end }));

  try {
    const result = await createConcreteDiagnosticAdapters({ databaseClient: { connect } } as any).probeConnector({
      role: "dwh", transport: "postgres_direct", host: "127.0.0.1", port: 5432, user: "reader",
      credentialFile: passwordFile, tlsCaFile: caFile,
      resource: { database: "warehouse", schema: "datawarehouse" }, timeoutMs: 5000,
      signal: new AbortController().signal,
    });

    expect(connect).toHaveBeenCalledWith(expect.objectContaining({ database: "warehouse", tlsCaFile: caFile }));
    expect(query).toHaveBeenCalledWith(expect.stringContaining("current_database"), []);
    expect(end).toHaveBeenCalledOnce();
    expect(result).toMatchObject({ authenticated: true, tlsVerified: true });
  } finally {
    await rm(directory, { recursive: true, force: true });
  }
});

test("uses system trust for direct and SSH PostgreSQL diagnostics when no CA binding exists", async () => {
  const directory = await mkdtemp(join(tmpdir(), "thothii-diagnostic-"));
  const passwordFile = join(directory, "password");
  await writeFile(passwordFile, "password\n", { mode: 0o600 });
  const query = vi.fn(async () => ({ rows: [{ database: "warehouse", schema: "datawarehouse" }] }));
  const connect = vi.fn(async () => ({ query, end: vi.fn(async () => undefined) }));
  const adapter = createConcreteDiagnosticAdapters({ databaseClient: { connect } } as any);
  try {
    for (const transport of ["postgres_direct", "ssh_tunnel"] as const) {
      await expect(adapter.probeConnector({
        role: "dwh", transport, host: "127.0.0.1", port: 5432, user: "reader", credentialFile: passwordFile,
        resource: { database: "warehouse", schema: "datawarehouse" }, timeoutMs: 5000,
        signal: new AbortController().signal,
      })).resolves.toMatchObject({ tlsVerified: true, authenticated: true });
    }
    expect(connect).toHaveBeenCalledTimes(2);
    expect(connect).toHaveBeenNthCalledWith(1, expect.objectContaining({ tlsCaFile: undefined }));
    expect(connect).toHaveBeenNthCalledWith(2, expect.objectContaining({ tlsCaFile: undefined }));
  } finally {
    await rm(directory, { recursive: true, force: true });
  }
});

test("selects the vector index containing the declared vector column for direct metadata", async () => {
  const directory = await mkdtemp(join(tmpdir(), "thothii-diagnostic-"));
  const passwordFile = join(directory, "password");
  const caFile = join(directory, "ca.pem");
  await Promise.all([writeFile(passwordFile, "password\n"), writeFile(caFile, "test-ca\n")]);
  const query = vi.fn(async () => ({ rows: [{ dimensions: 768, distance: "cosine" }] }));
  const connect = vi.fn(async () => ({ query, end: vi.fn(async () => undefined) }));
  try {
    const result = await createConcreteDiagnosticAdapters({ databaseClient: { connect } } as any).inspectVector({
      transport: "pgvector_direct", host: "127.0.0.1", port: 5432, user: "reader", credentialFile: passwordFile, tlsCaFile: caFile,
      resource: { database: "postgres", schema: "vectors" }, collection: "clinical_documents", timeoutMs: 5000, signal: new AbortController().signal,
    });
    expect(query).toHaveBeenCalledWith(expect.stringContaining("a.attnum = ANY(i.indkey)"), ["vectors", "clinical_documents"]);
    expect(result).toMatchObject({ dimensions: 768, distance: "cosine" });
  } finally {
    await rm(directory, { recursive: true, force: true });
  }
});

test("honors a declared unauthenticated REST diagnostic without reading a credential", async () => {
  const fetchSpy = vi.fn(async () => new Response(JSON.stringify({ database: "warehouse", schema: "datawarehouse" }), {
    status: 200, headers: { "content-type": "application/json" },
  }));
  vi.stubGlobal("fetch", fetchSpy);
  try {
    await expect(createConcreteDiagnosticAdapters().probeConnector({
      role: "dwh", transport: "rest_api", baseUrl: "https://dwh.example.test",
      resource: { database: "warehouse", schema: "datawarehouse" }, timeoutMs: 5000,
      signal: new AbortController().signal,
      diagnostic: { method: "POST", path: "/rpc/ping", auth: "none", response: { database: "database", schema: "schema" } } as any,
    })).resolves.toMatchObject({ authenticated: true });
    expect(fetchSpy.mock.calls[0]?.[1]).not.toMatchObject({ headers: expect.objectContaining({ authorization: expect.anything() }) });
  } finally {
    vi.unstubAllGlobals();
  }
});

test("applies declared auth modes and rejects private CA files across vector REST paths", async () => {
  const directory = await mkdtemp(join(tmpdir(), "thothii-diagnostic-"));
  const keyFile = join(directory, "api-key");
  const caFile = join(directory, "ca.pem");
  await Promise.all([writeFile(keyFile, "writer-key\n", { mode: 0o600 }), writeFile(caFile, "private-ca\n")]);
  const fetchSpy = vi.fn(async () => new Response(JSON.stringify({ collection: "clinical_documents", dimensions: 768, distance: "cosine", model: "embed", operation: "create" }), { status: 200, headers: { "content-type": "application/json" } }));
  vi.stubGlobal("fetch", fetchSpy);
  const adapter = createConcreteDiagnosticAdapters();
  const signal = new AbortController().signal;
  try {
    await adapter.inspectVector({ transport: "rest_api", baseUrl: "https://vector.example.test", collection: "clinical_documents", timeoutMs: 1, signal, diagnostic: { method: "GET", path: "/metadata", auth: "none", response: { collection: "collection", dimensions: "dimensions", distance: "distance" } } });
    await adapter.probeEmbedding({ baseUrl: "https://embed.example.test", model: "embed", timeoutMs: 1, signal, credentialFile: keyFile, diagnostic: { method: "POST", path: "/embed", auth: "x-api-key", response: { model: "model", dimensions: "dimensions" } } });
    expect(fetchSpy.mock.calls[0]?.[1]).toMatchObject({ headers: {} });
    expect(fetchSpy.mock.calls[1]?.[1]).toMatchObject({ headers: { "x-api-key": "writer-key" } });
    await expect(adapter.inspectVector({ transport: "rest_api", baseUrl: "https://vector.example.test", credentialFile: keyFile, tlsCaFile: caFile, collection: "clinical_documents", timeoutMs: 1, signal, diagnostic: { method: "GET", path: "/metadata", auth: "bearer", response: { collection: "collection", dimensions: "dimensions", distance: "distance" } } })).rejects.toThrow("vector metadata adapter is unavailable");
    await expect(adapter.probeEmbedding({ baseUrl: "https://embed.example.test", credentialFile: keyFile, tlsCaFile: caFile, model: "embed", timeoutMs: 1, signal, diagnostic: { method: "POST", path: "/embed", auth: "bearer", response: { model: "model", dimensions: "dimensions" } } })).rejects.toThrow("embedding probe failed");
    await expect(adapter.removeDiagnosticRecord({ baseUrl: "https://vector.example.test", credentialFile: keyFile, tlsCaFile: caFile, collection: "clinical_documents", id: "diagnostic:test", dimensions: 768, timeoutMs: 1, signal, diagnostic: { method: "POST", path: "/probe", auth: "bearer", response: { operation: "operation" } } })).rejects.toThrow("vector write adapter is unavailable");
  } finally {
    vi.unstubAllGlobals();
    await rm(directory, { recursive: true, force: true });
  }
});

test("validates that the reversible writer response confirms each requested operation", async () => {
  const directory = await mkdtemp(join(tmpdir(), "thothii-diagnostic-"));
  const keyFile = join(directory, "writer-key");
  await writeFile(keyFile, "writer\n", { mode: 0o600 });
  const fetchSpy = vi.fn(async (_url: string, init: RequestInit) => new Response(JSON.stringify({
    operation: JSON.parse(String(init.body)).operation === "create" ? "create" : "not-removed",
  }), { status: 200, headers: { "content-type": "application/json" } }));
  vi.stubGlobal("fetch", fetchSpy);
  const request = {
    baseUrl: "https://vector.example.test", credentialFile: keyFile, collection: "clinical_documents",
    dimensions: 768, id: "diagnostic:test", timeoutMs: 5000, signal: new AbortController().signal,
    diagnostic: { method: "POST" as const, path: "/probe", auth: "bearer" as const, response: { operation: "operation" } },
  };
  try {
    const adapter = createConcreteDiagnosticAdapters();
    await expect(adapter.writeDiagnosticRecord(request)).resolves.toBeUndefined();
    await expect(adapter.removeDiagnosticRecord(request)).rejects.toThrow("vector write adapter is unavailable");
  } finally {
    vi.unstubAllGlobals();
    await rm(directory, { recursive: true, force: true });
  }
});

test("constructs the production diagnoser with the configured timeout and injected adapters", async () => {
  const adapters = successfulAdapters();

  const result = await createProductionWorkspaceDiagnoser(1234, adapters)(workspace, bindings, {
    writeProbe: false,
  });

  expect(result.activatable).toBe(true);
  expect(adapters.probeConnector).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 1234 }));
  expect(adapters.probeEmbedding).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 1234 }));
});

test("retries bounded cleanup after a write-probe removal times out", async () => {
  const adapters = successfulAdapters({
    removeDiagnosticRecord: vi.fn(() => new Promise<void>(() => undefined)),
  });
  const diagnoseWithShortTimeout = createWorkspaceDiagnoser(adapters, { timeoutMs: 10 });

  const startedAt = Date.now();
  const result = await diagnoseWithShortTimeout(writerWorkspace, writerBindings, { writeProbe: true });

  expect(Date.now() - startedAt).toBeLessThan(250);
  expect(adapters.writeDiagnosticRecord).toHaveBeenCalledTimes(1);
  expect(adapters.removeDiagnosticRecord).toHaveBeenCalledTimes(2);
  expect(result).toMatchObject({ activatable: false });
  expect(JSON.stringify(result)).not.toContain("timeout");
});

test("attempts bounded cleanup when a timed-out write may already have created the record", async () => {
  const adapters = successfulAdapters({
    writeDiagnosticRecord: vi.fn(() => new Promise<void>(() => undefined)),
  });
  const diagnoseWithShortTimeout = createWorkspaceDiagnoser(adapters, { timeoutMs: 10 });

  const result = await diagnoseWithShortTimeout(writerWorkspace, writerBindings, { writeProbe: true });

  expect(adapters.writeDiagnosticRecord).toHaveBeenCalledOnce();
  expect(adapters.removeDiagnosticRecord).toHaveBeenCalledOnce();
  expect(result.activatable).toBe(false);
});