Files
ThothII/backend/test/workspaces-diagnostics.test.ts
T

559 lines
23 KiB
TypeScript

import { afterEach, expect, test, vi } from "vitest";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
createConcreteDiagnosticAdapters,
createProductionWorkspaceDiagnoser,
createWorkspaceDiagnoser,
type DiagnosticAdapters,
} from "../src/workspaces/diagnostics.js";
import type { RuntimeBindings } from "../src/workspaces/runtime-renderer.js";
import { parseWorkspaceYaml, resolveDiagnosticUrl } from "../src/workspaces/schema.js";
const workspace = parseWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: Policlinico San Donato
language: it
dwh:
engine: postgres
database: warehouse
schema: datawarehouse
timeout_ms: 8000
supported_transports: [postgres_direct, rest_api, ssh_tunnel]
`);
const bindings: RuntimeBindings = {
dwh: {
transport: "postgres_direct",
missing: [],
values: {
THT_WS_PSD_CLINICAL_DWH_HOST: "dwh.example.test",
THT_WS_PSD_CLINICAL_DWH_PORT: "5432",
THT_WS_PSD_CLINICAL_DWH_USER: "reader",
THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: "/run/secrets/dwh-password",
THT_WS_PSD_CLINICAL_DWH_TLS_CA_FILE: "/run/secrets/dwh-ca",
},
},
evidence: { missing: [], values: {} },
};
function successfulAdapters(overrides: Partial<DiagnosticAdapters> = {}): DiagnosticAdapters {
return {
probeConnector: vi.fn(async (request) => ({
resolved: true,
tlsVerified: true,
authenticated: true,
resource: request.resource,
})),
inspectQdrant: vi.fn(async (request) => ({
collection: request.collection,
dimensions: 1024,
distance: "cosine",
})),
probeEmbedding: vi.fn(async () => ({ available: true, dimensions: 1024 })),
...overrides,
};
}
function diagnose(adapters = successfulAdapters()) {
return createWorkspaceDiagnoser(adapters, { timeoutMs: 5_000 });
}
afterEach(() => {
vi.unstubAllGlobals();
vi.restoreAllMocks();
});
test("diagnoses workspace-v4 DWH plus installation-derived Qdrant and Ollama", async () => {
const adapters = successfulAdapters();
const result = await diagnose(adapters)(workspace, bindings, { writeProbe: false });
expect(result).toEqual({
activatable: true,
diagnostics: [{
level: "info", code: "binding_ok",
message: "Installation bindings and diagnostics succeeded.",
}],
});
expect(adapters.probeConnector).toHaveBeenCalledWith(expect.objectContaining({
role: "dwh", transport: "postgres_direct",
resource: { database: "warehouse", schema: "datawarehouse" },
}));
expect(adapters.inspectQdrant).toHaveBeenCalledWith(expect.objectContaining({
baseUrl: "http://qdrant:6333", collection: "psd-clinical",
}));
expect(adapters.probeEmbedding).toHaveBeenCalledWith(expect.objectContaining({
baseUrl: "http://embedding:11434", model: "qwen3-embedding:0.6b",
}));
});
test("reports incompatible internal Qdrant or Ollama metadata", async () => {
const vector = await diagnose(successfulAdapters({
inspectQdrant: vi.fn(async () => ({
collection: "psd-clinical", dimensions: 768, distance: "cosine",
})),
}))(workspace, bindings, { writeProbe: false });
expect(vector.activatable).toBe(false);
expect(vector.diagnostics).toContainEqual(expect.objectContaining({
code: "semantic_index_incompatible",
}));
const embedding = await diagnose(successfulAdapters({
probeEmbedding: vi.fn(async () => ({ available: true, dimensions: 768 })),
}))(workspace, bindings, { writeProbe: false });
expect(embedding.activatable).toBe(false);
expect(embedding.diagnostics).toContainEqual(expect.objectContaining({
code: "semantic_index_incompatible",
}));
});
test("reports only sanitized DWH and Evidence binding names before network diagnostics", async () => {
const adapters = successfulAdapters();
const result = await diagnose(adapters)(workspace, {
dwh: { ...bindings.dwh, missing: ["THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE"] },
evidence: {
values: {}, missing: ["THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE"],
},
}, { writeProbe: false });
expect(result.activatable).toBe(false);
expect(result.diagnostics.map(({ code }) => code)).toEqual(["binding_missing", "binding_missing"]);
expect(result.diagnostics[1]).toMatchObject({
variable: "THT_WS_PSD_CLINICAL_EVIDENCE_SIGNED_URLS_FILE",
});
expect(adapters.probeConnector).not.toHaveBeenCalled();
expect(adapters.inspectQdrant).not.toHaveBeenCalled();
});
test("keeps workspace-v4 DWH SSH diagnostic-only and runtime-inactive", async () => {
const adapters = successfulAdapters();
const result = await diagnose(adapters)(workspace, {
...bindings,
dwh: { transport: "ssh_tunnel", missing: [], values: {} },
}, { writeProbe: false });
expect(result).toEqual({
activatable: false,
diagnostics: [expect.objectContaining({ code: "workspace_not_activatable" })],
});
expect(adapters.probeConnector).not.toHaveBeenCalled();
});
test("uses the workspace-v4 declared DWH REST diagnostic and auth policy", async () => {
const restWorkspace = parseWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: REST workspace
language: en
dwh:
engine: postgres
database: warehouse
schema: datawarehouse
supported_transports: [rest_api]
diagnostics:
dwh_rest:
method: POST
path: /rpc/ping
auth: bearer
response: { database: database, schema: schema }
`);
const adapters = successfulAdapters();
const result = await diagnose(adapters)(restWorkspace, {
dwh: { transport: "rest_api", missing: [], values: {
THT_WS_PSD_CLINICAL_DWH_BASE_URL: "https://dwh.example.test",
THT_WS_PSD_CLINICAL_DWH_API_KEY_FILE: "/run/secrets/dwh-api-key",
} },
evidence: { missing: [], values: {} },
}, { writeProbe: false });
expect(result.activatable).toBe(true);
expect(adapters.probeConnector).toHaveBeenCalledWith(expect.objectContaining({
transport: "rest_api",
baseUrl: "https://dwh.example.test",
credentialFile: "/run/secrets/dwh-api-key",
diagnostic: expect.objectContaining({ path: "/rpc/ping", auth: "bearer" }),
}));
});
test("constructs the production diagnoser with its bounded configured timeout", async () => {
const adapters = successfulAdapters();
await createProductionWorkspaceDiagnoser(1_234, adapters)(workspace, bindings, {
writeProbe: false,
});
expect(adapters.probeConnector).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 1_234 }));
expect(adapters.inspectQdrant).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 1_234 }));
});
test("concrete DWH direct diagnostics authenticate, verify resource identity, and close", async () => {
const root = await mkdtemp(join(tmpdir(), "thoth-diagnostic-"));
const passwordFile = join(root, "password");
await writeFile(passwordFile, "password-value");
const end = vi.fn(async () => undefined);
const query = vi.fn(async () => ({ rows: [{ database: "warehouse", schema: "datawarehouse" }] }));
const connect = vi.fn(async () => ({
query,
end,
}));
try {
const adapter = createConcreteDiagnosticAdapters({ databaseClient: { connect } });
const result = await adapter.probeConnector({
role: "dwh", transport: "postgres_direct", host: "127.0.0.1", port: 5432,
user: "reader", credentialFile: passwordFile,
resource: { database: "warehouse", schema: "datawarehouse" },
timeoutMs: 1_000, signal: new AbortController().signal,
});
expect(result).toMatchObject({ resolved: true, authenticated: true, tlsVerified: true });
expect(connect).toHaveBeenCalledWith(expect.objectContaining({ credentialFile: passwordFile }));
expect(query).toHaveBeenCalledWith(
expect.stringContaining("pg_catalog.has_schema_privilege"),
["datawarehouse"],
);
expect(query.mock.calls[0]?.[0]).toContain("$1");
expect(query.mock.calls[0]?.[0]).not.toContain('"datawarehouse"');
expect(end).toHaveBeenCalledOnce();
} finally {
await rm(root, { recursive: true, force: true });
}
});
test("concrete DWH direct diagnostics disable TLS when no TLS binding is declared", async () => {
const root = await mkdtemp(join(tmpdir(), "thoth-diagnostic-plain-"));
const passwordFile = join(root, "password");
await writeFile(passwordFile, "password-value");
const connect = vi.fn(async () => undefined);
const query = vi.fn(async () => ({ rows: [{ database: "warehouse", schema: "datawarehouse" }] }));
const end = vi.fn(async () => undefined);
const createPostgresClient = vi.fn(() => ({ connect, query, end }));
try {
const adapter = createConcreteDiagnosticAdapters({ createPostgresClient });
await adapter.probeConnector({
role: "dwh", transport: "postgres_direct", host: "127.0.0.1", port: 5432,
user: "reader", credentialFile: passwordFile,
resource: { database: "warehouse", schema: "datawarehouse" },
timeoutMs: 1_000, signal: new AbortController().signal,
});
expect(createPostgresClient).toHaveBeenCalledWith(expect.objectContaining({ ssl: false }));
expect(connect).toHaveBeenCalledOnce();
expect(end).toHaveBeenCalledOnce();
} finally {
await rm(root, { recursive: true, force: true });
}
});
test("concrete DWH direct diagnostics use strict TLS only for explicit TLS bindings", async () => {
const root = await mkdtemp(join(tmpdir(), "thoth-diagnostic-tls-"));
const passwordFile = join(root, "password");
const caFile = join(root, "ca.pem");
await writeFile(passwordFile, "password-value");
await writeFile(caFile, "test-ca");
const connect = vi.fn(async () => undefined);
const query = vi.fn(async () => ({ rows: [{ database: "warehouse", schema: "datawarehouse" }] }));
const end = vi.fn(async () => undefined);
const createPostgresClient = vi.fn(() => ({ connect, query, end }));
try {
const adapter = createConcreteDiagnosticAdapters({ createPostgresClient });
await adapter.probeConnector({
role: "dwh", transport: "postgres_direct", host: "dwh.example.test", port: 5432,
user: "reader", credentialFile: passwordFile, tlsCaFile: caFile,
tlsServername: "dwh.example.test",
resource: { database: "warehouse", schema: "datawarehouse" },
timeoutMs: 1_000, signal: new AbortController().signal,
});
expect(createPostgresClient).toHaveBeenCalledWith(expect.objectContaining({
ssl: { ca: "test-ca", servername: "dwh.example.test", rejectUnauthorized: true },
}));
expect(connect).toHaveBeenCalledOnce();
expect(end).toHaveBeenCalledOnce();
} finally {
await rm(root, { recursive: true, force: true });
}
});
test("concrete DWH direct diagnostics fail closed when the declared schema is inaccessible", async () => {
const root = await mkdtemp(join(tmpdir(), "thoth-diagnostic-schema-"));
const passwordFile = join(root, "password");
await writeFile(passwordFile, "CANARY-DATABASE-SECRET");
const end = vi.fn(async () => undefined);
const connect = vi.fn(async () => ({
query: vi.fn(async () => ({ rows: [{ database: "warehouse", schema: null }] })),
end,
}));
try {
const adapter = createConcreteDiagnosticAdapters({ databaseClient: { connect } });
await expect(adapter.probeConnector({
role: "dwh", transport: "postgres_direct", host: "127.0.0.1", port: 5432,
user: "reader", credentialFile: passwordFile,
resource: { database: "warehouse", schema: "datawarehouse" },
timeoutMs: 1_000, signal: new AbortController().signal,
})).rejects.toThrow("direct probe failed");
expect(end).toHaveBeenCalledOnce();
} finally {
await rm(root, { recursive: true, force: true });
}
});
test("concrete DWH REST diagnostics honor auth-none without reading credentials", async () => {
const fetchMock = vi.fn(async () => new Response(JSON.stringify({
database: "warehouse", schema: "datawarehouse",
}), { status: 200 }));
vi.stubGlobal("fetch", fetchMock);
const result = await createConcreteDiagnosticAdapters().probeConnector({
role: "dwh", transport: "rest_api", baseUrl: "https://dwh.example.test",
resource: { database: "warehouse", schema: "datawarehouse" },
timeoutMs: 1_000, signal: new AbortController().signal,
diagnostic: {
method: "GET", path: "/health", auth: "none",
response: { database: "database", schema: "schema" },
},
});
expect(result).toMatchObject({ resolved: true, tlsVerified: true, authenticated: true });
expect(fetchMock).toHaveBeenCalledWith("https://dwh.example.test/health", expect.objectContaining({
headers: {}, redirect: "error",
}));
});
test("concrete internal semantic diagnostics use only Qdrant and Ollama protocols", async () => {
const fetchMock = vi.fn()
.mockResolvedValueOnce(new Response(JSON.stringify({
result: { config: { params: { vectors: { size: 1024, distance: "Cosine" } } } },
}), { status: 200 }))
.mockResolvedValueOnce(new Response(JSON.stringify({ embeddings: [Array(1024).fill(0)] }), {
status: 200,
}));
vi.stubGlobal("fetch", fetchMock);
const adapter = createConcreteDiagnosticAdapters();
await expect(adapter.inspectQdrant({
baseUrl: "http://qdrant:6333", collection: "psd-clinical",
timeoutMs: 1_000, signal: new AbortController().signal,
})).resolves.toEqual({ collection: "psd-clinical", dimensions: 1024, distance: "cosine" });
await expect(adapter.probeEmbedding({
baseUrl: "http://embedding:11434", model: "qwen3-embedding:0.6b",
timeoutMs: 1_000, signal: new AbortController().signal,
})).resolves.toEqual({ available: true, dimensions: 1024 });
expect(fetchMock.mock.calls[0][0]).toBe("http://qdrant:6333/collections/psd-clinical");
expect(fetchMock.mock.calls[1][0]).toBe("http://embedding:11434/api/embed");
expect(JSON.parse(fetchMock.mock.calls[1][1].body)).toEqual({
model: "qwen3-embedding:0.6b", input: "diagnostic",
});
expect(Object.keys(adapter).sort()).toEqual(["inspectQdrant", "probeConnector", "probeEmbedding"]);
});
test("preserves a configured production timeout above the default up to the global maximum", async () => {
const adapters = successfulAdapters();
await createProductionWorkspaceDiagnoser(8_000, adapters)(workspace, bindings, {
writeProbe: false,
});
expect(adapters.probeConnector).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 8_000 }));
expect(adapters.inspectQdrant).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 8_000 }));
expect(adapters.probeEmbedding).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 8_000 }));
});
test("bounds descriptor DWH timeout by the configured production timeout", async () => {
const adapters = successfulAdapters();
await createProductionWorkspaceDiagnoser(10_000, adapters)(workspace, bindings, {
writeProbe: false,
});
expect(adapters.probeConnector).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 8_000 }));
expect(adapters.inspectQdrant).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 10_000 }));
expect(adapters.probeEmbedding).toHaveBeenCalledWith(expect.objectContaining({ timeoutMs: 10_000 }));
});
test("aborts a timed-out production diagnostic and returns only a sanitized connector code", async () => {
let aborted = false;
const adapters = successfulAdapters({
probeConnector: vi.fn((request) => new Promise((_resolve, reject) => {
request.signal.addEventListener("abort", () => {
aborted = true;
reject(new Error("CANARY-TIMEOUT-SECRET"));
}, { once: true });
})),
});
const result = await createProductionWorkspaceDiagnoser(5, adapters)(workspace, bindings, {
writeProbe: false,
});
expect(aborted).toBe(true);
expect(result.activatable).toBe(false);
expect(result.diagnostics).toContainEqual(expect.objectContaining({ code: "connector_unavailable" }));
expect(JSON.stringify(result)).not.toContain("CANARY-TIMEOUT-SECRET");
});
test.each([
["rejection", () => Promise.reject(new Error("CANARY-CONNECTOR-SECRET"))],
["unresolved connector", async (request: Parameters<DiagnosticAdapters["probeConnector"]>[0]) => ({
resolved: false, tlsVerified: true, authenticated: true, resource: request.resource,
})],
["wrong resource", async () => ({
resolved: true, tlsVerified: true, authenticated: true,
resource: { database: "other", schema: "datawarehouse" },
})],
["failed TLS", async (request: Parameters<DiagnosticAdapters["probeConnector"]>[0]) => ({
resolved: true, tlsVerified: false, authenticated: true, resource: request.resource,
})],
["failed authentication", async (request: Parameters<DiagnosticAdapters["probeConnector"]>[0]) => ({
resolved: true, tlsVerified: true, authenticated: false, resource: request.resource,
})],
] as const)("sanitizes DWH connector %s", async (_label, probeConnector) => {
const result = await diagnose(successfulAdapters({ probeConnector: vi.fn(probeConnector) }))(
workspace, bindings, { writeProbe: false },
);
expect(result.activatable).toBe(false);
expect(result.diagnostics).toContainEqual(expect.objectContaining({ code: "connector_unavailable" }));
expect(JSON.stringify(result)).not.toContain("CANARY-CONNECTOR-SECRET");
});
test("uses a REST secret only as a header and redacts it from failed diagnostics", async () => {
const root = await mkdtemp(join(tmpdir(), "thoth-rest-diagnostic-"));
const credentialFile = join(root, "api-key");
const canary = "CANARY-REST-AUTH-SECRET";
await writeFile(credentialFile, canary);
const restDescriptor = parseWorkspaceYaml(`workspace:
schema_version: 4
id: psd-clinical
name: REST auth
language: en
dwh:
engine: postgres
database: warehouse
schema: datawarehouse
supported_transports: [rest_api]
diagnostics:
dwh_rest:
method: GET
path: /health
auth: bearer
response: { database: database, schema: schema }
`);
const fetchMock = vi.fn()
.mockResolvedValueOnce(new Response("upstream CANARY-REST-AUTH-SECRET", { status: 503 }))
.mockResolvedValueOnce(new Response(JSON.stringify({
result: { config: { params: { vectors: { size: 1024, distance: "Cosine" } } } },
}), { status: 200 }))
.mockResolvedValueOnce(new Response(JSON.stringify({ embeddings: [Array(1024).fill(0)] }), {
status: 200,
}));
vi.stubGlobal("fetch", fetchMock);
try {
const result = await createWorkspaceDiagnoser(createConcreteDiagnosticAdapters())(
restDescriptor,
{
dwh: { transport: "rest_api", missing: [], values: {
THT_WS_PSD_CLINICAL_DWH_BASE_URL: "https://dwh.example.test",
THT_WS_PSD_CLINICAL_DWH_API_KEY_FILE: credentialFile,
} },
evidence: { missing: [], values: {} },
},
{ writeProbe: false },
);
expect(fetchMock.mock.calls[0][1].headers).toEqual({ authorization: `Bearer ${canary}` });
expect(result.diagnostics).toContainEqual(expect.objectContaining({ code: "connector_unavailable" }));
expect(JSON.stringify(result)).not.toContain(canary);
} finally {
await rm(root, { recursive: true, force: true });
}
});
test.each([
["Qdrant non-2xx", [
new Response("CANARY-QDRANT-BODY", { status: 503 }),
new Response(JSON.stringify({ embeddings: [Array(1024).fill(0)] }), { status: 200 }),
]],
["Qdrant malformed", [
new Response(JSON.stringify({ result: "CANARY-QDRANT-BODY" }), { status: 200 }),
new Response(JSON.stringify({ embeddings: [Array(1024).fill(0)] }), { status: 200 }),
]],
["Ollama non-2xx", [
new Response(JSON.stringify({ result: { config: { params: { vectors: {
size: 1024, distance: "Cosine",
} } } } }), { status: 200 }),
new Response("CANARY-OLLAMA-BODY", { status: 503 }),
]],
["Ollama malformed", [
new Response(JSON.stringify({ result: { config: { params: { vectors: {
size: 1024, distance: "Cosine",
} } } } }), { status: 200 }),
new Response(JSON.stringify({ embeddings: "CANARY-OLLAMA-BODY" }), { status: 200 }),
]],
] as const)("sanitizes %s failures", async (_label, responses) => {
const fetchMock = vi.fn();
for (const response of responses) fetchMock.mockResolvedValueOnce(response);
vi.stubGlobal("fetch", fetchMock);
const adapters = createConcreteDiagnosticAdapters({
directProtocol: {
probe: async (request) => ({
resolved: true, tlsVerified: true, authenticated: true, resource: request.resource,
}),
},
});
const result = await createWorkspaceDiagnoser(adapters)(workspace, bindings, { writeProbe: false });
expect(result.activatable).toBe(false);
expect(result.diagnostics).toContainEqual(expect.objectContaining({ code: "connector_unavailable" }));
expect(JSON.stringify(result)).not.toMatch(/CANARY-(QDRANT|OLLAMA)-BODY/);
});
test("closes the concrete PostgreSQL diagnostic client when resource verification fails", async () => {
const root = await mkdtemp(join(tmpdir(), "thoth-diagnostic-close-"));
const passwordFile = join(root, "password");
await writeFile(passwordFile, "CANARY-DATABASE-SECRET");
const end = vi.fn(async () => undefined);
const connect = vi.fn(async () => ({
query: vi.fn(async () => ({ rows: [{ database: "wrong", schema: "datawarehouse" }] })),
end,
}));
try {
const adapter = createConcreteDiagnosticAdapters({ databaseClient: { connect } });
await expect(adapter.probeConnector({
role: "dwh", transport: "postgres_direct", host: "127.0.0.1", port: 5432,
user: "reader", credentialFile: passwordFile,
resource: { database: "warehouse", schema: "datawarehouse" },
timeoutMs: 1_000, signal: new AbortController().signal,
})).rejects.toThrow("direct probe failed");
expect(end).toHaveBeenCalledOnce();
} finally {
await rm(root, { recursive: true, force: true });
}
});
test("returns observed normalized Qdrant distance for semantic mismatch classification", async () => {
vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify({
result: { config: { params: { vectors: { size: 1024, distance: "Euclid" } } } },
}), { status: 200 })));
await expect(createConcreteDiagnosticAdapters().inspectQdrant({
baseUrl: "http://qdrant:6333", collection: "psd-clinical",
timeoutMs: 1_000, signal: new AbortController().signal,
})).resolves.toEqual({ collection: "psd-clinical", dimensions: 1024, distance: "euclid" });
});
test("classifies an observed non-cosine Qdrant distance as semantic incompatibility", async () => {
const fetchMock = vi.fn()
.mockResolvedValueOnce(new Response(JSON.stringify({
result: { config: { params: { vectors: { size: 1024, distance: "Euclid" } } } },
}), { status: 200 }))
.mockResolvedValueOnce(new Response(JSON.stringify({ embeddings: [Array(1024).fill(0)] }), {
status: 200,
}));
vi.stubGlobal("fetch", fetchMock);
const adapters = createConcreteDiagnosticAdapters({
directProtocol: {
probe: async (request) => ({
resolved: true, tlsVerified: true, authenticated: true, resource: request.resource,
}),
},
});
const result = await createWorkspaceDiagnoser(adapters)(workspace, bindings, { writeProbe: false });
expect(result.activatable).toBe(false);
expect(result.diagnostics).toContainEqual(expect.objectContaining({
code: "semantic_index_incompatible",
}));
expect(result.diagnostics).not.toContainEqual(expect.objectContaining({
code: "connector_unavailable",
}));
});
test("resolveDiagnosticUrl appends the path to a base URL with a path prefix", () => {
expect(resolveDiagnosticUrl("https://dwh.example.test/dwh/", "/rpc/ping").toString())
.toBe("https://dwh.example.test/dwh/rpc/ping");
expect(resolveDiagnosticUrl("https://dwh.example.test/dwh", "/rpc/ping").toString())
.toBe("https://dwh.example.test/dwh/rpc/ping");
});