393 lines
15 KiB
TypeScript
393 lines
15 KiB
TypeScript
import { spawnSync } from "node:child_process";
|
|
import { PostgreSqlContainer } from "@testcontainers/postgresql";
|
|
import { CamelCasePlugin, Kysely, PostgresDialect } from "kysely";
|
|
import { Pool } from "pg";
|
|
import { expect, test, vi } from "vitest";
|
|
import { buildApp } from "../src/app.js";
|
|
import type { DescriptionSourceSampler } from "../src/catalog/description-source-sampler.js";
|
|
import type { MetadataGenerationModels } from "../src/catalog/metadata-generation-models.js";
|
|
import { ModelCompletionProviderError, type ModelCompleter } from "../src/catalog/model-completer.js";
|
|
import { up as upDatabases } from "../src/catalog/migrations/001_workspace_databases.js";
|
|
import { up as upTables } from "../src/catalog/migrations/002_catalog_tables.js";
|
|
import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema_sync.js";
|
|
import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js";
|
|
import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js";
|
|
import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js";
|
|
import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js";
|
|
import { up as upCanonicalModelIds } from "../src/catalog/migrations/010_canonical_model_ids.js";
|
|
import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js";
|
|
import { loadConfig } from "../src/config.js";
|
|
import type { WorkspaceRegistry } from "../src/workspaces/registry.js";
|
|
|
|
const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
|
|
|
|
async function terminalRun(app: ReturnType<typeof buildApp>, runId: string) {
|
|
for (let attempt = 0; attempt < 200; attempt += 1) {
|
|
const response = await app.inject({
|
|
method: "GET",
|
|
url: `/catalog/description-generation-runs/${runId}`,
|
|
});
|
|
const run = response.json();
|
|
if (["completed", "completed_with_errors", "cancelled", "failed", "interrupted"].includes(run.status)) {
|
|
return run;
|
|
}
|
|
await new Promise((resolve) => setTimeout(resolve, 5));
|
|
}
|
|
throw new Error(`Description Generation Run ${runId} did not finish`);
|
|
}
|
|
|
|
test.skipIf(!dockerAvailable)("Fastify persists Description Generation success and failure through PostgreSQL", async () => {
|
|
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
|
|
const db = new Kysely<CatalogDatabase>({
|
|
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
|
|
plugins: [new CamelCasePlugin()],
|
|
});
|
|
let app: ReturnType<typeof buildApp> | undefined;
|
|
try {
|
|
await upDatabases(db);
|
|
await upTables(db);
|
|
await upSchemaSync(db);
|
|
await upSensitiveDataFlag(db);
|
|
await upDescriptionGeneration(db);
|
|
await upSensitiveSuggestionRuns(db);
|
|
await upAiTokenUsage(db);
|
|
await upCanonicalModelIds(db);
|
|
const repository = new KyselyCatalogRepository(db);
|
|
const database = await repository.create({
|
|
workspaceId: "psd-clinical",
|
|
engine: "postgres",
|
|
databaseName: "warehouse",
|
|
schema: "datawarehouse",
|
|
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
|
});
|
|
await repository.applySchemaSync(database.id, database.version, "all", [], {
|
|
schemaVersion: 1,
|
|
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
|
tables: [{ name: "patients", sourceComment: "Clinical patients" }],
|
|
columns: [
|
|
{
|
|
tableName: "patients",
|
|
name: "birth_date",
|
|
ordinalPosition: 1,
|
|
dataType: "date",
|
|
isNullable: true,
|
|
defaultExpression: null,
|
|
primaryKeyPosition: null,
|
|
sourceComment: "Patient date of birth",
|
|
},
|
|
{
|
|
tableName: "patients",
|
|
name: "status",
|
|
ordinalPosition: 2,
|
|
dataType: "text",
|
|
isNullable: true,
|
|
defaultExpression: null,
|
|
primaryKeyPosition: null,
|
|
sourceComment: "Patient status",
|
|
},
|
|
],
|
|
relationships: [],
|
|
});
|
|
const table = (await repository.listTables(database.id))[0]!;
|
|
const columns = await repository.listColumns(database.id, table.id);
|
|
const birthDate = columns.find((column) => column.name === "birth_date")!;
|
|
const status = columns.find((column) => column.name === "status")!;
|
|
const curatedTable = (await repository.updateTableMetadata(
|
|
database.id,
|
|
table.id,
|
|
table.version,
|
|
"Elenco curato dei pazienti.",
|
|
null,
|
|
))!;
|
|
const curatedStatus = (await repository.updateColumnMetadata(
|
|
database.id,
|
|
table.id,
|
|
status.id,
|
|
status.version,
|
|
"Stato curato del paziente.",
|
|
null,
|
|
))!;
|
|
let call = 0;
|
|
const modelCompleter: ModelCompleter = {
|
|
complete: vi.fn(async () => {
|
|
call += 1;
|
|
if (call === 1) {
|
|
return JSON.stringify({ results: [
|
|
{
|
|
targetId: birthDate.id,
|
|
outcome: "generated",
|
|
description: "Data di nascita del paziente.",
|
|
},
|
|
{ targetId: status.id, outcome: "non_generatable" },
|
|
] });
|
|
}
|
|
if (call === 2) {
|
|
return JSON.stringify({ results: [{
|
|
targetId: table.id,
|
|
outcome: "generated",
|
|
description: "Elenco dei pazienti e dei loro dati clinici.",
|
|
}] });
|
|
}
|
|
if (call === 5) {
|
|
return JSON.stringify({ results: [
|
|
{
|
|
targetId: birthDate.id,
|
|
outcome: "generated",
|
|
description: "Descrizione rigenerata della data di nascita.",
|
|
},
|
|
{
|
|
targetId: status.id,
|
|
outcome: "generated",
|
|
description: "Descrizione rigenerata dello stato.",
|
|
},
|
|
] });
|
|
}
|
|
if (call === 6) {
|
|
return JSON.stringify({ results: [{
|
|
targetId: table.id,
|
|
outcome: "generated",
|
|
description: "Descrizione rigenerata della tabella pazienti.",
|
|
}] });
|
|
}
|
|
if (call === 7) {
|
|
return JSON.stringify({ results: [{
|
|
targetId: birthDate.id,
|
|
outcome: "generated",
|
|
description: "Descrizione recuperata della data di nascita.",
|
|
}] });
|
|
}
|
|
throw new ModelCompletionProviderError();
|
|
}),
|
|
};
|
|
const models: MetadataGenerationModels = {
|
|
catalog: () => ({ models: [{ id: "openai/gpt-4.1-mini", label: "OpenAI Mini" }], default: "openai/gpt-4.1-mini" }),
|
|
resolve: () => ({
|
|
id: "openai/gpt-4.1-mini",
|
|
provider: "openai",
|
|
model: "gpt-4.1-mini",
|
|
apiKeyEnv: "OPENAI_API_KEY",
|
|
apiKey: "test-provider-secret",
|
|
}),
|
|
};
|
|
const registry = {
|
|
list: vi.fn(async () => []),
|
|
read: vi.fn(async () => ({
|
|
workspace: { workspace: { language: "it" } },
|
|
revision: {},
|
|
})),
|
|
} as unknown as WorkspaceRegistry;
|
|
const persistedSampleSecret = "POSTGRES_TRANSIENT_SAMPLE_6a0d7b";
|
|
const descriptionSourceSampler: DescriptionSourceSampler = {
|
|
sample: vi.fn(async (_database, targets) => targets.map((target) => ({
|
|
targetId: target.targetId,
|
|
tableName: target.tableName,
|
|
rows: [{
|
|
fields: target.columnNames.slice(0, 1).map((name) => ({
|
|
name,
|
|
value: persistedSampleSecret,
|
|
})),
|
|
}],
|
|
representativeValues: target.columnNames.slice(0, 1).map((column) => ({
|
|
column,
|
|
values: [persistedSampleSecret],
|
|
})),
|
|
}))),
|
|
};
|
|
app = buildApp(loadConfig({ NODE_ENV: "test", THT_HARNESS_DIR: "/missing" }), {
|
|
thtRunner: {} as never,
|
|
workspaceRegistry: registry,
|
|
workspaceDiagnoser: vi.fn(),
|
|
catalogRepository: repository,
|
|
metadataGenerationModels: models,
|
|
modelCompleter,
|
|
descriptionSourceSampler,
|
|
});
|
|
|
|
const successfulStart = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/description-generation-runs`,
|
|
payload: {
|
|
modelId: "openai/gpt-4.1-mini",
|
|
scope: "selected_columns",
|
|
targetIds: [status.id, birthDate.id],
|
|
},
|
|
});
|
|
expect(successfulStart.statusCode, successfulStart.body).toBe(202);
|
|
expect(await terminalRun(app, successfulStart.json().id)).toMatchObject({
|
|
status: "completed",
|
|
total: 2,
|
|
processed: 2,
|
|
generated: 1,
|
|
nonGeneratable: 1,
|
|
failed: 0,
|
|
});
|
|
expect(await repository.getColumn(database.id, table.id, birthDate.id)).toMatchObject({
|
|
generatedDescription: "Data di nascita del paziente.",
|
|
version: birthDate.version + 1,
|
|
});
|
|
const generatedStatus = (await repository.getColumn(database.id, table.id, status.id))!;
|
|
expect(generatedStatus).toMatchObject({
|
|
description: "Stato curato del paziente.",
|
|
generatedDescription: "Non generabile",
|
|
version: curatedStatus.version + 1,
|
|
});
|
|
|
|
const tableStart = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/description-generation-runs`,
|
|
payload: { modelId: "openai/gpt-4.1-mini", scope: "selected_tables", targetIds: [table.id] },
|
|
});
|
|
expect(tableStart.statusCode).toBe(202);
|
|
expect(await terminalRun(app, tableStart.json().id)).toMatchObject({
|
|
scope: "selected_tables",
|
|
status: "completed",
|
|
processed: 1,
|
|
generated: 1,
|
|
nonGeneratable: 0,
|
|
failed: 0,
|
|
});
|
|
expect(await repository.getTable(database.id, table.id)).toMatchObject({
|
|
description: "Elenco curato dei pazienti.",
|
|
generatedDescription: "Elenco dei pazienti e dei loro dati clinici.",
|
|
version: curatedTable.version + 1,
|
|
});
|
|
|
|
const failedStart = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/description-generation-runs`,
|
|
payload: { modelId: "openai/gpt-4.1-mini", scope: "selected_columns", targetIds: [status.id] },
|
|
});
|
|
expect(failedStart.statusCode).toBe(202);
|
|
const failedRun = await terminalRun(app, failedStart.json().id);
|
|
expect(failedRun).toMatchObject({
|
|
status: "completed_with_errors",
|
|
processed: 1,
|
|
generated: 0,
|
|
failed: 1,
|
|
errorSummary: "Description generation completed with errors.",
|
|
});
|
|
expect(await repository.getColumn(database.id, table.id, status.id)).toMatchObject({
|
|
description: "Stato curato del paziente.",
|
|
generatedDescription: "Non generabile",
|
|
version: generatedStatus.version,
|
|
});
|
|
const events = await app.inject({
|
|
method: "GET",
|
|
url: `/catalog/description-generation-runs/${failedRun.id}/events-list`,
|
|
});
|
|
expect(events.statusCode).toBe(200);
|
|
expect(events.json().find((event: { level: string }) => event.level === "error")).toEqual(expect.objectContaining({
|
|
level: "error",
|
|
message: `The model provider request failed. Affected Catalog Column target: ${status.id}.`,
|
|
}));
|
|
expect(events.body).not.toMatch(/test-provider-secret|gpt-4\.1-mini|raw provider/i);
|
|
|
|
const allStart = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/description-generation-runs`,
|
|
payload: { modelId: "openai/gpt-4.1-mini", scope: "all" },
|
|
});
|
|
expect(allStart.statusCode).toBe(202);
|
|
const allRun = await terminalRun(app, allStart.json().id);
|
|
expect(allRun).toMatchObject({
|
|
scope: "all",
|
|
status: "completed",
|
|
total: 3,
|
|
processed: 3,
|
|
generated: 3,
|
|
nonGeneratable: 0,
|
|
failed: 0,
|
|
});
|
|
expect(await repository.getColumn(database.id, table.id, birthDate.id)).toMatchObject({
|
|
generatedDescription: "Descrizione rigenerata della data di nascita.",
|
|
});
|
|
expect(await repository.getColumn(database.id, table.id, status.id)).toMatchObject({
|
|
generatedDescription: "Descrizione rigenerata dello stato.",
|
|
});
|
|
expect(await repository.getTable(database.id, table.id)).toMatchObject({
|
|
generatedDescription: "Descrizione rigenerata della tabella pazienti.",
|
|
});
|
|
const allEvents = await app.inject({
|
|
method: "GET",
|
|
url: `/catalog/description-generation-runs/${allRun.id}/events-list`,
|
|
});
|
|
expect(allEvents.json()[0]).toEqual(expect.objectContaining({
|
|
message: "Description generation queued (scope: all).",
|
|
}));
|
|
|
|
const regeneratedBirthDate = (await repository.getColumn(
|
|
database.id, table.id, birthDate.id,
|
|
))!;
|
|
await repository.updateColumnMetadata(
|
|
database.id,
|
|
table.id,
|
|
birthDate.id,
|
|
regeneratedBirthDate.version,
|
|
regeneratedBirthDate.description,
|
|
" ",
|
|
);
|
|
const missingStart = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/description-generation-runs`,
|
|
payload: { modelId: "openai/gpt-4.1-mini", scope: "missing" },
|
|
});
|
|
expect(missingStart.statusCode).toBe(202);
|
|
expect(await terminalRun(app, missingStart.json().id)).toMatchObject({
|
|
scope: "missing",
|
|
status: "completed",
|
|
total: 1,
|
|
processed: 1,
|
|
generated: 1,
|
|
nonGeneratable: 0,
|
|
failed: 0,
|
|
});
|
|
expect(await repository.getColumn(database.id, table.id, birthDate.id)).toMatchObject({
|
|
generatedDescription: "Descrizione recuperata della data di nascita.",
|
|
});
|
|
expect(modelCompleter.complete).toHaveBeenCalledTimes(7);
|
|
expect(JSON.stringify(vi.mocked(modelCompleter.complete).mock.calls)).toContain(persistedSampleSecret);
|
|
|
|
const runIds = [
|
|
successfulStart.json().id,
|
|
tableStart.json().id,
|
|
failedStart.json().id,
|
|
allStart.json().id,
|
|
missingStart.json().id,
|
|
];
|
|
const durableState = JSON.stringify({
|
|
runs: await Promise.all(runIds.map(async (runId) => (
|
|
await repository.getDescriptionGenerationRun(runId)
|
|
))),
|
|
events: await Promise.all(runIds.map(async (runId) => (
|
|
await repository.listDescriptionGenerationEvents(runId)
|
|
))),
|
|
database: await repository.get(database.id),
|
|
table: await repository.getTable(database.id, table.id),
|
|
columns: await repository.listColumns(database.id, table.id),
|
|
});
|
|
expect(durableState).not.toContain(persistedSampleSecret);
|
|
const apiResponses = await Promise.all([
|
|
...runIds.flatMap((runId) => [
|
|
app!.inject({ method: "GET", url: `/catalog/description-generation-runs/${runId}` }),
|
|
app!.inject({
|
|
method: "GET",
|
|
url: `/catalog/description-generation-runs/${runId}/events-list`,
|
|
}),
|
|
]),
|
|
app.inject({ method: "GET", url: `/catalog/databases/${database.id}` }),
|
|
app.inject({ method: "GET", url: `/catalog/databases/${database.id}/tables` }),
|
|
app.inject({
|
|
method: "GET",
|
|
url: `/catalog/databases/${database.id}/tables/${table.id}/columns`,
|
|
}),
|
|
]);
|
|
expect(apiResponses.map((response) => response.body).join("\n")).not.toContain(
|
|
persistedSampleSecret,
|
|
);
|
|
} finally {
|
|
if (app) await app.close();
|
|
await db.destroy();
|
|
await container.stop();
|
|
}
|
|
}, 60_000);
|