feat: add AI catalog description generation
This commit is contained in:
Codex
@@ -0,0 +1,384 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { PostgreSqlContainer } from "@testcontainers/postgresql";
|
||||
import { CamelCasePlugin, Kysely, PostgresDialect } from "kysely";
|
||||
import { Pool } from "pg";
|
||||
import { expect, test, vi } from "vitest";
|
||||
import { buildApp } from "../src/app.js";
|
||||
import type { DescriptionSourceSampler } from "../src/catalog/description-source-sampler.js";
|
||||
import type { MetadataGenerationModels } from "../src/catalog/metadata-generation-models.js";
|
||||
import { ModelCompletionProviderError, type ModelCompleter } from "../src/catalog/model-completer.js";
|
||||
import { up as upDatabases } from "../src/catalog/migrations/001_workspace_databases.js";
|
||||
import { up as upTables } from "../src/catalog/migrations/002_catalog_tables.js";
|
||||
import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema_sync.js";
|
||||
import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js";
|
||||
import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js";
|
||||
import { loadConfig } from "../src/config.js";
|
||||
import type { WorkspaceRegistry } from "../src/workspaces/registry.js";
|
||||
|
||||
const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
|
||||
|
||||
async function terminalRun(app: ReturnType<typeof buildApp>, runId: string) {
|
||||
for (let attempt = 0; attempt < 200; attempt += 1) {
|
||||
const response = await app.inject({
|
||||
method: "GET",
|
||||
url: `/catalog/description-generation-runs/${runId}`,
|
||||
});
|
||||
const run = response.json();
|
||||
if (["completed", "completed_with_errors", "cancelled", "failed", "interrupted"].includes(run.status)) {
|
||||
return run;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
throw new Error(`Description Generation Run ${runId} did not finish`);
|
||||
}
|
||||
|
||||
test.skipIf(!dockerAvailable)("Fastify persists Description Generation success and failure through PostgreSQL", async () => {
|
||||
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
|
||||
const db = new Kysely<CatalogDatabase>({
|
||||
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
|
||||
plugins: [new CamelCasePlugin()],
|
||||
});
|
||||
let app: ReturnType<typeof buildApp> | undefined;
|
||||
try {
|
||||
await upDatabases(db);
|
||||
await upTables(db);
|
||||
await upSchemaSync(db);
|
||||
await upDescriptionGeneration(db);
|
||||
const repository = new KyselyCatalogRepository(db);
|
||||
const database = await repository.create({
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "datawarehouse",
|
||||
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
||||
});
|
||||
await repository.applySchemaSync(database.id, database.version, "all", [], {
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||
tables: [{ name: "patients", sourceComment: "Clinical patients" }],
|
||||
columns: [
|
||||
{
|
||||
tableName: "patients",
|
||||
name: "birth_date",
|
||||
ordinalPosition: 1,
|
||||
dataType: "date",
|
||||
isNullable: true,
|
||||
defaultExpression: null,
|
||||
primaryKeyPosition: null,
|
||||
sourceComment: "Patient date of birth",
|
||||
},
|
||||
{
|
||||
tableName: "patients",
|
||||
name: "status",
|
||||
ordinalPosition: 2,
|
||||
dataType: "text",
|
||||
isNullable: true,
|
||||
defaultExpression: null,
|
||||
primaryKeyPosition: null,
|
||||
sourceComment: "Patient status",
|
||||
},
|
||||
],
|
||||
relationships: [],
|
||||
});
|
||||
const table = (await repository.listTables(database.id))[0]!;
|
||||
const columns = await repository.listColumns(database.id, table.id);
|
||||
const birthDate = columns.find((column) => column.name === "birth_date")!;
|
||||
const status = columns.find((column) => column.name === "status")!;
|
||||
const curatedTable = (await repository.updateTableMetadata(
|
||||
database.id,
|
||||
table.id,
|
||||
table.version,
|
||||
"Elenco curato dei pazienti.",
|
||||
null,
|
||||
))!;
|
||||
const curatedStatus = (await repository.updateColumnMetadata(
|
||||
database.id,
|
||||
table.id,
|
||||
status.id,
|
||||
status.version,
|
||||
"Stato curato del paziente.",
|
||||
null,
|
||||
))!;
|
||||
let call = 0;
|
||||
const modelCompleter: ModelCompleter = {
|
||||
complete: vi.fn(async () => {
|
||||
call += 1;
|
||||
if (call === 1) {
|
||||
return JSON.stringify({ results: [
|
||||
{
|
||||
targetId: birthDate.id,
|
||||
outcome: "generated",
|
||||
description: "Data di nascita del paziente.",
|
||||
},
|
||||
{ targetId: status.id, outcome: "non_generatable" },
|
||||
] });
|
||||
}
|
||||
if (call === 2) {
|
||||
return JSON.stringify({ results: [{
|
||||
targetId: table.id,
|
||||
outcome: "generated",
|
||||
description: "Elenco dei pazienti e dei loro dati clinici.",
|
||||
}] });
|
||||
}
|
||||
if (call === 4) {
|
||||
return JSON.stringify({ results: [
|
||||
{
|
||||
targetId: birthDate.id,
|
||||
outcome: "generated",
|
||||
description: "Descrizione rigenerata della data di nascita.",
|
||||
},
|
||||
{
|
||||
targetId: status.id,
|
||||
outcome: "generated",
|
||||
description: "Descrizione rigenerata dello stato.",
|
||||
},
|
||||
] });
|
||||
}
|
||||
if (call === 5) {
|
||||
return JSON.stringify({ results: [{
|
||||
targetId: table.id,
|
||||
outcome: "generated",
|
||||
description: "Descrizione rigenerata della tabella pazienti.",
|
||||
}] });
|
||||
}
|
||||
if (call === 6) {
|
||||
return JSON.stringify({ results: [{
|
||||
targetId: birthDate.id,
|
||||
outcome: "generated",
|
||||
description: "Descrizione recuperata della data di nascita.",
|
||||
}] });
|
||||
}
|
||||
throw new ModelCompletionProviderError();
|
||||
}),
|
||||
};
|
||||
const models: MetadataGenerationModels = {
|
||||
catalog: () => ({ models: [{ id: "openai-mini", label: "OpenAI Mini" }], default: "openai-mini" }),
|
||||
resolve: () => ({
|
||||
id: "openai-mini",
|
||||
provider: "openai",
|
||||
model: "gpt-4.1-mini",
|
||||
apiKeyEnv: "OPENAI_API_KEY",
|
||||
apiKey: "test-provider-secret",
|
||||
}),
|
||||
};
|
||||
const registry = {
|
||||
list: vi.fn(async () => []),
|
||||
read: vi.fn(async () => ({
|
||||
workspace: { workspace: { language: "it" } },
|
||||
revision: {},
|
||||
})),
|
||||
} as unknown as WorkspaceRegistry;
|
||||
const persistedSampleSecret = "POSTGRES_TRANSIENT_SAMPLE_6a0d7b";
|
||||
const descriptionSourceSampler: DescriptionSourceSampler = {
|
||||
sample: vi.fn(async (_database, targets) => targets.map((target) => ({
|
||||
targetId: target.targetId,
|
||||
tableName: target.tableName,
|
||||
rows: [{
|
||||
fields: target.columnNames.slice(0, 1).map((name) => ({
|
||||
name,
|
||||
value: persistedSampleSecret,
|
||||
})),
|
||||
}],
|
||||
representativeValues: target.columnNames.slice(0, 1).map((column) => ({
|
||||
column,
|
||||
values: [persistedSampleSecret],
|
||||
})),
|
||||
}))),
|
||||
};
|
||||
app = buildApp(loadConfig({ NODE_ENV: "test", THT_HARNESS_DIR: "/missing" }), {
|
||||
thtRunner: {} as never,
|
||||
workspaceRegistry: registry,
|
||||
workspaceDiagnoser: vi.fn(),
|
||||
catalogRepository: repository,
|
||||
metadataGenerationModels: models,
|
||||
modelCompleter,
|
||||
descriptionSourceSampler,
|
||||
});
|
||||
|
||||
const successfulStart = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/description-generation-runs`,
|
||||
payload: {
|
||||
modelId: "openai-mini",
|
||||
scope: "selected_columns",
|
||||
targetIds: [status.id, birthDate.id],
|
||||
},
|
||||
});
|
||||
expect(successfulStart.statusCode).toBe(202);
|
||||
expect(await terminalRun(app, successfulStart.json().id)).toMatchObject({
|
||||
status: "completed",
|
||||
total: 2,
|
||||
processed: 2,
|
||||
generated: 1,
|
||||
nonGeneratable: 1,
|
||||
failed: 0,
|
||||
});
|
||||
expect(await repository.getColumn(database.id, table.id, birthDate.id)).toMatchObject({
|
||||
generatedDescription: "Data di nascita del paziente.",
|
||||
version: birthDate.version + 1,
|
||||
});
|
||||
const generatedStatus = (await repository.getColumn(database.id, table.id, status.id))!;
|
||||
expect(generatedStatus).toMatchObject({
|
||||
description: "Stato curato del paziente.",
|
||||
generatedDescription: "Non generabile",
|
||||
version: curatedStatus.version + 1,
|
||||
});
|
||||
|
||||
const tableStart = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/description-generation-runs`,
|
||||
payload: { modelId: "openai-mini", scope: "selected_tables", targetIds: [table.id] },
|
||||
});
|
||||
expect(tableStart.statusCode).toBe(202);
|
||||
expect(await terminalRun(app, tableStart.json().id)).toMatchObject({
|
||||
scope: "selected_tables",
|
||||
status: "completed",
|
||||
processed: 1,
|
||||
generated: 1,
|
||||
nonGeneratable: 0,
|
||||
failed: 0,
|
||||
});
|
||||
expect(await repository.getTable(database.id, table.id)).toMatchObject({
|
||||
description: "Elenco curato dei pazienti.",
|
||||
generatedDescription: "Elenco dei pazienti e dei loro dati clinici.",
|
||||
version: curatedTable.version + 1,
|
||||
});
|
||||
|
||||
const failedStart = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/description-generation-runs`,
|
||||
payload: { modelId: "openai-mini", scope: "selected_columns", targetIds: [status.id] },
|
||||
});
|
||||
expect(failedStart.statusCode).toBe(202);
|
||||
const failedRun = await terminalRun(app, failedStart.json().id);
|
||||
expect(failedRun).toMatchObject({
|
||||
status: "completed_with_errors",
|
||||
processed: 1,
|
||||
generated: 0,
|
||||
failed: 1,
|
||||
errorSummary: "Description generation completed with errors.",
|
||||
});
|
||||
expect(await repository.getColumn(database.id, table.id, status.id)).toMatchObject({
|
||||
description: "Stato curato del paziente.",
|
||||
generatedDescription: "Non generabile",
|
||||
version: generatedStatus.version,
|
||||
});
|
||||
const events = await app.inject({
|
||||
method: "GET",
|
||||
url: `/catalog/description-generation-runs/${failedRun.id}/events-list`,
|
||||
});
|
||||
expect(events.statusCode).toBe(200);
|
||||
expect(events.json().find((event: { level: string }) => event.level === "error")).toEqual(expect.objectContaining({
|
||||
level: "error",
|
||||
message: `The model provider request failed. Affected Catalog Column target: ${status.id}.`,
|
||||
}));
|
||||
expect(events.body).not.toMatch(/test-provider-secret|gpt-4\.1-mini|raw provider/i);
|
||||
|
||||
const allStart = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/description-generation-runs`,
|
||||
payload: { modelId: "openai-mini", scope: "all" },
|
||||
});
|
||||
expect(allStart.statusCode).toBe(202);
|
||||
const allRun = await terminalRun(app, allStart.json().id);
|
||||
expect(allRun).toMatchObject({
|
||||
scope: "all",
|
||||
status: "completed",
|
||||
total: 3,
|
||||
processed: 3,
|
||||
generated: 3,
|
||||
nonGeneratable: 0,
|
||||
failed: 0,
|
||||
});
|
||||
expect(await repository.getColumn(database.id, table.id, birthDate.id)).toMatchObject({
|
||||
generatedDescription: "Descrizione rigenerata della data di nascita.",
|
||||
});
|
||||
expect(await repository.getColumn(database.id, table.id, status.id)).toMatchObject({
|
||||
generatedDescription: "Descrizione rigenerata dello stato.",
|
||||
});
|
||||
expect(await repository.getTable(database.id, table.id)).toMatchObject({
|
||||
generatedDescription: "Descrizione rigenerata della tabella pazienti.",
|
||||
});
|
||||
const allEvents = await app.inject({
|
||||
method: "GET",
|
||||
url: `/catalog/description-generation-runs/${allRun.id}/events-list`,
|
||||
});
|
||||
expect(allEvents.json()[0]).toEqual(expect.objectContaining({
|
||||
message: "Description generation queued (scope: all).",
|
||||
}));
|
||||
|
||||
const regeneratedBirthDate = (await repository.getColumn(
|
||||
database.id, table.id, birthDate.id,
|
||||
))!;
|
||||
await repository.updateColumnMetadata(
|
||||
database.id,
|
||||
table.id,
|
||||
birthDate.id,
|
||||
regeneratedBirthDate.version,
|
||||
regeneratedBirthDate.description,
|
||||
" ",
|
||||
);
|
||||
const missingStart = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/description-generation-runs`,
|
||||
payload: { modelId: "openai-mini", scope: "missing" },
|
||||
});
|
||||
expect(missingStart.statusCode).toBe(202);
|
||||
expect(await terminalRun(app, missingStart.json().id)).toMatchObject({
|
||||
scope: "missing",
|
||||
status: "completed",
|
||||
total: 1,
|
||||
processed: 1,
|
||||
generated: 1,
|
||||
nonGeneratable: 0,
|
||||
failed: 0,
|
||||
});
|
||||
expect(await repository.getColumn(database.id, table.id, birthDate.id)).toMatchObject({
|
||||
generatedDescription: "Descrizione recuperata della data di nascita.",
|
||||
});
|
||||
expect(modelCompleter.complete).toHaveBeenCalledTimes(6);
|
||||
expect(JSON.stringify(vi.mocked(modelCompleter.complete).mock.calls)).toContain(persistedSampleSecret);
|
||||
|
||||
const runIds = [
|
||||
successfulStart.json().id,
|
||||
tableStart.json().id,
|
||||
failedStart.json().id,
|
||||
allStart.json().id,
|
||||
missingStart.json().id,
|
||||
];
|
||||
const durableState = JSON.stringify({
|
||||
runs: await Promise.all(runIds.map(async (runId) => (
|
||||
await repository.getDescriptionGenerationRun(runId)
|
||||
))),
|
||||
events: await Promise.all(runIds.map(async (runId) => (
|
||||
await repository.listDescriptionGenerationEvents(runId)
|
||||
))),
|
||||
database: await repository.get(database.id),
|
||||
table: await repository.getTable(database.id, table.id),
|
||||
columns: await repository.listColumns(database.id, table.id),
|
||||
});
|
||||
expect(durableState).not.toContain(persistedSampleSecret);
|
||||
const apiResponses = await Promise.all([
|
||||
...runIds.flatMap((runId) => [
|
||||
app!.inject({ method: "GET", url: `/catalog/description-generation-runs/${runId}` }),
|
||||
app!.inject({
|
||||
method: "GET",
|
||||
url: `/catalog/description-generation-runs/${runId}/events-list`,
|
||||
}),
|
||||
]),
|
||||
app.inject({ method: "GET", url: `/catalog/databases/${database.id}` }),
|
||||
app.inject({ method: "GET", url: `/catalog/databases/${database.id}/tables` }),
|
||||
app.inject({
|
||||
method: "GET",
|
||||
url: `/catalog/databases/${database.id}/tables/${table.id}/columns`,
|
||||
}),
|
||||
]);
|
||||
expect(apiResponses.map((response) => response.body).join("\n")).not.toContain(
|
||||
persistedSampleSecret,
|
||||
);
|
||||
} finally {
|
||||
if (app) await app.close();
|
||||
await db.destroy();
|
||||
await container.stop();
|
||||
}
|
||||
}, 60_000);
|
||||
Reference in New Issue
Block a user