feat: add AI catalog description generation
This commit is contained in:
Codex
@@ -9,6 +9,7 @@ import { up as upDatabases } from "../src/catalog/migrations/001_workspace_datab
|
||||
import { up as upTables } from "../src/catalog/migrations/002_catalog_tables.js";
|
||||
import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema_sync.js";
|
||||
import { up as upRuntimeSequencePrivileges } from "../src/catalog/migrations/004_catalog_runtime_sequence_privileges.js";
|
||||
import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js";
|
||||
|
||||
const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
|
||||
|
||||
@@ -200,3 +201,285 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository performs scoped metadata cl
|
||||
await container.stop();
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates selected table and column descriptions", async () => {
|
||||
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
|
||||
const db = new Kysely<CatalogDatabase>({
|
||||
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
|
||||
plugins: [new CamelCasePlugin()],
|
||||
});
|
||||
try {
|
||||
await upDatabases(db);
|
||||
await upTables(db);
|
||||
await upSchemaSync(db);
|
||||
const repository = new KyselyCatalogRepository(db);
|
||||
const database = await repository.create({
|
||||
workspaceId: "consolidation-test",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "public",
|
||||
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
||||
});
|
||||
const snapshot: ObservedSchemaSnapshot = {
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||
tables: [
|
||||
{ name: "patients", sourceComment: null },
|
||||
{ name: "visits", sourceComment: null },
|
||||
],
|
||||
columns: [
|
||||
{ tableName: "visits", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
||||
{ tableName: "visits", name: "patient_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
||||
],
|
||||
relationships: [],
|
||||
};
|
||||
await repository.applySchemaSync(database.id, database.version, "all", [], snapshot);
|
||||
const tables = await repository.listTables(database.id);
|
||||
const patients = tables.find((table) => table.name === "patients")!;
|
||||
const visits = tables.find((table) => table.name === "visits")!;
|
||||
await repository.updateTableMetadata(
|
||||
database.id, patients.id, patients.version, "Old patients", "Generated patients",
|
||||
);
|
||||
await repository.updateTableMetadata(
|
||||
database.id, visits.id, visits.version, "Keep visits", " ",
|
||||
);
|
||||
|
||||
expect(await repository.consolidateGeneratedDescriptions(
|
||||
database.id, "tables", [patients.id, visits.id],
|
||||
)).toEqual({ copied: 1, skipped: 1 });
|
||||
expect(await repository.getTable(database.id, patients.id)).toMatchObject({
|
||||
description: "Generated patients",
|
||||
generatedDescription: "Generated patients",
|
||||
version: patients.version + 2,
|
||||
});
|
||||
expect(await repository.getTable(database.id, visits.id)).toMatchObject({
|
||||
description: "Keep visits",
|
||||
generatedDescription: " ",
|
||||
version: visits.version + 1,
|
||||
});
|
||||
|
||||
const columns = await repository.listColumns(database.id, visits.id);
|
||||
const id = columns.find((column) => column.name === "id")!;
|
||||
const patientId = columns.find((column) => column.name === "patient_id")!;
|
||||
await repository.updateColumnMetadata(
|
||||
database.id, visits.id, id.id, id.version, "Old id", "Generated id",
|
||||
);
|
||||
await repository.updateColumnMetadata(
|
||||
database.id, visits.id, patientId.id, patientId.version, "Keep patient reference", null,
|
||||
);
|
||||
|
||||
expect(await repository.consolidateGeneratedDescriptions(
|
||||
database.id, "columns", [id.id, patientId.id],
|
||||
)).toEqual({ copied: 1, skipped: 1 });
|
||||
expect(await repository.getColumn(database.id, visits.id, id.id)).toMatchObject({
|
||||
description: "Generated id",
|
||||
generatedDescription: "Generated id",
|
||||
version: id.version + 2,
|
||||
});
|
||||
expect(await repository.getColumn(database.id, visits.id, patientId.id)).toMatchObject({
|
||||
description: "Keep patient reference",
|
||||
generatedDescription: null,
|
||||
version: patientId.version + 1,
|
||||
});
|
||||
|
||||
const currentVisits = (await repository.getTable(database.id, visits.id))!;
|
||||
await repository.updateTableMetadata(
|
||||
database.id, visits.id, currentVisits.version, "Still curated visits", "Generated visits",
|
||||
);
|
||||
expect(await repository.consolidateGeneratedDescriptions(database.id, "tables", [
|
||||
visits.id,
|
||||
"99999999-9999-4999-8999-999999999999",
|
||||
])).toBeUndefined();
|
||||
expect(await repository.getTable(database.id, visits.id)).toMatchObject({
|
||||
description: "Still curated visits",
|
||||
generatedDescription: "Generated visits",
|
||||
});
|
||||
} finally {
|
||||
await db.destroy();
|
||||
await container.stop();
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
test.skipIf(!dockerAvailable)("PostgreSQL repository persists globally exclusive Description Generation Runs and ordered events", async () => {
|
||||
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
|
||||
const db = new Kysely<CatalogDatabase>({
|
||||
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
|
||||
plugins: [new CamelCasePlugin()],
|
||||
});
|
||||
try {
|
||||
await upDatabases(db);
|
||||
await upTables(db);
|
||||
await upSchemaSync(db);
|
||||
await upDescriptionGeneration(db);
|
||||
const repository = new KyselyCatalogRepository(db);
|
||||
const firstDatabase = await repository.create({
|
||||
workspaceId: "generation-one",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse_one",
|
||||
schema: "public",
|
||||
binding: { transport: "postgres_direct", host: "one.internal", port: 5432, username: "reader" },
|
||||
});
|
||||
const secondDatabase = await repository.create({
|
||||
workspaceId: "generation-two",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse_two",
|
||||
schema: "public",
|
||||
binding: { transport: "postgres_direct", host: "two.internal", port: 5432, username: "reader" },
|
||||
});
|
||||
await repository.applySchemaSync(firstDatabase.id, firstDatabase.version, "all", [], {
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||
tables: [{ name: "patients", sourceComment: "Clinical patients" }],
|
||||
columns: [{
|
||||
tableName: "patients",
|
||||
name: "birth_date",
|
||||
ordinalPosition: 1,
|
||||
dataType: "date",
|
||||
isNullable: true,
|
||||
defaultExpression: null,
|
||||
primaryKeyPosition: null,
|
||||
sourceComment: "Patient date of birth",
|
||||
}],
|
||||
relationships: [],
|
||||
});
|
||||
const table = (await repository.listTables(firstDatabase.id))[0]!;
|
||||
const column = (await repository.listColumns(firstDatabase.id, table.id))[0]!;
|
||||
|
||||
const run = await repository.createDescriptionGenerationRun(
|
||||
firstDatabase.id,
|
||||
"selected_columns",
|
||||
"openai-mini",
|
||||
"it",
|
||||
1,
|
||||
);
|
||||
expect(run).toMatchObject({
|
||||
databaseId: firstDatabase.id,
|
||||
scope: "selected_columns",
|
||||
modelId: "openai-mini",
|
||||
language: "it",
|
||||
status: "queued",
|
||||
total: 1,
|
||||
processed: 0,
|
||||
generated: 0,
|
||||
nonGeneratable: 0,
|
||||
failed: 0,
|
||||
startedAt: null,
|
||||
finishedAt: null,
|
||||
errorSummary: null,
|
||||
});
|
||||
await expect(repository.createDescriptionGenerationRun(
|
||||
secondDatabase.id,
|
||||
"selected_columns",
|
||||
"openai-mini",
|
||||
"en",
|
||||
1,
|
||||
)).rejects.toThrow("A description generation run is already active");
|
||||
await repository.updateDescriptionGenerationRun(run.id, {
|
||||
status: "running",
|
||||
startedAt: new Date().toISOString(),
|
||||
});
|
||||
await expect(repository.createDescriptionGenerationRun(
|
||||
secondDatabase.id,
|
||||
"selected_columns",
|
||||
"openai-mini",
|
||||
"en",
|
||||
1,
|
||||
)).rejects.toThrow("A description generation run is already active");
|
||||
|
||||
await repository.appendDescriptionGenerationEvent(run.id, "info", "Description generation queued.");
|
||||
await repository.appendDescriptionGenerationEvent(run.id, "info", "Description generation started.");
|
||||
expect(await repository.listDescriptionGenerationEvents(run.id, 1)).toEqual([
|
||||
expect.objectContaining({
|
||||
runId: run.id,
|
||||
sequence: 2,
|
||||
level: "info",
|
||||
message: "Description generation started.",
|
||||
createdAt: expect.any(String),
|
||||
}),
|
||||
]);
|
||||
|
||||
const updatedColumn = await repository.updateColumnMetadata(
|
||||
firstDatabase.id,
|
||||
table.id,
|
||||
column.id,
|
||||
column.version,
|
||||
column.description,
|
||||
"Data di nascita del paziente.",
|
||||
);
|
||||
expect(updatedColumn).toMatchObject({
|
||||
generatedDescription: "Data di nascita del paziente.",
|
||||
version: column.version + 1,
|
||||
});
|
||||
expect(await repository.updateDescriptionGenerationRun(run.id, {
|
||||
status: "completed",
|
||||
processed: 1,
|
||||
generated: 1,
|
||||
startedAt: new Date().toISOString(),
|
||||
finishedAt: new Date().toISOString(),
|
||||
})).toMatchObject({
|
||||
status: "completed",
|
||||
processed: 1,
|
||||
generated: 1,
|
||||
});
|
||||
|
||||
const next = await repository.createDescriptionGenerationRun(
|
||||
secondDatabase.id,
|
||||
"missing",
|
||||
"openai-mini",
|
||||
"en",
|
||||
1,
|
||||
);
|
||||
expect(await repository.getDescriptionGenerationRun(next.id)).toMatchObject({
|
||||
scope: "missing",
|
||||
total: 1,
|
||||
});
|
||||
await repository.updateDescriptionGenerationRun(next.id, {
|
||||
status: "running",
|
||||
startedAt: new Date().toISOString(),
|
||||
});
|
||||
expect(await repository.updateDescriptionGenerationRun(next.id, {
|
||||
status: "failed",
|
||||
processed: 1,
|
||||
failed: 1,
|
||||
finishedAt: new Date().toISOString(),
|
||||
errorSummary: "The model provider request failed.",
|
||||
})).toMatchObject({
|
||||
status: "failed",
|
||||
processed: 1,
|
||||
failed: 1,
|
||||
errorSummary: "The model provider request failed.",
|
||||
});
|
||||
|
||||
const allRun = await repository.createDescriptionGenerationRun(
|
||||
firstDatabase.id,
|
||||
"all",
|
||||
"openai-mini",
|
||||
"it",
|
||||
2,
|
||||
);
|
||||
expect(await repository.getDescriptionGenerationRun(allRun.id)).toMatchObject({
|
||||
scope: "all",
|
||||
total: 2,
|
||||
});
|
||||
expect(await repository.getActiveDescriptionGenerationRun()).toMatchObject({ id: allRun.id });
|
||||
expect((await repository.listDescriptionGenerationRuns(2)).map((candidate) => candidate.id)).toEqual([
|
||||
allRun.id,
|
||||
next.id,
|
||||
]);
|
||||
|
||||
expect(await repository.interruptActiveDescriptionGenerationRuns(
|
||||
"Description generation was interrupted by backend restart.",
|
||||
)).toEqual([
|
||||
expect.objectContaining({
|
||||
id: allRun.id,
|
||||
status: "interrupted",
|
||||
finishedAt: expect.any(String),
|
||||
errorSummary: "Description generation was interrupted by backend restart.",
|
||||
}),
|
||||
]);
|
||||
expect(await repository.getActiveDescriptionGenerationRun()).toBeUndefined();
|
||||
} finally {
|
||||
await db.destroy();
|
||||
await container.stop();
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
Reference in New Issue
Block a user