686 lines
29 KiB
TypeScript
686 lines
29 KiB
TypeScript
import { spawnSync } from "node:child_process";
|
|
import { PostgreSqlContainer } from "@testcontainers/postgresql";
|
|
import { CamelCasePlugin, Kysely, PostgresDialect, sql } from "kysely";
|
|
import { Pool } from "pg";
|
|
import { expect, test } from "vitest";
|
|
import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js";
|
|
import type { ObservedSchemaSnapshot } from "../src/catalog/types.js";
|
|
import { up as upDatabases } from "../src/catalog/migrations/001_workspace_databases.js";
|
|
import { up as upTables } from "../src/catalog/migrations/002_catalog_tables.js";
|
|
import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema_sync.js";
|
|
import { up as upRuntimeSequencePrivileges } from "../src/catalog/migrations/004_catalog_runtime_sequence_privileges.js";
|
|
import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js";
|
|
import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js";
|
|
import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js";
|
|
import { up as upLogicalRelationships } from "../src/catalog/migrations/008_catalog_logical_relationships.js";
|
|
import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js";
|
|
import { up as upCanonicalModelIds } from "../src/catalog/migrations/010_canonical_model_ids.js";
|
|
|
|
const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
|
|
|
|
test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per workspace and optimistic updates", async () => {
|
|
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
|
|
const db = new Kysely<CatalogDatabase>({
|
|
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
|
|
plugins: [new CamelCasePlugin()],
|
|
});
|
|
try {
|
|
await upDatabases(db);
|
|
await upTables(db);
|
|
await upSchemaSync(db);
|
|
await upSensitiveDataFlag(db);
|
|
await upLogicalRelationships(db);
|
|
await upDescriptionGeneration(db);
|
|
await upSensitiveSuggestionRuns(db);
|
|
await upAiTokenUsage(db);
|
|
await upCanonicalModelIds(db);
|
|
await sql`CREATE ROLE thothii_catalog_runtime`.execute(db);
|
|
await upRuntimeSequencePrivileges(db);
|
|
const sequencePrivilege = await sql<{ allowed: boolean }>`
|
|
SELECT has_sequence_privilege(
|
|
'thothii_catalog_runtime',
|
|
'catalog_sync_events_id_seq',
|
|
'USAGE'
|
|
) AS allowed
|
|
`.execute(db);
|
|
expect(sequencePrivilege.rows[0]?.allowed).toBe(true);
|
|
const repository = new KyselyCatalogRepository(db);
|
|
const input = {
|
|
workspaceId: "psd-clinical",
|
|
engine: "postgres" as const,
|
|
databaseName: "warehouse",
|
|
schema: "datawarehouse",
|
|
binding: { transport: "rest_api" as const, baseUrl: "https://psd.example/api", restPath: "/health", restAuth: "bearer" as const },
|
|
};
|
|
const created = await repository.create(input);
|
|
expect(created).toMatchObject({ version: 1, connectionStatus: "untested", binding: { transport: "rest_api" } });
|
|
await expect(repository.create(input)).rejects.toThrow("Workspace database already exists");
|
|
expect(await repository.update(created.id, 99, input)).toBeUndefined();
|
|
const synchronized = await repository.reconcileTables(created.id, 1, [
|
|
{ name: "patients", sourceComment: "Clinical patients" },
|
|
{ name: "visits", sourceComment: null },
|
|
], []);
|
|
expect(synchronized).toMatchObject({ kind: "applied", createdCount: 2, deletedCount: 0 });
|
|
const patients = (await repository.listTables(created.id))[0];
|
|
expect(await repository.updateTableDescription(
|
|
created.id,
|
|
patients.id,
|
|
patients.version,
|
|
"Curated patients",
|
|
)).toMatchObject({ description: "Curated patients", sourceComment: "Clinical patients", version: 2 });
|
|
const visits = (await repository.listTables(created.id)).find((table) => table.name === "visits")!;
|
|
const fullColumnsSnapshot: ObservedSchemaSnapshot = {
|
|
schemaVersion: 1,
|
|
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
|
tables: [
|
|
{ name: "patients", sourceComment: "Clinical patients" },
|
|
{ name: "visits", sourceComment: null },
|
|
],
|
|
columns: [
|
|
{ tableName: "patients", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
|
{ tableName: "patients", name: "name", ordinalPosition: 2, dataType: "text", isNullable: true, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
|
{ tableName: "visits", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
|
{ tableName: "visits", name: "patient_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
|
],
|
|
relationships: [],
|
|
};
|
|
expect(await repository.applySchemaSync(created.id, 1, "columns", [], fullColumnsSnapshot))
|
|
.toMatchObject({ created: 4, deleted: 0 });
|
|
expect((await repository.listColumns(created.id, patients.id)).map((column) => ({
|
|
name: column.name,
|
|
sensitive: column.sensitive,
|
|
}))).toEqual([
|
|
{ name: "id", sensitive: false },
|
|
{ name: "name", sensitive: false },
|
|
]);
|
|
expect((await repository.listColumns(created.id, visits.id)).map((column) => column.name))
|
|
.toEqual(["id", "patient_id"]);
|
|
|
|
const patientName = (await repository.listColumns(created.id, patients.id))
|
|
.find((column) => column.name === "name")!;
|
|
expect(await repository.updateColumnMetadata(
|
|
created.id,
|
|
patients.id,
|
|
patientName.id,
|
|
patientName.version,
|
|
patientName.description,
|
|
patientName.generatedDescription,
|
|
true,
|
|
)).toMatchObject({ sensitive: true });
|
|
expect(await repository.getCatalogMetrics(created.id)).toEqual({
|
|
scope: "database",
|
|
databaseId: created.id,
|
|
tables: 2,
|
|
columns: 4,
|
|
sensitiveColumns: 1,
|
|
relationships: 0,
|
|
descriptionTargets: 6,
|
|
describedTargets: 1,
|
|
descriptionCoverage: 17,
|
|
updatedAt: expect.any(String),
|
|
});
|
|
expect(await repository.getCatalogMetrics()).toEqual({
|
|
scope: "global",
|
|
databaseId: null,
|
|
tables: 2,
|
|
columns: 4,
|
|
sensitiveColumns: 1,
|
|
relationships: 0,
|
|
descriptionTargets: 6,
|
|
describedTargets: 1,
|
|
descriptionCoverage: 17,
|
|
updatedAt: expect.any(String),
|
|
});
|
|
expect(await repository.getCatalogMetrics(
|
|
"99999999-9999-4999-8999-999999999999",
|
|
)).toBeUndefined();
|
|
const refreshedColumnsSnapshot: ObservedSchemaSnapshot = {
|
|
...fullColumnsSnapshot,
|
|
schemaVersion: 2,
|
|
columns: fullColumnsSnapshot.columns.map((column) => column.tableName === "patients"
|
|
&& column.name === "name"
|
|
? { ...column, sourceComment: "Sensitive patient name" }
|
|
: column),
|
|
};
|
|
expect(await repository.applySchemaSync(
|
|
created.id,
|
|
1,
|
|
"columns",
|
|
[patients.id],
|
|
refreshedColumnsSnapshot,
|
|
)).toMatchObject({ updated: 1 });
|
|
expect(await repository.getColumn(created.id, patients.id, patientName.id)).toMatchObject({
|
|
sensitive: true,
|
|
sourceComment: "Sensitive patient name",
|
|
});
|
|
|
|
const reducedColumnsSnapshot: ObservedSchemaSnapshot = {
|
|
...fullColumnsSnapshot,
|
|
columns: fullColumnsSnapshot.columns.filter((column) => column.name === "id"),
|
|
};
|
|
expect(await repository.planSchemaSync(created.id, "columns", [], reducedColumnsSnapshot)).toMatchObject({
|
|
deletedColumns: [
|
|
{ tableName: "patients", columnName: "name" },
|
|
{ tableName: "visits", columnName: "patient_id" },
|
|
],
|
|
});
|
|
expect(await repository.planSchemaSync(created.id, "columns", [patients.id], reducedColumnsSnapshot)).toMatchObject({
|
|
deletedColumns: [{ tableName: "patients", columnName: "name" }],
|
|
});
|
|
expect(await repository.applySchemaSync(created.id, 1, "columns", [patients.id], reducedColumnsSnapshot))
|
|
.toMatchObject({ deleted: 1 });
|
|
expect((await repository.listColumns(created.id, patients.id)).map((column) => column.name))
|
|
.toEqual(["id"]);
|
|
expect((await repository.listColumns(created.id, visits.id)).map((column) => column.name))
|
|
.toEqual(["id", "patient_id"]);
|
|
expect(await repository.reconcileTables(created.id, 1, [
|
|
{ name: "patients", sourceComment: "Updated physical comment" },
|
|
], [])).toEqual({ kind: "confirmation_required", deletedNames: ["visits"] });
|
|
expect(await repository.reconcileTables(created.id, 1, [
|
|
{ name: "patients", sourceComment: "Updated physical comment" },
|
|
], ["visits"])).toMatchObject({ kind: "applied", updatedCount: 1, deletedCount: 1 });
|
|
const syncRun = await repository.createSyncRun(created.id, "columns", [patients.id], 1);
|
|
expect(syncRun).toMatchObject({
|
|
databaseId: created.id,
|
|
scope: "columns",
|
|
tableIds: [patients.id],
|
|
state: "queued",
|
|
});
|
|
expect(await repository.listSyncRuns(created.id)).toEqual([
|
|
expect.objectContaining({ id: syncRun.id, tableIds: [patients.id] }),
|
|
]);
|
|
expect(await repository.update(created.id, 1, { ...input, schema: "public" })).toMatchObject({ version: 2, schema: "public" });
|
|
expect(await repository.delete(created.id, 2)).toBe(true);
|
|
expect(await repository.list()).toEqual([]);
|
|
expect(await repository.listTables(created.id)).toEqual([]);
|
|
} finally {
|
|
await db.destroy();
|
|
await container.stop();
|
|
}
|
|
}, 60_000);
|
|
|
|
test.skipIf(!dockerAvailable)("PostgreSQL repository performs scoped metadata cleanup with catalog cascades", async () => {
|
|
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
|
|
const db = new Kysely<CatalogDatabase>({
|
|
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
|
|
plugins: [new CamelCasePlugin()],
|
|
});
|
|
try {
|
|
await upDatabases(db);
|
|
await upTables(db);
|
|
await upSchemaSync(db);
|
|
await upSensitiveDataFlag(db);
|
|
await upLogicalRelationships(db);
|
|
const repository = new KyselyCatalogRepository(db);
|
|
const database = await repository.create({
|
|
workspaceId: "cleanup-test",
|
|
engine: "postgres",
|
|
databaseName: "warehouse",
|
|
schema: "public",
|
|
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
|
});
|
|
const snapshot: ObservedSchemaSnapshot = {
|
|
schemaVersion: 1,
|
|
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
|
tables: [
|
|
{ name: "patients", sourceComment: null },
|
|
{ name: "visits", sourceComment: null },
|
|
],
|
|
columns: [
|
|
{ tableName: "patients", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
|
{ tableName: "patients", name: "name", ordinalPosition: 2, dataType: "text", isNullable: true, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
|
{ tableName: "visits", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
|
{ tableName: "visits", name: "patient_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
|
],
|
|
relationships: [{
|
|
constraintName: "visits_patient_id_fkey",
|
|
sourceTableName: "visits",
|
|
targetTableName: "patients",
|
|
updateRule: "NO ACTION",
|
|
deleteRule: "CASCADE",
|
|
deferrable: false,
|
|
initiallyDeferred: false,
|
|
columns: [{ position: 1, sourceColumnName: "patient_id", targetColumnName: "id" }],
|
|
}],
|
|
};
|
|
await repository.applySchemaSync(database.id, database.version, "all", [], snapshot);
|
|
const patients = (await repository.listTables(database.id)).find((table) => table.name === "patients")!;
|
|
|
|
expect(await repository.deleteTableMetadata(database.id, [patients.id], "relationships"))
|
|
.toEqual({ tables: 0, columns: 0, relationships: 1 });
|
|
expect(await repository.listRelationships(database.id)).toEqual([]);
|
|
expect(await repository.listColumns(database.id, patients.id)).toHaveLength(2);
|
|
expect((await repository.get(database.id))?.schemaSyncedVersion).toBeUndefined();
|
|
|
|
await repository.applySchemaSync(database.id, database.version, "relationships", [], snapshot);
|
|
expect(await repository.deleteTableMetadata(database.id, [patients.id], "columns"))
|
|
.toEqual({ tables: 0, columns: 2, relationships: 0 });
|
|
expect(await repository.listColumns(database.id, patients.id)).toEqual([]);
|
|
expect(await repository.listRelationships(database.id)).toMatchObject([
|
|
{ constraintName: "visits_patient_id_fkey", columns: [] },
|
|
]);
|
|
|
|
await repository.applySchemaSync(database.id, database.version, "columns", [patients.id], snapshot);
|
|
await repository.applySchemaSync(database.id, database.version, "relationships", [], snapshot);
|
|
expect(await repository.deleteDatabaseMetadata([
|
|
database.id,
|
|
"99999999-9999-4999-8999-999999999999",
|
|
], "tables")).toBeUndefined();
|
|
expect(await repository.listTables(database.id)).toHaveLength(2);
|
|
expect(await repository.deleteDatabaseMetadata([database.id], "tables"))
|
|
.toEqual({ tables: 2, columns: 4, relationships: 1 });
|
|
expect(await repository.get(database.id)).toBeDefined();
|
|
expect(await repository.listTables(database.id)).toEqual([]);
|
|
expect(await repository.listRelationships(database.id)).toEqual([]);
|
|
} finally {
|
|
await db.destroy();
|
|
await container.stop();
|
|
}
|
|
}, 60_000);
|
|
|
|
test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates selected table and column descriptions", async () => {
|
|
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
|
|
const db = new Kysely<CatalogDatabase>({
|
|
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
|
|
plugins: [new CamelCasePlugin()],
|
|
});
|
|
try {
|
|
await upDatabases(db);
|
|
await upTables(db);
|
|
await upSchemaSync(db);
|
|
await upSensitiveDataFlag(db);
|
|
await upLogicalRelationships(db);
|
|
const repository = new KyselyCatalogRepository(db);
|
|
const database = await repository.create({
|
|
workspaceId: "consolidation-test",
|
|
engine: "postgres",
|
|
databaseName: "warehouse",
|
|
schema: "public",
|
|
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
|
});
|
|
const snapshot: ObservedSchemaSnapshot = {
|
|
schemaVersion: 1,
|
|
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
|
tables: [
|
|
{ name: "patients", sourceComment: null },
|
|
{ name: "visits", sourceComment: null },
|
|
],
|
|
columns: [
|
|
{ tableName: "visits", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
|
{ tableName: "visits", name: "patient_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
|
],
|
|
relationships: [],
|
|
};
|
|
await repository.applySchemaSync(database.id, database.version, "all", [], snapshot);
|
|
const tables = await repository.listTables(database.id);
|
|
const patients = tables.find((table) => table.name === "patients")!;
|
|
const visits = tables.find((table) => table.name === "visits")!;
|
|
await repository.updateTableMetadata(
|
|
database.id, patients.id, patients.version, "Old patients", "Generated patients",
|
|
);
|
|
await repository.updateTableMetadata(
|
|
database.id, visits.id, visits.version, "Keep visits", " ",
|
|
);
|
|
|
|
expect(await repository.consolidateGeneratedDescriptions(
|
|
database.id, "tables", [patients.id, visits.id],
|
|
)).toEqual({ copied: 1, skipped: 1 });
|
|
expect(await repository.getTable(database.id, patients.id)).toMatchObject({
|
|
description: "Generated patients",
|
|
generatedDescription: "Generated patients",
|
|
version: patients.version + 2,
|
|
});
|
|
expect(await repository.getTable(database.id, visits.id)).toMatchObject({
|
|
description: "Keep visits",
|
|
generatedDescription: " ",
|
|
version: visits.version + 1,
|
|
});
|
|
|
|
const columns = await repository.listColumns(database.id, visits.id);
|
|
const id = columns.find((column) => column.name === "id")!;
|
|
const patientId = columns.find((column) => column.name === "patient_id")!;
|
|
await repository.updateColumnMetadata(
|
|
database.id, visits.id, id.id, id.version, "Old id", "Generated id",
|
|
);
|
|
await repository.updateColumnMetadata(
|
|
database.id, visits.id, patientId.id, patientId.version, "Keep patient reference", null,
|
|
);
|
|
|
|
expect(await repository.consolidateGeneratedDescriptions(
|
|
database.id, "columns", [id.id, patientId.id],
|
|
)).toEqual({ copied: 1, skipped: 1 });
|
|
expect(await repository.getColumn(database.id, visits.id, id.id)).toMatchObject({
|
|
description: "Generated id",
|
|
generatedDescription: "Generated id",
|
|
version: id.version + 2,
|
|
});
|
|
expect(await repository.getColumn(database.id, visits.id, patientId.id)).toMatchObject({
|
|
description: "Keep patient reference",
|
|
generatedDescription: null,
|
|
version: patientId.version + 1,
|
|
});
|
|
|
|
const currentVisits = (await repository.getTable(database.id, visits.id))!;
|
|
await repository.updateTableMetadata(
|
|
database.id, visits.id, currentVisits.version, "Still curated visits", "Generated visits",
|
|
);
|
|
expect(await repository.consolidateGeneratedDescriptions(database.id, "tables", [
|
|
visits.id,
|
|
"99999999-9999-4999-8999-999999999999",
|
|
])).toBeUndefined();
|
|
expect(await repository.getTable(database.id, visits.id)).toMatchObject({
|
|
description: "Still curated visits",
|
|
generatedDescription: "Generated visits",
|
|
});
|
|
} finally {
|
|
await db.destroy();
|
|
await container.stop();
|
|
}
|
|
}, 60_000);
|
|
|
|
test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and sensitive suggestion run histories", async () => {
|
|
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
|
|
const db = new Kysely<CatalogDatabase>({
|
|
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
|
|
plugins: [new CamelCasePlugin()],
|
|
});
|
|
try {
|
|
await upDatabases(db);
|
|
await upTables(db);
|
|
await upSchemaSync(db);
|
|
await upSensitiveDataFlag(db);
|
|
await upLogicalRelationships(db);
|
|
await upDescriptionGeneration(db);
|
|
await upSensitiveSuggestionRuns(db);
|
|
await upAiTokenUsage(db);
|
|
await upCanonicalModelIds(db);
|
|
const repository = new KyselyCatalogRepository(db);
|
|
const firstDatabase = await repository.create({
|
|
workspaceId: "generation-one",
|
|
engine: "postgres",
|
|
databaseName: "warehouse_one",
|
|
schema: "public",
|
|
binding: { transport: "postgres_direct", host: "one.internal", port: 5432, username: "reader" },
|
|
});
|
|
const secondDatabase = await repository.create({
|
|
workspaceId: "generation-two",
|
|
engine: "postgres",
|
|
databaseName: "warehouse_two",
|
|
schema: "public",
|
|
binding: { transport: "postgres_direct", host: "two.internal", port: 5432, username: "reader" },
|
|
});
|
|
await repository.applySchemaSync(firstDatabase.id, firstDatabase.version, "all", [], {
|
|
schemaVersion: 1,
|
|
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
|
tables: [{ name: "patients", sourceComment: "Clinical patients" }],
|
|
columns: [{
|
|
tableName: "patients",
|
|
name: "birth_date",
|
|
ordinalPosition: 1,
|
|
dataType: "date",
|
|
isNullable: true,
|
|
defaultExpression: null,
|
|
primaryKeyPosition: null,
|
|
sourceComment: "Patient date of birth",
|
|
}],
|
|
relationships: [],
|
|
});
|
|
const table = (await repository.listTables(firstDatabase.id))[0]!;
|
|
const column = (await repository.listColumns(firstDatabase.id, table.id))[0]!;
|
|
|
|
const run = await repository.createDescriptionGenerationRun(
|
|
firstDatabase.id,
|
|
"selected_columns",
|
|
"openai/gpt-4.1-mini",
|
|
"it",
|
|
1,
|
|
);
|
|
expect(run).toMatchObject({
|
|
databaseId: firstDatabase.id,
|
|
scope: "selected_columns",
|
|
modelId: "openai/gpt-4.1-mini",
|
|
language: "it",
|
|
status: "queued",
|
|
total: 1,
|
|
processed: 0,
|
|
generated: 0,
|
|
nonGeneratable: 0,
|
|
failed: 0,
|
|
startedAt: null,
|
|
finishedAt: null,
|
|
errorSummary: null,
|
|
});
|
|
await expect(repository.createDescriptionGenerationRun(
|
|
secondDatabase.id,
|
|
"selected_columns",
|
|
"openai/gpt-4.1-mini",
|
|
"en",
|
|
1,
|
|
)).rejects.toThrow("A description generation run is already active");
|
|
await repository.updateDescriptionGenerationRun(run.id, {
|
|
status: "running",
|
|
startedAt: new Date().toISOString(),
|
|
});
|
|
await expect(repository.createDescriptionGenerationRun(
|
|
secondDatabase.id,
|
|
"selected_columns",
|
|
"openai/gpt-4.1-mini",
|
|
"en",
|
|
1,
|
|
)).rejects.toThrow("A description generation run is already active");
|
|
|
|
await repository.appendDescriptionGenerationEvent(run.id, "info", "Description generation queued.");
|
|
await repository.appendDescriptionGenerationEvent(run.id, "info", "Description generation started.");
|
|
expect(await repository.listDescriptionGenerationEvents(run.id, 1)).toEqual([
|
|
expect.objectContaining({
|
|
runId: run.id,
|
|
sequence: 2,
|
|
level: "info",
|
|
message: "Description generation started.",
|
|
createdAt: expect.any(String),
|
|
}),
|
|
]);
|
|
|
|
const updatedColumn = await repository.updateColumnMetadata(
|
|
firstDatabase.id,
|
|
table.id,
|
|
column.id,
|
|
column.version,
|
|
column.description,
|
|
"Data di nascita del paziente.",
|
|
);
|
|
expect(updatedColumn).toMatchObject({
|
|
generatedDescription: "Data di nascita del paziente.",
|
|
version: column.version + 1,
|
|
});
|
|
expect(await repository.updateDescriptionGenerationRun(run.id, {
|
|
status: "completed",
|
|
processed: 1,
|
|
generated: 1,
|
|
startedAt: new Date().toISOString(),
|
|
finishedAt: new Date().toISOString(),
|
|
})).toMatchObject({
|
|
status: "completed",
|
|
processed: 1,
|
|
generated: 1,
|
|
});
|
|
|
|
const next = await repository.createDescriptionGenerationRun(
|
|
secondDatabase.id,
|
|
"missing",
|
|
"openai/gpt-4.1-mini",
|
|
"en",
|
|
1,
|
|
);
|
|
expect(await repository.getDescriptionGenerationRun(next.id)).toMatchObject({
|
|
scope: "missing",
|
|
total: 1,
|
|
});
|
|
await repository.updateDescriptionGenerationRun(next.id, {
|
|
status: "running",
|
|
startedAt: new Date().toISOString(),
|
|
});
|
|
expect(await repository.updateDescriptionGenerationRun(next.id, {
|
|
status: "failed",
|
|
processed: 1,
|
|
failed: 1,
|
|
finishedAt: new Date().toISOString(),
|
|
errorSummary: "The model provider request failed.",
|
|
})).toMatchObject({
|
|
status: "failed",
|
|
processed: 1,
|
|
failed: 1,
|
|
errorSummary: "The model provider request failed.",
|
|
});
|
|
|
|
const allRun = await repository.createDescriptionGenerationRun(
|
|
firstDatabase.id,
|
|
"all",
|
|
"openai/gpt-4.1-mini",
|
|
"it",
|
|
2,
|
|
);
|
|
expect(await repository.getDescriptionGenerationRun(allRun.id)).toMatchObject({
|
|
scope: "all",
|
|
total: 2,
|
|
});
|
|
expect(await repository.getActiveDescriptionGenerationRun()).toMatchObject({ id: allRun.id });
|
|
expect((await repository.listDescriptionGenerationRuns(2)).map((candidate) => candidate.id)).toEqual([
|
|
allRun.id,
|
|
next.id,
|
|
]);
|
|
|
|
expect(await repository.interruptActiveDescriptionGenerationRuns(
|
|
"Description generation was interrupted by backend restart.",
|
|
)).toEqual([
|
|
expect.objectContaining({
|
|
id: allRun.id,
|
|
status: "interrupted",
|
|
finishedAt: expect.any(String),
|
|
errorSummary: "Description generation was interrupted by backend restart.",
|
|
}),
|
|
]);
|
|
expect(await repository.getActiveDescriptionGenerationRun()).toBeUndefined();
|
|
|
|
const suggestionRun = await repository.createSensitiveDataSuggestionRun(
|
|
firstDatabase.id,
|
|
"selected_columns",
|
|
"openai/gpt-4.1-mini",
|
|
);
|
|
expect(suggestionRun).toMatchObject({
|
|
databaseId: firstDatabase.id,
|
|
status: "running",
|
|
total: 0,
|
|
suggestedSensitive: 0,
|
|
suggestedNonSensitive: 0,
|
|
startedAt: expect.any(String),
|
|
});
|
|
await repository.appendSensitiveDataSuggestionEvent(
|
|
suggestionRun.id,
|
|
"info",
|
|
"Sensitive-field suggestion generation started.",
|
|
);
|
|
await repository.appendSensitiveDataSuggestionEvent(
|
|
suggestionRun.id,
|
|
"info",
|
|
"Sensitive-field suggestion generation completed for 2 columns.",
|
|
);
|
|
expect(await repository.updateSensitiveDataSuggestionRun(suggestionRun.id, {
|
|
status: "completed",
|
|
total: 2,
|
|
suggestedSensitive: 1,
|
|
suggestedNonSensitive: 1,
|
|
finishedAt: new Date().toISOString(),
|
|
})).toMatchObject({
|
|
status: "completed",
|
|
total: 2,
|
|
suggestedSensitive: 1,
|
|
suggestedNonSensitive: 1,
|
|
});
|
|
expect(await repository.listSensitiveDataSuggestionEvents(suggestionRun.id, 1)).toEqual([
|
|
expect.objectContaining({ sequence: 2, level: "info" }),
|
|
]);
|
|
expect((await repository.listSensitiveDataSuggestionRuns(1))[0]).toMatchObject({
|
|
id: suggestionRun.id,
|
|
});
|
|
|
|
const interruptedSuggestionRun = await repository.createSensitiveDataSuggestionRun(
|
|
secondDatabase.id,
|
|
"all",
|
|
"openai/gpt-4.1-mini",
|
|
);
|
|
expect(await repository.interruptActiveSensitiveDataSuggestionRuns(
|
|
"Sensitive-field suggestion generation was interrupted by backend restart.",
|
|
)).toEqual([
|
|
expect.objectContaining({
|
|
id: interruptedSuggestionRun.id,
|
|
status: "interrupted",
|
|
finishedAt: expect.any(String),
|
|
}),
|
|
]);
|
|
} finally {
|
|
await db.destroy();
|
|
await container.stop();
|
|
}
|
|
}, 60_000);
|
|
|
|
test.skipIf(!dockerAvailable)("PostgreSQL repository persists logical relationship lifecycle and tombstones", async () => {
|
|
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
|
|
const db = new Kysely<CatalogDatabase>({
|
|
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
|
|
plugins: [new CamelCasePlugin()],
|
|
});
|
|
try {
|
|
await upDatabases(db);
|
|
await upTables(db);
|
|
await upSchemaSync(db);
|
|
await upSensitiveDataFlag(db);
|
|
await upLogicalRelationships(db);
|
|
const repository = new KyselyCatalogRepository(db);
|
|
const database = await repository.create({
|
|
workspaceId: "logical-relationships",
|
|
engine: "postgres",
|
|
databaseName: "warehouse",
|
|
schema: "public",
|
|
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
|
});
|
|
await repository.applySchemaSync(database.id, database.version, "all", [], {
|
|
schemaVersion: 1,
|
|
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
|
tables: [{ name: "users", sourceComment: null }, { name: "orders", sourceComment: null }],
|
|
columns: [
|
|
{ tableName: "users", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
|
{ tableName: "orders", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
|
{ tableName: "orders", name: "user_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
|
],
|
|
relationships: [],
|
|
});
|
|
const context = (await repository.getLogicalRelationshipContext(database.id))!;
|
|
const source = context.endpoints.find((item) => item.tableName === "orders" && item.columnName === "user_id")!;
|
|
const target = context.endpoints.find((item) => item.tableName === "users" && item.columnName === "id")!;
|
|
const created = await repository.insertLogicalRelationship(database.id, source.columnId, target.columnId, false);
|
|
expect(created).toMatchObject({ origin: "manual", status: "active" });
|
|
await expect(repository.insertLogicalRelationship(database.id, source.columnId, target.columnId, true))
|
|
.resolves.toBeUndefined();
|
|
|
|
expect(await repository.setLogicalRelationshipStatus(database.id, created!.id, "excluded"))
|
|
.toMatchObject({ status: "excluded" });
|
|
await expect(repository.insertGeneratedLogicalRelationships(database.id, [{
|
|
sourceColumnId: source.columnId, targetColumnId: target.columnId,
|
|
}])).resolves.toBe(0);
|
|
|
|
await expect(repository.deleteLogicalRelationship(database.id, created!.id)).resolves.toBe(true);
|
|
await expect(repository.insertGeneratedLogicalRelationships(database.id, [{
|
|
sourceColumnId: source.columnId, targetColumnId: target.columnId,
|
|
}])).resolves.toBe(1);
|
|
expect(await repository.listLogicalRelationships(database.id))
|
|
.toMatchObject([{ origin: "generated", status: "active" }]);
|
|
|
|
await db.deleteFrom("catalogColumns").where("id", "=", source.columnId).execute();
|
|
expect(await repository.listLogicalRelationships(database.id)).toEqual([]);
|
|
} finally {
|
|
await db.destroy();
|
|
await container.stop();
|
|
}
|
|
}, 60_000);
|