Codex
82e2c91f42
Publish documentation / publish (push) Successful in 1m27s
Add PostgreSQL-backed memory, editable evidence with source review and activation, and human-approved archive repairs across the harness, API, and UI. Include migrations, deployment support, regression coverage, and validation documentation. Refresh permissions from validated session roles so existing administrator logins can access newly deployed archive management features.
873 lines
38 KiB
TypeScript
873 lines
38 KiB
TypeScript
import { spawnSync } from "node:child_process";
|
|
import { randomUUID } from "node:crypto";
|
|
import { PostgreSqlContainer } from "@testcontainers/postgresql";
|
|
import { CamelCasePlugin, Kysely, PostgresDialect, sql } from "kysely";
|
|
import { Pool } from "pg";
|
|
import { expect, test } from "vitest";
|
|
import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js";
|
|
import type { ObservedSchemaSnapshot } from "../src/catalog/types.js";
|
|
import { up as upDatabases } from "../src/catalog/migrations/001_workspace_databases.js";
|
|
import { up as upTables } from "../src/catalog/migrations/002_catalog_tables.js";
|
|
import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema_sync.js";
|
|
import { up as upRuntimeSequencePrivileges } from "../src/catalog/migrations/004_catalog_runtime_sequence_privileges.js";
|
|
import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js";
|
|
import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js";
|
|
import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js";
|
|
import { up as upLogicalRelationships } from "../src/catalog/migrations/008_catalog_logical_relationships.js";
|
|
import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js";
|
|
import { up as upCanonicalModelIds } from "../src/catalog/migrations/010_canonical_model_ids.js";
|
|
import { up as upLocalSensitivityAnalysis } from "../src/catalog/migrations/011_local_sensitivity_analysis.js";
|
|
import { up as upSensitivityReason } from "../src/catalog/migrations/012_sensitivity_reason.js";
|
|
import { up as upCatalogPreprocessingState } from "../src/catalog/migrations/013_catalog_preprocessing_state.js";
|
|
|
|
const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
|
|
|
|
test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per workspace and optimistic updates", async () => {
|
|
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
|
|
const db = new Kysely<CatalogDatabase>({
|
|
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
|
|
plugins: [new CamelCasePlugin()],
|
|
});
|
|
try {
|
|
await upDatabases(db);
|
|
await upTables(db);
|
|
await upSchemaSync(db);
|
|
await upSensitiveDataFlag(db);
|
|
await upLogicalRelationships(db);
|
|
await upDescriptionGeneration(db);
|
|
await upSensitiveSuggestionRuns(db);
|
|
await upAiTokenUsage(db);
|
|
const historicalDatabaseId = randomUUID();
|
|
await db.insertInto("workspaceDatabases").values({
|
|
id: historicalDatabaseId,
|
|
workspaceId: "migration-history",
|
|
engine: "postgres",
|
|
databaseName: "warehouse",
|
|
schemaName: "public",
|
|
}).execute();
|
|
await db.insertInto("descriptionGenerationRuns").values({
|
|
id: randomUUID(), databaseId: historicalDatabaseId, scope: "all",
|
|
modelId: "openai-mini", language: "en", status: "completed", total: 1,
|
|
processed: 1, generated: 1,
|
|
}).execute();
|
|
const historicalSuggestionRunId = randomUUID();
|
|
await db.insertInto("sensitiveDataSuggestionRuns").values({
|
|
id: historicalSuggestionRunId, databaseId: historicalDatabaseId, scope: "all",
|
|
modelId: "openai-mini", status: "completed", total: 1,
|
|
suggestedSensitive: 1,
|
|
}).execute();
|
|
await upCanonicalModelIds(db);
|
|
await upLocalSensitivityAnalysis(db);
|
|
await upSensitivityReason(db);
|
|
await upCatalogPreprocessingState(db);
|
|
await expect(db.selectFrom("sensitiveDataSuggestionRuns")
|
|
.select(["engine", "modelId", "policyVersion", "unknown"])
|
|
.where("id", "=", historicalSuggestionRunId)
|
|
.executeTakeFirstOrThrow()).resolves.toMatchObject({
|
|
engine: "llm",
|
|
modelId: "openai-mini",
|
|
policyVersion: null,
|
|
unknown: 0,
|
|
});
|
|
await expect(db.insertInto("descriptionGenerationRuns").values({
|
|
id: randomUUID(), databaseId: historicalDatabaseId, scope: "all",
|
|
modelId: "openai/gpt-5-mini", language: "en", status: "completed", total: 1,
|
|
processed: 1, generated: 1,
|
|
}).execute()).resolves.toBeDefined();
|
|
await sql`CREATE ROLE thothii_catalog_runtime`.execute(db);
|
|
await upRuntimeSequencePrivileges(db);
|
|
const sequencePrivilege = await sql<{ allowed: boolean }>`
|
|
SELECT has_sequence_privilege(
|
|
'thothii_catalog_runtime',
|
|
'catalog_sync_events_id_seq',
|
|
'USAGE'
|
|
) AS allowed
|
|
`.execute(db);
|
|
expect(sequencePrivilege.rows[0]?.allowed).toBe(true);
|
|
const repository = new KyselyCatalogRepository(db);
|
|
const input = {
|
|
workspaceId: "psd-clinical",
|
|
engine: "postgres" as const,
|
|
databaseName: "warehouse",
|
|
schema: "datawarehouse",
|
|
binding: { transport: "rest_api" as const, baseUrl: "https://psd.example/api", restPath: "/health", restAuth: "bearer" as const },
|
|
};
|
|
const created = await repository.create(input);
|
|
expect(created).toMatchObject({ version: 1, connectionStatus: "untested", binding: { transport: "rest_api" } });
|
|
await expect(repository.create(input)).rejects.toThrow("Workspace database already exists");
|
|
expect(await repository.update(created.id, 99, input)).toBeUndefined();
|
|
const synchronized = await repository.reconcileTables(created.id, 1, [
|
|
{ name: "patients", sourceComment: "Clinical patients" },
|
|
{ name: "visits", sourceComment: null },
|
|
], []);
|
|
expect(synchronized).toMatchObject({ kind: "applied", createdCount: 2, deletedCount: 0 });
|
|
const patients = (await repository.listTables(created.id))[0];
|
|
expect(await repository.updateTableDescription(
|
|
created.id,
|
|
patients.id,
|
|
patients.version,
|
|
"Curated patients",
|
|
)).toMatchObject({ description: "Curated patients", sourceComment: "Clinical patients", version: 2 });
|
|
const visits = (await repository.listTables(created.id)).find((table) => table.name === "visits")!;
|
|
const fullColumnsSnapshot: ObservedSchemaSnapshot = {
|
|
schemaVersion: 1,
|
|
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
|
tables: [
|
|
{ name: "patients", sourceComment: "Clinical patients" },
|
|
{ name: "visits", sourceComment: null },
|
|
],
|
|
columns: [
|
|
{ tableName: "patients", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
|
{ tableName: "patients", name: "name", ordinalPosition: 2, dataType: "text", isNullable: true, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
|
{ tableName: "visits", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
|
{ tableName: "visits", name: "patient_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
|
],
|
|
relationships: [],
|
|
};
|
|
const memoryCleanupRun = await repository.createSyncRun(created.id, "columns", [], 1);
|
|
await repository.updateSyncRun(memoryCleanupRun.id, { state: "applying", phase: "applying" });
|
|
await expect(repository.applySchemaSync(created.id, 999, "columns", [], fullColumnsSnapshot, memoryCleanupRun.id))
|
|
.resolves.toBeUndefined();
|
|
expect((await repository.getSyncRun(memoryCleanupRun.id))?.phase).toBe("applying");
|
|
expect(await repository.applySchemaSync(created.id, 1, "columns", [], fullColumnsSnapshot, memoryCleanupRun.id))
|
|
.toMatchObject({ created: 4, deleted: 0 });
|
|
expect((await repository.getSyncRun(memoryCleanupRun.id))?.phase).toBe("memory_cleanup");
|
|
await repository.interruptActiveSyncRuns();
|
|
expect(await repository.getSyncRun(memoryCleanupRun.id))
|
|
.toMatchObject({ state: "interrupted", phase: "memory_cleanup" });
|
|
expect((await repository.listColumns(created.id, patients.id)).map((column) => ({
|
|
name: column.name,
|
|
sensitive: column.sensitive,
|
|
}))).toEqual([
|
|
{ name: "id", sensitive: false },
|
|
{ name: "name", sensitive: false },
|
|
]);
|
|
expect((await repository.listColumns(created.id, visits.id)).map((column) => column.name))
|
|
.toEqual(["id", "patient_id"]);
|
|
|
|
const patientName = (await repository.listColumns(created.id, patients.id))
|
|
.find((column) => column.name === "name")!;
|
|
expect(await repository.updateColumnMetadata(
|
|
created.id,
|
|
patients.id,
|
|
patientName.id,
|
|
patientName.version,
|
|
patientName.description,
|
|
patientName.generatedDescription,
|
|
true,
|
|
"Local assessment matched content rule pii.person_name.",
|
|
)).toMatchObject({
|
|
sensitive: true,
|
|
sensitivityReason: "Local assessment matched content rule pii.person_name.",
|
|
});
|
|
expect(await repository.getCatalogMetrics(created.id)).toEqual({
|
|
scope: "database",
|
|
databaseId: created.id,
|
|
tables: 2,
|
|
columns: 4,
|
|
sensitiveColumns: 1,
|
|
relationships: 0,
|
|
descriptionTargets: 6,
|
|
describedTargets: 1,
|
|
descriptionCoverage: 17,
|
|
updatedAt: expect.any(String),
|
|
});
|
|
expect(await repository.getCatalogMetrics()).toEqual({
|
|
scope: "global",
|
|
databaseId: null,
|
|
tables: 2,
|
|
columns: 4,
|
|
sensitiveColumns: 1,
|
|
relationships: 0,
|
|
descriptionTargets: 6,
|
|
describedTargets: 1,
|
|
descriptionCoverage: 17,
|
|
updatedAt: expect.any(String),
|
|
});
|
|
expect(await repository.getCatalogMetrics(
|
|
"99999999-9999-4999-8999-999999999999",
|
|
)).toBeUndefined();
|
|
const refreshedColumnsSnapshot: ObservedSchemaSnapshot = {
|
|
...fullColumnsSnapshot,
|
|
schemaVersion: 2,
|
|
columns: fullColumnsSnapshot.columns.map((column) => column.tableName === "patients"
|
|
&& column.name === "name"
|
|
? { ...column, sourceComment: "Sensitive patient name" }
|
|
: column),
|
|
};
|
|
expect(await repository.applySchemaSync(
|
|
created.id,
|
|
1,
|
|
"columns",
|
|
[patients.id],
|
|
refreshedColumnsSnapshot,
|
|
)).toMatchObject({ updated: 1 });
|
|
expect(await repository.getColumn(created.id, patients.id, patientName.id)).toMatchObject({
|
|
sensitive: true,
|
|
sensitivityReason: "Local assessment matched content rule pii.person_name.",
|
|
sourceComment: "Sensitive patient name",
|
|
});
|
|
|
|
const reducedColumnsSnapshot: ObservedSchemaSnapshot = {
|
|
...fullColumnsSnapshot,
|
|
columns: fullColumnsSnapshot.columns.filter((column) => column.name === "id"),
|
|
};
|
|
expect(await repository.planSchemaSync(created.id, "columns", [], reducedColumnsSnapshot)).toMatchObject({
|
|
deletedColumns: [
|
|
{ tableName: "patients", columnName: "name" },
|
|
{ tableName: "visits", columnName: "patient_id" },
|
|
],
|
|
});
|
|
expect(await repository.planSchemaSync(created.id, "columns", [patients.id], reducedColumnsSnapshot)).toMatchObject({
|
|
deletedColumns: [{ tableName: "patients", columnName: "name" }],
|
|
});
|
|
expect(await repository.applySchemaSync(created.id, 1, "columns", [patients.id], reducedColumnsSnapshot))
|
|
.toMatchObject({ deleted: 1 });
|
|
expect((await repository.listColumns(created.id, patients.id)).map((column) => column.name))
|
|
.toEqual(["id"]);
|
|
expect((await repository.listColumns(created.id, visits.id)).map((column) => column.name))
|
|
.toEqual(["id", "patient_id"]);
|
|
expect(await repository.reconcileTables(created.id, 1, [
|
|
{ name: "patients", sourceComment: "Updated physical comment" },
|
|
], [])).toEqual({ kind: "confirmation_required", deletedNames: ["visits"] });
|
|
expect(await repository.reconcileTables(created.id, 1, [
|
|
{ name: "patients", sourceComment: "Updated physical comment" },
|
|
], ["visits"])).toMatchObject({ kind: "applied", updatedCount: 1, deletedCount: 1 });
|
|
const syncRun = await repository.createSyncRun(created.id, "columns", [patients.id], 1);
|
|
expect(syncRun).toMatchObject({
|
|
databaseId: created.id,
|
|
scope: "columns",
|
|
tableIds: [patients.id],
|
|
state: "queued",
|
|
});
|
|
expect(await repository.listSyncRuns(created.id)).toEqual([
|
|
expect.objectContaining({ id: syncRun.id, tableIds: [patients.id] }),
|
|
expect.objectContaining({ id: memoryCleanupRun.id, phase: "memory_cleanup" }),
|
|
]);
|
|
|
|
const lockedDatabase = await repository.create({
|
|
workspaceId: "preprocessing-lock",
|
|
engine: "postgres",
|
|
databaseName: "warehouse",
|
|
schema: "public",
|
|
binding: {
|
|
transport: "postgres_direct", host: "lock.internal", port: 5432, username: "reader",
|
|
},
|
|
});
|
|
await repository.applySchemaSync(
|
|
lockedDatabase.id,
|
|
lockedDatabase.version,
|
|
"all",
|
|
[],
|
|
{
|
|
schemaVersion: 1,
|
|
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
|
tables: [], columns: [], relationships: [],
|
|
},
|
|
);
|
|
const beforePreprocessing = (await repository.get(lockedDatabase.id))!;
|
|
const preprocessing = await repository.beginPreprocessing(
|
|
"preprocessing-lock",
|
|
"sha256:" + "1".repeat(64),
|
|
);
|
|
expect(preprocessing).toMatchObject({ kind: "started" });
|
|
await expect(db.insertInto("catalogTables").values({
|
|
id: randomUUID(),
|
|
databaseId: lockedDatabase.id,
|
|
name: "blocked_write",
|
|
sourceComment: null,
|
|
description: null,
|
|
generatedDescription: null,
|
|
}).execute()).rejects.toThrow("catalog preprocessing is running");
|
|
await expect(repository.createDescriptionGenerationRun(
|
|
lockedDatabase.id,
|
|
"all",
|
|
"openai/gpt-5-mini",
|
|
"en",
|
|
0,
|
|
)).rejects.toThrow("catalog preprocessing is running");
|
|
await repository.recordTest(lockedDatabase.id, lockedDatabase.version, {
|
|
connectionStatus: "reachable",
|
|
testedVersion: lockedDatabase.version,
|
|
lastTestedAt: "2026-01-01T00:00:00Z",
|
|
});
|
|
expect((await repository.get(lockedDatabase.id))?.metadataContentRevision)
|
|
.toBe(beforePreprocessing.metadataContentRevision);
|
|
await expect(repository.finishPreprocessing(
|
|
"preprocessing-lock",
|
|
beforePreprocessing.metadataContentRevision,
|
|
"sha256:" + "1".repeat(64),
|
|
{ status: "succeeded" },
|
|
)).resolves.toMatchObject({
|
|
preprocessingStatus: "succeeded",
|
|
preprocessedMetadataRevision: beforePreprocessing.metadataContentRevision,
|
|
});
|
|
await db.insertInto("catalogTables").values({
|
|
id: randomUUID(),
|
|
databaseId: lockedDatabase.id,
|
|
name: "allowed_after_preprocessing",
|
|
sourceComment: null,
|
|
description: null,
|
|
generatedDescription: null,
|
|
}).execute();
|
|
await expect(repository.get(lockedDatabase.id)).resolves.toMatchObject({
|
|
preprocessingStatus: "failed",
|
|
metadataContentRevision: beforePreprocessing.metadataContentRevision + 1,
|
|
});
|
|
expect(await repository.delete(lockedDatabase.id, lockedDatabase.version)).toBe(true);
|
|
expect(await repository.update(created.id, 1, { ...input, schema: "public" })).toMatchObject({ version: 2, schema: "public" });
|
|
expect(await repository.delete(created.id, 2)).toBe(true);
|
|
expect(await repository.list()).toEqual([]);
|
|
expect(await repository.listTables(created.id)).toEqual([]);
|
|
} finally {
|
|
await db.destroy();
|
|
await container.stop();
|
|
}
|
|
}, 60_000);
|
|
|
|
test.skipIf(!dockerAvailable)("PostgreSQL repository performs scoped metadata cleanup with catalog cascades", async () => {
|
|
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
|
|
const db = new Kysely<CatalogDatabase>({
|
|
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
|
|
plugins: [new CamelCasePlugin()],
|
|
});
|
|
try {
|
|
await upDatabases(db);
|
|
await upTables(db);
|
|
await upSchemaSync(db);
|
|
await upSensitiveDataFlag(db);
|
|
await upLogicalRelationships(db);
|
|
const repository = new KyselyCatalogRepository(db);
|
|
const database = await repository.create({
|
|
workspaceId: "cleanup-test",
|
|
engine: "postgres",
|
|
databaseName: "warehouse",
|
|
schema: "public",
|
|
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
|
});
|
|
const snapshot: ObservedSchemaSnapshot = {
|
|
schemaVersion: 1,
|
|
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
|
tables: [
|
|
{ name: "patients", sourceComment: null },
|
|
{ name: "visits", sourceComment: null },
|
|
],
|
|
columns: [
|
|
{ tableName: "patients", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
|
{ tableName: "patients", name: "name", ordinalPosition: 2, dataType: "text", isNullable: true, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
|
{ tableName: "visits", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
|
{ tableName: "visits", name: "patient_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
|
],
|
|
relationships: [{
|
|
constraintName: "visits_patient_id_fkey",
|
|
sourceTableName: "visits",
|
|
targetTableName: "patients",
|
|
updateRule: "NO ACTION",
|
|
deleteRule: "CASCADE",
|
|
deferrable: false,
|
|
initiallyDeferred: false,
|
|
columns: [{ position: 1, sourceColumnName: "patient_id", targetColumnName: "id" }],
|
|
}],
|
|
};
|
|
await repository.applySchemaSync(database.id, database.version, "all", [], snapshot);
|
|
const patients = (await repository.listTables(database.id)).find((table) => table.name === "patients")!;
|
|
const context = (await repository.getLogicalRelationshipContext(database.id))!;
|
|
const source = context.endpoints.find((endpoint) => (
|
|
endpoint.tableName === "visits" && endpoint.columnName === "id"
|
|
))!;
|
|
const target = context.endpoints.find((endpoint) => (
|
|
endpoint.tableName === "patients" && endpoint.columnName === "id"
|
|
))!;
|
|
const generatedCandidate = { sourceColumnId: source.columnId, targetColumnId: target.columnId };
|
|
|
|
await expect(repository.insertGeneratedLogicalRelationships(database.id, [generatedCandidate]))
|
|
.resolves.toBe(1);
|
|
await expect(repository.getCatalogMetrics(database.id))
|
|
.resolves.toMatchObject({ relationships: 2 });
|
|
expect(await repository.deleteDatabaseMetadata([database.id], "relationships"))
|
|
.toEqual({ tables: 0, columns: 0, relationships: 2 });
|
|
expect(await repository.listRelationships(database.id)).toEqual([]);
|
|
expect(await repository.listLogicalRelationships(database.id)).toEqual([]);
|
|
await expect(repository.getCatalogMetrics(database.id))
|
|
.resolves.toMatchObject({ relationships: 0 });
|
|
|
|
await repository.applySchemaSync(database.id, database.version, "relationships", [], snapshot);
|
|
await expect(repository.insertGeneratedLogicalRelationships(database.id, [generatedCandidate]))
|
|
.resolves.toBe(1);
|
|
expect(await repository.deleteTableMetadata(database.id, [patients.id], "relationships"))
|
|
.toEqual({ tables: 0, columns: 0, relationships: 2 });
|
|
expect(await repository.listRelationships(database.id)).toEqual([]);
|
|
expect(await repository.listLogicalRelationships(database.id)).toEqual([]);
|
|
expect(await repository.listColumns(database.id, patients.id)).toHaveLength(2);
|
|
expect((await repository.get(database.id))?.schemaSyncedVersion).toBeUndefined();
|
|
|
|
await repository.applySchemaSync(database.id, database.version, "relationships", [], snapshot);
|
|
expect(await repository.deleteTableMetadata(database.id, [patients.id], "columns"))
|
|
.toEqual({ tables: 0, columns: 2, relationships: 0 });
|
|
expect(await repository.listColumns(database.id, patients.id)).toEqual([]);
|
|
expect(await repository.listRelationships(database.id)).toMatchObject([
|
|
{ constraintName: "visits_patient_id_fkey", columns: [] },
|
|
]);
|
|
|
|
await repository.applySchemaSync(database.id, database.version, "columns", [patients.id], snapshot);
|
|
await repository.applySchemaSync(database.id, database.version, "relationships", [], snapshot);
|
|
const refreshedContext = (await repository.getLogicalRelationshipContext(database.id))!;
|
|
const refreshedSource = refreshedContext.endpoints.find((endpoint) => (
|
|
endpoint.tableName === "visits" && endpoint.columnName === "id"
|
|
))!;
|
|
const refreshedTarget = refreshedContext.endpoints.find((endpoint) => (
|
|
endpoint.tableName === "patients" && endpoint.columnName === "id"
|
|
))!;
|
|
await expect(repository.insertGeneratedLogicalRelationships(database.id, [{
|
|
sourceColumnId: refreshedSource.columnId,
|
|
targetColumnId: refreshedTarget.columnId,
|
|
}])).resolves.toBe(1);
|
|
expect(await repository.deleteDatabaseMetadata([
|
|
database.id,
|
|
"99999999-9999-4999-8999-999999999999",
|
|
], "tables")).toBeUndefined();
|
|
expect(await repository.listTables(database.id)).toHaveLength(2);
|
|
expect(await repository.deleteDatabaseMetadata([database.id], "tables"))
|
|
.toEqual({ tables: 2, columns: 4, relationships: 2 });
|
|
expect(await repository.get(database.id)).toBeDefined();
|
|
expect(await repository.listTables(database.id)).toEqual([]);
|
|
expect(await repository.listRelationships(database.id)).toEqual([]);
|
|
} finally {
|
|
await db.destroy();
|
|
await container.stop();
|
|
}
|
|
}, 60_000);
|
|
|
|
test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates selected and database-wide descriptions", async () => {
|
|
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
|
|
const db = new Kysely<CatalogDatabase>({
|
|
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
|
|
plugins: [new CamelCasePlugin()],
|
|
});
|
|
try {
|
|
await upDatabases(db);
|
|
await upTables(db);
|
|
await upSchemaSync(db);
|
|
await upSensitiveDataFlag(db);
|
|
await upLogicalRelationships(db);
|
|
const repository = new KyselyCatalogRepository(db);
|
|
const database = await repository.create({
|
|
workspaceId: "consolidation-test",
|
|
engine: "postgres",
|
|
databaseName: "warehouse",
|
|
schema: "public",
|
|
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
|
});
|
|
const snapshot: ObservedSchemaSnapshot = {
|
|
schemaVersion: 1,
|
|
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
|
tables: [
|
|
{ name: "patients", sourceComment: null },
|
|
{ name: "visits", sourceComment: null },
|
|
],
|
|
columns: [
|
|
{ tableName: "visits", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
|
{ tableName: "visits", name: "patient_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
|
],
|
|
relationships: [],
|
|
};
|
|
await repository.applySchemaSync(database.id, database.version, "all", [], snapshot);
|
|
const tables = await repository.listTables(database.id);
|
|
const patients = tables.find((table) => table.name === "patients")!;
|
|
const visits = tables.find((table) => table.name === "visits")!;
|
|
await repository.updateTableMetadata(
|
|
database.id, patients.id, patients.version, "Old patients", "Generated patients",
|
|
);
|
|
await repository.updateTableMetadata(
|
|
database.id, visits.id, visits.version, "Keep visits", " ",
|
|
);
|
|
|
|
expect(await repository.consolidateGeneratedDescriptions(
|
|
database.id, "tables", [patients.id, visits.id],
|
|
)).toEqual({ copied: 1, skipped: 1 });
|
|
expect(await repository.getTable(database.id, patients.id)).toMatchObject({
|
|
description: "Generated patients",
|
|
generatedDescription: "Generated patients",
|
|
version: patients.version + 2,
|
|
});
|
|
expect(await repository.getTable(database.id, visits.id)).toMatchObject({
|
|
description: "Keep visits",
|
|
generatedDescription: " ",
|
|
version: visits.version + 1,
|
|
});
|
|
|
|
const columns = await repository.listColumns(database.id, visits.id);
|
|
const id = columns.find((column) => column.name === "id")!;
|
|
const patientId = columns.find((column) => column.name === "patient_id")!;
|
|
await repository.updateColumnMetadata(
|
|
database.id, visits.id, id.id, id.version, "Old id", "Generated id",
|
|
);
|
|
await repository.updateColumnMetadata(
|
|
database.id, visits.id, patientId.id, patientId.version, "Keep patient reference", null,
|
|
);
|
|
|
|
expect(await repository.consolidateGeneratedDescriptions(
|
|
database.id, "columns", [id.id, patientId.id],
|
|
)).toEqual({ copied: 1, skipped: 1 });
|
|
expect(await repository.getColumn(database.id, visits.id, id.id)).toMatchObject({
|
|
description: "Generated id",
|
|
generatedDescription: "Generated id",
|
|
version: id.version + 2,
|
|
});
|
|
expect(await repository.getColumn(database.id, visits.id, patientId.id)).toMatchObject({
|
|
description: "Keep patient reference",
|
|
generatedDescription: null,
|
|
version: patientId.version + 1,
|
|
});
|
|
|
|
const currentVisits = (await repository.getTable(database.id, visits.id))!;
|
|
await repository.updateTableMetadata(
|
|
database.id, visits.id, currentVisits.version, "Still curated visits", "Generated visits",
|
|
);
|
|
expect(await repository.consolidateGeneratedDescriptions(database.id, "tables", [
|
|
visits.id,
|
|
"99999999-9999-4999-8999-999999999999",
|
|
])).toBeUndefined();
|
|
expect(await repository.getTable(database.id, visits.id)).toMatchObject({
|
|
description: "Still curated visits",
|
|
generatedDescription: "Generated visits",
|
|
});
|
|
|
|
expect(await repository.consolidateGeneratedDescriptions(
|
|
database.id, "database", [],
|
|
)).toEqual({ copied: 3, skipped: 1 });
|
|
expect(await repository.getTable(database.id, visits.id)).toMatchObject({
|
|
description: "Generated visits",
|
|
generatedDescription: "Generated visits",
|
|
});
|
|
expect(await repository.getColumn(database.id, visits.id, id.id)).toMatchObject({
|
|
description: "Generated id",
|
|
generatedDescription: "Generated id",
|
|
});
|
|
expect(await repository.getColumn(database.id, visits.id, patientId.id)).toMatchObject({
|
|
description: "Keep patient reference",
|
|
generatedDescription: null,
|
|
});
|
|
} finally {
|
|
await db.destroy();
|
|
await container.stop();
|
|
}
|
|
}, 60_000);
|
|
|
|
test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and sensitive suggestion run histories", async () => {
|
|
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
|
|
const db = new Kysely<CatalogDatabase>({
|
|
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
|
|
plugins: [new CamelCasePlugin()],
|
|
});
|
|
try {
|
|
await upDatabases(db);
|
|
await upTables(db);
|
|
await upSchemaSync(db);
|
|
await upSensitiveDataFlag(db);
|
|
await upLogicalRelationships(db);
|
|
await upDescriptionGeneration(db);
|
|
await upSensitiveSuggestionRuns(db);
|
|
await upAiTokenUsage(db);
|
|
await upCanonicalModelIds(db);
|
|
await upLocalSensitivityAnalysis(db);
|
|
await upSensitivityReason(db);
|
|
await upCatalogPreprocessingState(db);
|
|
const repository = new KyselyCatalogRepository(db);
|
|
const firstDatabase = await repository.create({
|
|
workspaceId: "generation-one",
|
|
engine: "postgres",
|
|
databaseName: "warehouse_one",
|
|
schema: "public",
|
|
binding: { transport: "postgres_direct", host: "one.internal", port: 5432, username: "reader" },
|
|
});
|
|
const secondDatabase = await repository.create({
|
|
workspaceId: "generation-two",
|
|
engine: "postgres",
|
|
databaseName: "warehouse_two",
|
|
schema: "public",
|
|
binding: { transport: "postgres_direct", host: "two.internal", port: 5432, username: "reader" },
|
|
});
|
|
await repository.applySchemaSync(firstDatabase.id, firstDatabase.version, "all", [], {
|
|
schemaVersion: 1,
|
|
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
|
tables: [{ name: "patients", sourceComment: "Clinical patients" }],
|
|
columns: [{
|
|
tableName: "patients",
|
|
name: "birth_date",
|
|
ordinalPosition: 1,
|
|
dataType: "date",
|
|
isNullable: true,
|
|
defaultExpression: null,
|
|
primaryKeyPosition: null,
|
|
sourceComment: "Patient date of birth",
|
|
}],
|
|
relationships: [],
|
|
});
|
|
const table = (await repository.listTables(firstDatabase.id))[0]!;
|
|
const column = (await repository.listColumns(firstDatabase.id, table.id))[0]!;
|
|
|
|
const run = await repository.createDescriptionGenerationRun(
|
|
firstDatabase.id,
|
|
"selected_columns",
|
|
"openai/gpt-4.1-mini",
|
|
"it",
|
|
1,
|
|
);
|
|
expect(run).toMatchObject({
|
|
databaseId: firstDatabase.id,
|
|
scope: "selected_columns",
|
|
modelId: "openai/gpt-4.1-mini",
|
|
language: "it",
|
|
status: "queued",
|
|
total: 1,
|
|
processed: 0,
|
|
generated: 0,
|
|
nonGeneratable: 0,
|
|
failed: 0,
|
|
startedAt: null,
|
|
finishedAt: null,
|
|
errorSummary: null,
|
|
});
|
|
await expect(repository.createDescriptionGenerationRun(
|
|
secondDatabase.id,
|
|
"selected_columns",
|
|
"openai/gpt-4.1-mini",
|
|
"en",
|
|
1,
|
|
)).rejects.toThrow("A description generation run is already active");
|
|
await repository.updateDescriptionGenerationRun(run.id, {
|
|
status: "running",
|
|
startedAt: new Date().toISOString(),
|
|
});
|
|
await expect(repository.createDescriptionGenerationRun(
|
|
secondDatabase.id,
|
|
"selected_columns",
|
|
"openai/gpt-4.1-mini",
|
|
"en",
|
|
1,
|
|
)).rejects.toThrow("A description generation run is already active");
|
|
|
|
await repository.appendDescriptionGenerationEvent(run.id, "info", "Description generation queued.");
|
|
await repository.appendDescriptionGenerationEvent(run.id, "info", "Description generation started.");
|
|
expect(await repository.listDescriptionGenerationEvents(run.id, 1)).toEqual([
|
|
expect.objectContaining({
|
|
runId: run.id,
|
|
sequence: 2,
|
|
level: "info",
|
|
message: "Description generation started.",
|
|
createdAt: expect.any(String),
|
|
}),
|
|
]);
|
|
|
|
const updatedColumn = await repository.updateColumnMetadata(
|
|
firstDatabase.id,
|
|
table.id,
|
|
column.id,
|
|
column.version,
|
|
column.description,
|
|
"Data di nascita del paziente.",
|
|
);
|
|
expect(updatedColumn).toMatchObject({
|
|
generatedDescription: "Data di nascita del paziente.",
|
|
version: column.version + 1,
|
|
});
|
|
expect(await repository.updateDescriptionGenerationRun(run.id, {
|
|
status: "completed",
|
|
processed: 1,
|
|
generated: 1,
|
|
startedAt: new Date().toISOString(),
|
|
finishedAt: new Date().toISOString(),
|
|
})).toMatchObject({
|
|
status: "completed",
|
|
processed: 1,
|
|
generated: 1,
|
|
});
|
|
|
|
const next = await repository.createDescriptionGenerationRun(
|
|
secondDatabase.id,
|
|
"missing",
|
|
"openai/gpt-4.1-mini",
|
|
"en",
|
|
1,
|
|
);
|
|
expect(await repository.getDescriptionGenerationRun(next.id)).toMatchObject({
|
|
scope: "missing",
|
|
total: 1,
|
|
});
|
|
await repository.updateDescriptionGenerationRun(next.id, {
|
|
status: "running",
|
|
startedAt: new Date().toISOString(),
|
|
});
|
|
expect(await repository.updateDescriptionGenerationRun(next.id, {
|
|
status: "failed",
|
|
processed: 1,
|
|
failed: 1,
|
|
finishedAt: new Date().toISOString(),
|
|
errorSummary: "The model provider request failed.",
|
|
})).toMatchObject({
|
|
status: "failed",
|
|
processed: 1,
|
|
failed: 1,
|
|
errorSummary: "The model provider request failed.",
|
|
});
|
|
|
|
const allRun = await repository.createDescriptionGenerationRun(
|
|
firstDatabase.id,
|
|
"all",
|
|
"openai/gpt-4.1-mini",
|
|
"it",
|
|
2,
|
|
);
|
|
expect(await repository.getDescriptionGenerationRun(allRun.id)).toMatchObject({
|
|
scope: "all",
|
|
total: 2,
|
|
});
|
|
expect(await repository.getActiveDescriptionGenerationRun()).toMatchObject({ id: allRun.id });
|
|
expect((await repository.listDescriptionGenerationRuns(2)).map((candidate) => candidate.id)).toEqual([
|
|
allRun.id,
|
|
next.id,
|
|
]);
|
|
|
|
expect(await repository.interruptActiveDescriptionGenerationRuns(
|
|
"Description generation was interrupted by backend restart.",
|
|
)).toEqual([
|
|
expect.objectContaining({
|
|
id: allRun.id,
|
|
status: "interrupted",
|
|
finishedAt: expect.any(String),
|
|
errorSummary: "Description generation was interrupted by backend restart.",
|
|
}),
|
|
]);
|
|
expect(await repository.getActiveDescriptionGenerationRun()).toBeUndefined();
|
|
|
|
const suggestionRun = await repository.createSensitivityAnalysisRun(
|
|
firstDatabase.id,
|
|
"selected_columns",
|
|
{ engine: "local", policyVersion: "sensitivity-v1" },
|
|
);
|
|
expect(suggestionRun).toMatchObject({
|
|
databaseId: firstDatabase.id,
|
|
status: "running",
|
|
total: 0,
|
|
suggestedSensitive: 0,
|
|
suggestedNonSensitive: 0,
|
|
unknown: 0,
|
|
engine: "local",
|
|
modelId: null,
|
|
policyVersion: "sensitivity-v1",
|
|
startedAt: expect.any(String),
|
|
});
|
|
await repository.appendSensitivityAnalysisEvent(
|
|
suggestionRun.id,
|
|
"info",
|
|
"Sensitive-field suggestion generation started.",
|
|
);
|
|
await repository.appendSensitivityAnalysisEvent(
|
|
suggestionRun.id,
|
|
"info",
|
|
"Sensitive-field suggestion generation completed for 2 columns.",
|
|
);
|
|
expect(await repository.updateSensitivityAnalysisRun(suggestionRun.id, {
|
|
status: "completed",
|
|
total: 2,
|
|
suggestedSensitive: 1,
|
|
suggestedNonSensitive: 0,
|
|
unknown: 1,
|
|
finishedAt: new Date().toISOString(),
|
|
})).toMatchObject({
|
|
status: "completed",
|
|
total: 2,
|
|
suggestedSensitive: 1,
|
|
suggestedNonSensitive: 0,
|
|
unknown: 1,
|
|
});
|
|
expect(await repository.listSensitivityAnalysisEvents(suggestionRun.id, 1)).toEqual([
|
|
expect.objectContaining({ sequence: 2, level: "info" }),
|
|
]);
|
|
expect((await repository.listSensitivityAnalysisRuns(1))[0]).toMatchObject({
|
|
id: suggestionRun.id,
|
|
});
|
|
|
|
const interruptedSuggestionRun = await repository.createSensitivityAnalysisRun(
|
|
secondDatabase.id,
|
|
"all",
|
|
{ engine: "local", policyVersion: "sensitivity-v1" },
|
|
);
|
|
expect(await repository.interruptActiveSensitivityAnalysisRuns(
|
|
"Sensitive-field suggestion generation was interrupted by backend restart.",
|
|
)).toEqual([
|
|
expect.objectContaining({
|
|
id: interruptedSuggestionRun.id,
|
|
status: "interrupted",
|
|
finishedAt: expect.any(String),
|
|
}),
|
|
]);
|
|
} finally {
|
|
await db.destroy();
|
|
await container.stop();
|
|
}
|
|
}, 60_000);
|
|
|
|
test.skipIf(!dockerAvailable)("PostgreSQL repository persists logical relationship lifecycle and tombstones", async () => {
|
|
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
|
|
const db = new Kysely<CatalogDatabase>({
|
|
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
|
|
plugins: [new CamelCasePlugin()],
|
|
});
|
|
try {
|
|
await upDatabases(db);
|
|
await upTables(db);
|
|
await upSchemaSync(db);
|
|
await upSensitiveDataFlag(db);
|
|
await upLogicalRelationships(db);
|
|
const repository = new KyselyCatalogRepository(db);
|
|
const database = await repository.create({
|
|
workspaceId: "logical-relationships",
|
|
engine: "postgres",
|
|
databaseName: "warehouse",
|
|
schema: "public",
|
|
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
|
});
|
|
await repository.applySchemaSync(database.id, database.version, "all", [], {
|
|
schemaVersion: 1,
|
|
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
|
tables: [{ name: "users", sourceComment: null }, { name: "orders", sourceComment: null }],
|
|
columns: [
|
|
{ tableName: "users", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
|
{ tableName: "orders", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
|
{ tableName: "orders", name: "user_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
|
],
|
|
relationships: [],
|
|
});
|
|
const context = (await repository.getLogicalRelationshipContext(database.id))!;
|
|
const source = context.endpoints.find((item) => item.tableName === "orders" && item.columnName === "user_id")!;
|
|
const target = context.endpoints.find((item) => item.tableName === "users" && item.columnName === "id")!;
|
|
const created = await repository.insertLogicalRelationship(database.id, source.columnId, target.columnId, false);
|
|
expect(created).toMatchObject({ origin: "manual", status: "active" });
|
|
expect(await repository.getCatalogMetrics(database.id))
|
|
.toMatchObject({ relationships: 1 });
|
|
await expect(repository.insertLogicalRelationship(database.id, source.columnId, target.columnId, true))
|
|
.resolves.toBeUndefined();
|
|
|
|
expect(await repository.setLogicalRelationshipStatus(database.id, created!.id, "excluded"))
|
|
.toMatchObject({ status: "excluded" });
|
|
await expect(repository.insertGeneratedLogicalRelationships(database.id, [{
|
|
sourceColumnId: source.columnId, targetColumnId: target.columnId,
|
|
}])).resolves.toBe(0);
|
|
|
|
await expect(repository.deleteLogicalRelationship(database.id, created!.id)).resolves.toBe(true);
|
|
await expect(repository.insertGeneratedLogicalRelationships(database.id, [{
|
|
sourceColumnId: source.columnId, targetColumnId: target.columnId,
|
|
}])).resolves.toBe(1);
|
|
expect(await repository.listLogicalRelationships(database.id))
|
|
.toMatchObject([{ origin: "generated", status: "active" }]);
|
|
|
|
await db.deleteFrom("catalogColumns").where("id", "=", source.columnId).execute();
|
|
expect(await repository.listLogicalRelationships(database.id)).toEqual([]);
|
|
} finally {
|
|
await db.destroy();
|
|
await container.stop();
|
|
}
|
|
}, 60_000);
|