Codex
82e2c91f42
Publish documentation / publish (push) Successful in 1m27s
Add PostgreSQL-backed memory, editable evidence with source review and activation, and human-approved archive repairs across the harness, API, and UI. Include migrations, deployment support, regression coverage, and validation documentation. Refresh permissions from validated session roles so existing administrator logins can access newly deployed archive management features.
904 lines
40 KiB
TypeScript
904 lines
40 KiB
TypeScript
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { afterEach, expect, test, vi } from "vitest";
|
|
import { buildApp } from "../src/app.js";
|
|
import { loadConfig } from "../src/config.js";
|
|
import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
|
|
import { CatalogOperationCoordinator } from "../src/catalog/operation-coordinator.js";
|
|
import type { CatalogSchemaIntrospector } from "../src/catalog/schema-introspector.js";
|
|
import {
|
|
CatalogConnectorError,
|
|
type CatalogSyncRun,
|
|
type ObservedSchemaSnapshot,
|
|
} from "../src/catalog/types.js";
|
|
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
|
import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/registry.js";
|
|
import type { WorkspaceDescriptor } from "../src/workspaces/schema.js";
|
|
|
|
const roots: string[] = [];
|
|
afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
|
|
|
|
const workspace: WorkspaceDescriptor = {
|
|
workspace: { schema_version: 4, id: "psd-clinical", name: "Policlinico San Donato", language: "it" },
|
|
dwh: { engine: "postgres", database: "warehouse", schema: "datawarehouse", port: 5432, supported_transports: ["postgres_direct"] },
|
|
};
|
|
const revision: WorkspaceRevision = { id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/tmp/psd.yaml" };
|
|
|
|
function snapshot(): ObservedSchemaSnapshot {
|
|
return {
|
|
schemaVersion: 1,
|
|
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
|
tables: [
|
|
{ name: "patients", sourceComment: "Clinical patients" },
|
|
{ name: "visits", sourceComment: "Patient visits" },
|
|
],
|
|
columns: [
|
|
{ tableName: "patients", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: "Patient key" },
|
|
{ tableName: "visits", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
|
{ tableName: "visits", name: "patient_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: "Owning patient" },
|
|
],
|
|
relationships: [{
|
|
constraintName: "visits_patient_id_fkey",
|
|
sourceTableName: "visits",
|
|
targetTableName: "patients",
|
|
updateRule: "NO ACTION",
|
|
deleteRule: "CASCADE",
|
|
deferrable: false,
|
|
initiallyDeferred: false,
|
|
columns: [{ position: 1, sourceColumnName: "patient_id", targetColumnName: "id" }],
|
|
}],
|
|
};
|
|
}
|
|
|
|
function expandedSnapshot(): ObservedSchemaSnapshot {
|
|
const expanded = snapshot();
|
|
expanded.tables.push(
|
|
{ name: "encounters", sourceComment: null },
|
|
{ name: "departments", sourceComment: null },
|
|
);
|
|
expanded.columns.push(
|
|
{ tableName: "encounters", name: "visit_id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
|
{ tableName: "departments", name: "patient_id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
|
);
|
|
expanded.relationships.push(
|
|
{
|
|
constraintName: "encounters_visit_id_fkey",
|
|
sourceTableName: "encounters",
|
|
targetTableName: "visits",
|
|
updateRule: "NO ACTION",
|
|
deleteRule: "CASCADE",
|
|
deferrable: false,
|
|
initiallyDeferred: false,
|
|
columns: [{ position: 1, sourceColumnName: "visit_id", targetColumnName: "id" }],
|
|
},
|
|
{
|
|
constraintName: "departments_patient_id_fkey",
|
|
sourceTableName: "departments",
|
|
targetTableName: "patients",
|
|
updateRule: "NO ACTION",
|
|
deleteRule: "CASCADE",
|
|
deferrable: false,
|
|
initiallyDeferred: false,
|
|
columns: [{ position: 1, sourceColumnName: "patient_id", targetColumnName: "id" }],
|
|
},
|
|
);
|
|
return expanded;
|
|
}
|
|
|
|
async function waitFor(repository: MemoryCatalogRepository, runId: string, state: CatalogSyncRun["state"]): Promise<CatalogSyncRun> {
|
|
for (let attempt = 0; attempt < 100; attempt += 1) {
|
|
const run = await repository.getSyncRun(runId);
|
|
if (run?.state === state) return run;
|
|
await new Promise((resolve) => setTimeout(resolve, 5));
|
|
}
|
|
throw new Error(`Run ${runId} did not reach ${state}`);
|
|
}
|
|
|
|
async function setup(env: Record<string, string> = {}) {
|
|
const secretRoot = mkdtempSync(join(tmpdir(), "catalog-schema-secret-"));
|
|
const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-schema-runtime-"));
|
|
roots.push(secretRoot, runtimeRoot);
|
|
const repository = new MemoryCatalogRepository();
|
|
const created = await repository.create({
|
|
workspaceId: "psd-clinical", engine: "postgres", databaseName: "warehouse", schema: "datawarehouse",
|
|
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
|
});
|
|
await repository.recordTest(created.id, created.version, {
|
|
connectionStatus: "reachable", testedVersion: created.version, lastTestedAt: new Date().toISOString(),
|
|
});
|
|
let observed = snapshot();
|
|
const scan = vi.fn(async (_database, _signal, progress) => {
|
|
await progress?.("connecting");
|
|
await progress?.("scanning_tables", { tables: observed.tables.length });
|
|
await progress?.("scanning_columns", { tables: observed.tables.length, columns: observed.columns.length });
|
|
await progress?.("scanning_relationships", { relationships: observed.relationships.length });
|
|
return structuredClone(observed);
|
|
});
|
|
const introspector: CatalogSchemaIntrospector = { scan };
|
|
const operations = new CatalogOperationCoordinator();
|
|
const cleanup = vi.fn(async () => ({ code: 0, stdout: JSON.stringify({ indexed: true, deleted: 0 }), stderr: "" }));
|
|
const registry = {
|
|
list: vi.fn(async () => [revision]),
|
|
listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]),
|
|
read: vi.fn(async () => ({ workspace, revision })),
|
|
readPinned: vi.fn(async () => ({ workspace, workspaceConfigPath: revision.snapshotPath })),
|
|
} as unknown as WorkspaceRegistry;
|
|
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test", ...env }), {
|
|
thtRunner: { withPrincipal: () => ({ runWithRuntimeSnapshot: cleanup }) } as never,
|
|
workspaceRegistry: registry,
|
|
workspaceSecretStore: new WorkspaceSecretStore({ root: secretRoot, runtimeRoot, installationId: "test" }),
|
|
catalogRepository: repository,
|
|
catalogSchemaIntrospector: introspector,
|
|
catalogOperationCoordinator: operations,
|
|
workspaceDiagnoser: vi.fn(),
|
|
});
|
|
return {
|
|
app, repository, database: (await repository.get(created.id))!, scan, operations, cleanup,
|
|
setObserved(next: ObservedSchemaSnapshot) { observed = next; },
|
|
};
|
|
}
|
|
|
|
async function seedCatalog(
|
|
repository: MemoryCatalogRepository,
|
|
database: { id: string; version: number },
|
|
observed: ObservedSchemaSnapshot = snapshot(),
|
|
): Promise<void> {
|
|
await repository.applySchemaSync(database.id, database.version, "all", [], observed);
|
|
}
|
|
|
|
test("synchronizes a full physical schema and derives primary and foreign key flags", async () => {
|
|
const { app, repository, database } = await setup();
|
|
const started = await app.inject({
|
|
method: "POST", url: `/catalog/databases/${database.id}/sync-runs`,
|
|
payload: { version: database.version, scope: "all", tableIds: [] },
|
|
});
|
|
expect(started.statusCode).toBe(202);
|
|
const completed = await waitFor(repository, started.json().id, "succeeded");
|
|
expect(completed.counts).toMatchObject({ tables: 2, columns: 3, relationships: 1 });
|
|
|
|
const tables = await repository.listTables(database.id);
|
|
const visits = tables.find((table) => table.name === "visits")!;
|
|
const columns = (await app.inject({ method: "GET", url: `/catalog/databases/${database.id}/tables/${visits.id}/columns` })).json();
|
|
expect(columns).toMatchObject([
|
|
{ name: "id", isPrimaryKey: true, primaryKeyPosition: 1, isForeignKey: false },
|
|
{ name: "patient_id", isPrimaryKey: false, isForeignKey: true, foreignKeyCount: 1 },
|
|
]);
|
|
const relationships = (await app.inject({ method: "GET", url: `/catalog/databases/${database.id}/relationships` })).json();
|
|
expect(relationships).toMatchObject([{ constraintName: "visits_patient_id_fkey", columns: [{ sourceColumnName: "patient_id", targetColumnName: "id" }] }]);
|
|
expect((await repository.get(database.id))?.schemaSyncedVersion).toBe(database.version);
|
|
});
|
|
|
|
test("attempts synchronization after a failed connection test and reports the live access failure", async () => {
|
|
const { app, repository, database, scan } = await setup();
|
|
await repository.recordTest(database.id, database.version, {
|
|
connectionStatus: "failed",
|
|
testedVersion: database.version,
|
|
lastTestedAt: new Date().toISOString(),
|
|
lastErrorCode: "connector_unavailable",
|
|
lastErrorMessage: "The database connector could not be reached or authenticated.",
|
|
});
|
|
scan.mockRejectedValueOnce(new CatalogConnectorError("upstream credentials must not escape"));
|
|
|
|
const started = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/sync-runs`,
|
|
payload: { version: database.version, scope: "all", tableIds: [] },
|
|
});
|
|
|
|
expect(started.statusCode).toBe(202);
|
|
const failed = await waitFor(repository, started.json().id, "failed");
|
|
expect(scan).toHaveBeenCalledOnce();
|
|
expect(failed).toMatchObject({
|
|
errorCode: "schema_introspection_failed",
|
|
errorMessage: "The database schema could not be read. Check the connection and credentials, then try again.",
|
|
});
|
|
});
|
|
|
|
test("synchronizes columns for every catalog table when no table selection is supplied", async () => {
|
|
const { app, repository, database, setObserved } = await setup();
|
|
const tablesRun = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/sync-runs`,
|
|
payload: { version: database.version, scope: "tables", tableIds: [] },
|
|
});
|
|
expect(tablesRun.statusCode).toBe(202);
|
|
await waitFor(repository, tablesRun.json().id, "succeeded");
|
|
const tables = await repository.listTables(database.id);
|
|
expect(tables.map((table) => table.name)).toEqual(["patients", "visits"]);
|
|
|
|
const columnsRun = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/sync-runs`,
|
|
payload: { version: database.version, scope: "columns", tableIds: [] },
|
|
});
|
|
expect(columnsRun.statusCode).toBe(202);
|
|
await waitFor(repository, columnsRun.json().id, "succeeded");
|
|
const patients = tables.find((table) => table.name === "patients")!;
|
|
const visits = tables.find((table) => table.name === "visits")!;
|
|
expect((await repository.listColumns(database.id, patients.id)).map((column) => column.name)).toEqual(["id"]);
|
|
expect((await repository.listColumns(database.id, visits.id)).map((column) => column.name)).toEqual(["id", "patient_id"]);
|
|
|
|
const next = snapshot();
|
|
next.columns = next.columns.filter((column) => column.name !== "id");
|
|
setObserved(next);
|
|
const selectedDestructiveRun = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/sync-runs`,
|
|
payload: { version: database.version, scope: "columns", tableIds: [patients.id] },
|
|
});
|
|
expect(selectedDestructiveRun.statusCode).toBe(202);
|
|
const selectedWaiting = await waitFor(repository, selectedDestructiveRun.json().id, "awaiting_confirmation");
|
|
expect(selectedWaiting.plannedDiff?.deletedColumns).toEqual([
|
|
{ tableName: "patients", columnName: "id" },
|
|
]);
|
|
expect((await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/sync-runs/${selectedWaiting.id}/cancel`,
|
|
})).statusCode).toBe(200);
|
|
|
|
const destructiveRun = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/sync-runs`,
|
|
payload: { version: database.version, scope: "columns", tableIds: [] },
|
|
});
|
|
expect(destructiveRun.statusCode).toBe(202);
|
|
const waiting = await waitFor(repository, destructiveRun.json().id, "awaiting_confirmation");
|
|
expect(waiting.plannedDiff?.deletedColumns).toEqual([
|
|
{ tableName: "patients", columnName: "id" },
|
|
{ tableName: "visits", columnName: "id" },
|
|
]);
|
|
});
|
|
|
|
test("keeps generated descriptions editable and preserves them across synchronization", async () => {
|
|
const { app, repository, database } = await setup();
|
|
const first = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
|
await waitFor(repository, first.json().id, "succeeded");
|
|
const patients = (await repository.listTables(database.id)).find((table) => table.name === "patients")!;
|
|
const editedTable = await app.inject({
|
|
method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}`,
|
|
payload: { version: patients.version, description: null, generatedDescription: "Generated table draft" },
|
|
});
|
|
expect(editedTable.json()).toMatchObject({ description: null, generatedDescription: "Generated table draft" });
|
|
const idColumn = (await repository.listColumns(database.id, patients.id))[0];
|
|
expect(idColumn.sensitive).toBe(false);
|
|
const editedColumn = await app.inject({
|
|
method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}/columns/${idColumn.id}`,
|
|
payload: {
|
|
version: idColumn.version,
|
|
description: "Reviewed key",
|
|
generatedDescription: "Generated key draft",
|
|
sensitive: false,
|
|
},
|
|
});
|
|
expect(editedColumn.json()).toMatchObject({
|
|
description: "Reviewed key",
|
|
generatedDescription: "Generated key draft",
|
|
sensitive: false,
|
|
});
|
|
const sensitiveOnly = await app.inject({
|
|
method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}/columns/${idColumn.id}`,
|
|
payload: {
|
|
version: editedColumn.json().version,
|
|
sensitive: true,
|
|
sensitivityReason: "Local assessment matched content rule pii.email.",
|
|
},
|
|
});
|
|
expect(sensitiveOnly.statusCode).toBe(200);
|
|
expect(sensitiveOnly.json()).toMatchObject({
|
|
description: "Reviewed key",
|
|
generatedDescription: "Generated key draft",
|
|
sensitive: true,
|
|
sensitivityReason: "Local assessment matched content rule pii.email.",
|
|
});
|
|
const emptyPatch = await app.inject({
|
|
method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}/columns/${idColumn.id}`,
|
|
payload: { version: sensitiveOnly.json().version },
|
|
});
|
|
expect(emptyPatch.statusCode).toBe(400);
|
|
|
|
const second = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
|
await waitFor(repository, second.json().id, "succeeded");
|
|
expect(await repository.getTable(database.id, patients.id)).toMatchObject({ generatedDescription: "Generated table draft" });
|
|
expect(await repository.getColumn(database.id, patients.id, idColumn.id)).toMatchObject({
|
|
description: "Reviewed key",
|
|
generatedDescription: "Generated key draft",
|
|
sensitive: true,
|
|
sensitivityReason: "Local assessment matched content rule pii.email.",
|
|
});
|
|
});
|
|
|
|
test("consolidates non-empty generated table descriptions and reports skipped selections", async () => {
|
|
const { app, repository, database, scan } = await setup();
|
|
await seedCatalog(repository, database);
|
|
const tables = await repository.listTables(database.id);
|
|
const patients = tables.find((table) => table.name === "patients")!;
|
|
const visits = tables.find((table) => table.name === "visits")!;
|
|
await repository.updateTableMetadata(
|
|
database.id,
|
|
patients.id,
|
|
patients.version,
|
|
"Curated patients",
|
|
"Generated patients",
|
|
);
|
|
await repository.updateTableMetadata(
|
|
database.id,
|
|
visits.id,
|
|
visits.version,
|
|
"Keep curated visits",
|
|
null,
|
|
);
|
|
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
|
payload: { target: "tables", targetIds: [patients.id, visits.id] },
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(response.json()).toEqual({ copied: 1, skipped: 1 });
|
|
expect(await repository.getTable(database.id, patients.id)).toMatchObject({
|
|
description: "Generated patients",
|
|
generatedDescription: "Generated patients",
|
|
version: patients.version + 2,
|
|
});
|
|
expect(await repository.getTable(database.id, visits.id)).toMatchObject({
|
|
description: "Keep curated visits",
|
|
generatedDescription: null,
|
|
version: visits.version + 1,
|
|
});
|
|
expect(scan).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("consolidates non-empty generated column descriptions and preserves curated text for empty proposals", async () => {
|
|
const { app, repository, database, scan } = await setup();
|
|
await seedCatalog(repository, database);
|
|
const visits = (await repository.listTables(database.id)).find((table) => table.name === "visits")!;
|
|
const columns = await repository.listColumns(database.id, visits.id);
|
|
const id = columns.find((column) => column.name === "id")!;
|
|
const patientId = columns.find((column) => column.name === "patient_id")!;
|
|
await repository.updateColumnMetadata(
|
|
database.id,
|
|
visits.id,
|
|
id.id,
|
|
id.version,
|
|
"Curated visit identifier",
|
|
"Generated visit identifier",
|
|
);
|
|
await repository.updateColumnMetadata(
|
|
database.id,
|
|
visits.id,
|
|
patientId.id,
|
|
patientId.version,
|
|
"Keep curated patient reference",
|
|
"",
|
|
);
|
|
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
|
payload: { target: "columns", targetIds: [id.id, patientId.id] },
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(response.json()).toEqual({ copied: 1, skipped: 1 });
|
|
expect(await repository.getColumn(database.id, visits.id, id.id)).toMatchObject({
|
|
description: "Generated visit identifier",
|
|
generatedDescription: "Generated visit identifier",
|
|
version: id.version + 2,
|
|
});
|
|
expect(await repository.getColumn(database.id, visits.id, patientId.id)).toMatchObject({
|
|
description: "Keep curated patient reference",
|
|
generatedDescription: "",
|
|
version: patientId.version + 1,
|
|
});
|
|
expect(scan).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("consolidates generated descriptions for every table and column in a database", async () => {
|
|
const { app, repository, database, scan } = await setup();
|
|
await seedCatalog(repository, database);
|
|
const tables = await repository.listTables(database.id);
|
|
const patients = tables.find((table) => table.name === "patients")!;
|
|
const visits = tables.find((table) => table.name === "visits")!;
|
|
await repository.updateTableMetadata(
|
|
database.id,
|
|
patients.id,
|
|
patients.version,
|
|
"Curated patients",
|
|
"Generated patients",
|
|
);
|
|
const patientId = (await repository.listColumns(database.id, patients.id))[0]!;
|
|
const visitColumns = await repository.listColumns(database.id, visits.id);
|
|
const visitId = visitColumns.find((column) => column.name === "id")!;
|
|
const visitPatientId = visitColumns.find((column) => column.name === "patient_id")!;
|
|
await repository.updateColumnMetadata(
|
|
database.id,
|
|
patients.id,
|
|
patientId.id,
|
|
patientId.version,
|
|
"Curated patient identifier",
|
|
"Generated patient identifier",
|
|
);
|
|
await repository.updateColumnMetadata(
|
|
database.id,
|
|
visits.id,
|
|
visitId.id,
|
|
visitId.version,
|
|
"Curated visit identifier",
|
|
"Generated visit identifier",
|
|
);
|
|
await repository.updateColumnMetadata(
|
|
database.id,
|
|
visits.id,
|
|
visitPatientId.id,
|
|
visitPatientId.version,
|
|
"Keep curated patient reference",
|
|
"",
|
|
);
|
|
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
|
payload: { target: "database" },
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(response.json()).toEqual({ copied: 3, skipped: 2 });
|
|
expect(await repository.getTable(database.id, patients.id)).toMatchObject({
|
|
description: "Generated patients",
|
|
generatedDescription: "Generated patients",
|
|
version: patients.version + 2,
|
|
});
|
|
expect(await repository.getColumn(database.id, patients.id, patientId.id)).toMatchObject({
|
|
description: "Generated patient identifier",
|
|
generatedDescription: "Generated patient identifier",
|
|
version: patientId.version + 2,
|
|
});
|
|
expect(await repository.getColumn(database.id, visits.id, visitId.id)).toMatchObject({
|
|
description: "Generated visit identifier",
|
|
generatedDescription: "Generated visit identifier",
|
|
version: visitId.version + 2,
|
|
});
|
|
expect(await repository.getColumn(database.id, visits.id, visitPatientId.id)).toMatchObject({
|
|
description: "Keep curated patient reference",
|
|
generatedDescription: "",
|
|
version: visitPatientId.version + 1,
|
|
});
|
|
expect(scan).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("rejects description consolidation while the Workspace Database is reserved", async () => {
|
|
const { app, repository, database, operations } = await setup();
|
|
await seedCatalog(repository, database);
|
|
const table = (await repository.listTables(database.id))[0]!;
|
|
const edited = await repository.updateTableMetadata(
|
|
database.id,
|
|
table.id,
|
|
table.version,
|
|
"Existing curated text",
|
|
"Generated text",
|
|
);
|
|
const release = operations.reserve(database.id);
|
|
try {
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
|
payload: { target: "tables", targetIds: [table.id] },
|
|
});
|
|
|
|
expect(response.statusCode).toBe(409);
|
|
expect(response.json()).toEqual({
|
|
code: "database_operation_in_progress",
|
|
message: "A database operation is already in progress.",
|
|
});
|
|
expect(await repository.getTable(database.id, table.id)).toMatchObject({
|
|
description: "Existing curated text",
|
|
generatedDescription: "Generated text",
|
|
version: edited!.version,
|
|
});
|
|
} finally {
|
|
release();
|
|
}
|
|
});
|
|
|
|
test("requires database.manage for description consolidation", async () => {
|
|
const { app, repository, database } = await setup({ AUTH_MODE: "upstream" });
|
|
await seedCatalog(repository, database);
|
|
const table = (await repository.listTables(database.id))[0]!;
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
|
headers: {
|
|
"x-thoth-principal-issuer": "portal",
|
|
"x-thoth-principal-subject": "catalog-reader",
|
|
"x-thoth-is-admin": "0",
|
|
},
|
|
payload: { target: "tables", targetIds: [table.id] },
|
|
});
|
|
|
|
expect(response.statusCode).toBe(403);
|
|
expect(response.json()).toEqual({ code: "auth_forbidden", error: "This operation is not permitted" });
|
|
});
|
|
|
|
test("strictly validates description consolidation database and target ids", async () => {
|
|
const { app, repository, database } = await setup();
|
|
await seedCatalog(repository, database);
|
|
const table = (await repository.listTables(database.id))[0]!;
|
|
|
|
const responses = await Promise.all([
|
|
app.inject({
|
|
method: "POST",
|
|
url: "/catalog/databases/not-a-uuid/descriptions/consolidate",
|
|
payload: { target: "tables", targetIds: [table.id] },
|
|
}),
|
|
app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
|
payload: { target: "tables", targetIds: ["not-a-uuid"] },
|
|
}),
|
|
app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
|
payload: { target: "tables", targetIds: [table.id], unexpected: true },
|
|
}),
|
|
app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
|
payload: { target: "database", targetIds: [table.id] },
|
|
}),
|
|
app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
|
payload: { target: "database_columns", targetIds: [table.id] },
|
|
}),
|
|
]);
|
|
|
|
expect(responses.map((response) => response.statusCode)).toEqual([400, 400, 400, 400, 400]);
|
|
for (const response of responses) {
|
|
expect(response.json()).toEqual({
|
|
code: "description_consolidation_invalid",
|
|
message: "Description consolidation request is invalid.",
|
|
});
|
|
}
|
|
});
|
|
|
|
test("rejects a missing consolidation target without copying valid selections", async () => {
|
|
const { app, repository, database } = await setup();
|
|
await seedCatalog(repository, database);
|
|
const table = (await repository.listTables(database.id))[0]!;
|
|
const edited = await repository.updateTableMetadata(
|
|
database.id,
|
|
table.id,
|
|
table.version,
|
|
"Existing curated text",
|
|
"Generated text",
|
|
);
|
|
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
|
payload: {
|
|
target: "tables",
|
|
targetIds: [table.id, "99999999-9999-4999-8999-999999999999"],
|
|
},
|
|
});
|
|
|
|
expect(response.statusCode).toBe(404);
|
|
expect(await repository.getTable(database.id, table.id)).toMatchObject({
|
|
description: "Existing curated text",
|
|
generatedDescription: "Generated text",
|
|
version: edited!.version,
|
|
});
|
|
});
|
|
|
|
test("waits for confirmation and rescans before applying destructive changes", async () => {
|
|
const { app, repository, database, scan, setObserved } = await setup();
|
|
const first = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
|
await waitFor(repository, first.json().id, "succeeded");
|
|
const next = snapshot();
|
|
next.tables = next.tables.filter((table) => table.name !== "visits");
|
|
next.columns = next.columns.filter((column) => column.tableName !== "visits");
|
|
next.relationships = [];
|
|
setObserved(next);
|
|
|
|
const destructive = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
|
const waiting = await waitFor(repository, destructive.json().id, "awaiting_confirmation");
|
|
expect(waiting.plannedDiff).toMatchObject({ deletedTables: ["visits"] });
|
|
expect(await repository.listTables(database.id)).toHaveLength(2);
|
|
const confirmed = await app.inject({
|
|
method: "POST", url: `/catalog/sync-runs/${waiting.id}/confirm`, payload: { confirmationToken: waiting.confirmationToken },
|
|
});
|
|
expect(confirmed.statusCode).toBe(200);
|
|
await waitFor(repository, waiting.id, "succeeded");
|
|
expect((await repository.listTables(database.id)).map((table) => table.name)).toEqual(["patients"]);
|
|
expect(scan).toHaveBeenCalledTimes(3);
|
|
});
|
|
|
|
test("retries committed Memory cleanup with the original removals without rescanning", async () => {
|
|
const { app, repository, database, scan, setObserved, cleanup } = await setup();
|
|
await seedCatalog(repository, database);
|
|
const observed = snapshot();
|
|
observed.tables = observed.tables.filter(t => t.name !== "visits");
|
|
observed.columns = observed.columns.filter(c => c.tableName !== "visits");
|
|
observed.relationships = [];
|
|
setObserved(observed);
|
|
cleanup.mockResolvedValueOnce({ code: 0, stdout: JSON.stringify({ indexed: false, deleted: 2 }), stderr: "" });
|
|
const started = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`,
|
|
payload: { version: database.version, scope: "all", tableIds: [] } });
|
|
const waiting = await waitFor(repository, started.json().id, "awaiting_confirmation");
|
|
expect(cleanup).not.toHaveBeenCalled();
|
|
await app.inject({ method: "POST", url: `/catalog/sync-runs/${waiting.id}/confirm`,
|
|
payload: { confirmationToken: waiting.confirmationToken } });
|
|
const failed = await waitFor(repository, waiting.id, "failed");
|
|
expect(failed).toMatchObject({ phase: "memory_cleanup", errorCode: "memory_cleanup_pending",
|
|
plannedDiff: { deletedTables: ["visits"] } });
|
|
expect((await repository.listTables(database.id)).map(t => t.name)).toEqual(["patients"]);
|
|
const callsBeforeRetry = scan.mock.calls.length;
|
|
const blocked = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`,
|
|
payload: { version: database.version, scope: "all", tableIds: [] } });
|
|
expect(blocked.statusCode).toBe(409);
|
|
// Simulate startup recovery and an unreachable DWH after schema application.
|
|
await repository.interruptActiveSyncRuns();
|
|
scan.mockRejectedValue(new Error("DWH unavailable"));
|
|
cleanup.mockResolvedValue({ code: 0, stdout: JSON.stringify({ indexed: true, deleted: 2 }), stderr: "" });
|
|
const retried = await app.inject({ method: "POST", url: `/catalog/sync-runs/${waiting.id}/retry` });
|
|
expect(retried.statusCode).toBe(202);
|
|
const completed = await waitFor(repository, waiting.id, "succeeded");
|
|
expect(completed.counts).toMatchObject({ memoryDeleted: 2 });
|
|
expect(scan.mock.calls.length).toBe(callsBeforeRetry);
|
|
expect(cleanup).toHaveBeenCalledTimes(2);
|
|
expect(cleanup.mock.calls[1]).toEqual(cleanup.mock.calls[0]);
|
|
const request = JSON.parse((cleanup.mock.calls[0] as unknown as string[])[1]!).request;
|
|
expect(request).toMatchObject({ sync_id: waiting.id, database: "warehouse",
|
|
schema_name: "datawarehouse", removed_tables: ["visits"] });
|
|
});
|
|
|
|
test("deletes every catalog table for multiple selected databases and cascades dependent metadata", async () => {
|
|
const { app, repository, database } = await setup();
|
|
const second = await repository.create({
|
|
workspaceId: "radiology", engine: "postgres", databaseName: "radiology", schema: "public",
|
|
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
|
});
|
|
await seedCatalog(repository, database);
|
|
await seedCatalog(repository, second);
|
|
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: "/catalog/databases/metadata-cleanup",
|
|
payload: { databaseIds: [database.id, second.id], target: "tables" },
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(response.json()).toEqual({ tables: 4, columns: 6, relationships: 2 });
|
|
expect(await repository.get(database.id)).toBeDefined();
|
|
expect(await repository.get(second.id)).toBeDefined();
|
|
expect(await repository.listTables(database.id)).toEqual([]);
|
|
expect(await repository.listTables(second.id)).toEqual([]);
|
|
expect(await repository.listRelationships(database.id)).toEqual([]);
|
|
expect(await repository.listRelationships(second.id)).toEqual([]);
|
|
});
|
|
|
|
test("deletes relationships for selected databases without deleting their tables or columns", async () => {
|
|
const { app, repository, database } = await setup();
|
|
await seedCatalog(repository, database);
|
|
const tables = await repository.listTables(database.id);
|
|
const context = (await repository.getLogicalRelationshipContext(database.id))!;
|
|
const source = context.endpoints.find((endpoint) => (
|
|
endpoint.tableName === "visits" && endpoint.columnName === "id"
|
|
))!;
|
|
const target = context.endpoints.find((endpoint) => (
|
|
endpoint.tableName === "patients" && endpoint.columnName === "id"
|
|
))!;
|
|
await expect(repository.insertGeneratedLogicalRelationships(database.id, [{
|
|
sourceColumnId: source.columnId,
|
|
targetColumnId: target.columnId,
|
|
}])).resolves.toBe(1);
|
|
await expect(repository.getCatalogMetrics(database.id)).resolves.toMatchObject({ relationships: 2 });
|
|
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: "/catalog/databases/metadata-cleanup",
|
|
payload: { databaseIds: [database.id], target: "relationships" },
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(response.json()).toEqual({ tables: 0, columns: 0, relationships: 2 });
|
|
expect(await repository.listTables(database.id)).toHaveLength(2);
|
|
expect(await repository.listColumns(database.id, tables[0]!.id)).not.toEqual([]);
|
|
expect(await repository.listRelationships(database.id)).toEqual([]);
|
|
expect(await repository.listLogicalRelationships(database.id)).toEqual([]);
|
|
await expect(repository.getCatalogMetrics(database.id)).resolves.toMatchObject({ relationships: 0 });
|
|
expect((await repository.get(database.id))?.schemaSyncedVersion).toBeUndefined();
|
|
});
|
|
|
|
test("deletes columns from multiple selected catalog tables only", async () => {
|
|
const { app, repository, database } = await setup();
|
|
await seedCatalog(repository, database, expandedSnapshot());
|
|
const tables = await repository.listTables(database.id);
|
|
const patients = tables.find((table) => table.name === "patients")!;
|
|
const visits = tables.find((table) => table.name === "visits")!;
|
|
const encounters = tables.find((table) => table.name === "encounters")!;
|
|
const departments = tables.find((table) => table.name === "departments")!;
|
|
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/tables/metadata-cleanup`,
|
|
payload: { tableIds: [patients.id, visits.id], target: "columns" },
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(response.json()).toEqual({ tables: 0, columns: 3, relationships: 0 });
|
|
expect(await repository.listColumns(database.id, patients.id)).toEqual([]);
|
|
expect(await repository.listColumns(database.id, visits.id)).toEqual([]);
|
|
expect((await repository.listColumns(database.id, encounters.id)).map((column) => column.name))
|
|
.toEqual(["visit_id"]);
|
|
expect((await repository.listColumns(database.id, departments.id)).map((column) => column.name))
|
|
.toEqual(["patient_id"]);
|
|
expect((await repository.get(database.id))?.schemaSyncedVersion).toBeUndefined();
|
|
});
|
|
|
|
test("deletes incoming and outgoing relationships for multiple selected catalog tables only", async () => {
|
|
const { app, repository, database } = await setup();
|
|
await seedCatalog(repository, database, expandedSnapshot());
|
|
const tables = await repository.listTables(database.id);
|
|
const visits = tables.find((table) => table.name === "visits")!;
|
|
const encounters = tables.find((table) => table.name === "encounters")!;
|
|
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/tables/metadata-cleanup`,
|
|
payload: { tableIds: [visits.id, encounters.id], target: "relationships" },
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(response.json()).toEqual({ tables: 0, columns: 0, relationships: 2 });
|
|
expect((await repository.listRelationships(database.id)).map((relationship) => relationship.constraintName))
|
|
.toEqual(["departments_patient_id_fkey"]);
|
|
expect(await repository.listTables(database.id)).toHaveLength(4);
|
|
});
|
|
|
|
test("rejects a missing database without partially cleaning valid selections", async () => {
|
|
const { app, repository, database } = await setup();
|
|
await seedCatalog(repository, database);
|
|
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: "/catalog/databases/metadata-cleanup",
|
|
payload: {
|
|
databaseIds: [database.id, "99999999-9999-4999-8999-999999999999"],
|
|
target: "tables",
|
|
},
|
|
});
|
|
|
|
expect(response.statusCode).toBe(404);
|
|
expect(await repository.listTables(database.id)).toHaveLength(2);
|
|
});
|
|
|
|
test("serves one relationship map and supports the manual relationship lifecycle", async () => {
|
|
const { app, repository, database } = await setup();
|
|
await seedCatalog(repository, database);
|
|
const tables = await repository.listTables(database.id);
|
|
const patients = tables.find((table) => table.name === "patients")!;
|
|
const visits = tables.find((table) => table.name === "visits")!;
|
|
const patientId = (await repository.listColumns(database.id, patients.id)).find((item) => item.name === "id")!;
|
|
const visitId = (await repository.listColumns(database.id, visits.id)).find((item) => item.name === "id")!;
|
|
|
|
const created = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/relationships`,
|
|
payload: { sourceColumnId: visitId.id, targetColumnId: patientId.id },
|
|
});
|
|
expect(created.statusCode).toBe(201);
|
|
expect(created.json()).toMatchObject({ origin: "manual", status: "active", constraintName: null });
|
|
|
|
const listed = (await app.inject({
|
|
method: "GET", url: `/catalog/databases/${database.id}/relationships`,
|
|
})).json();
|
|
expect(listed.map((item: { origin: string }) => item.origin).sort()).toEqual(["manual", "physical"]);
|
|
|
|
const relationshipId = created.json().id;
|
|
const excluded = await app.inject({
|
|
method: "PATCH",
|
|
url: `/catalog/databases/${database.id}/relationships/${relationshipId}`,
|
|
payload: { status: "excluded" },
|
|
});
|
|
expect(excluded.statusCode).toBe(200);
|
|
expect(excluded.json()).toMatchObject({ origin: "manual", status: "excluded" });
|
|
|
|
const restored = await app.inject({
|
|
method: "PATCH",
|
|
url: `/catalog/databases/${database.id}/relationships/${relationshipId}`,
|
|
payload: { status: "active" },
|
|
});
|
|
expect(restored.json()).toMatchObject({ status: "active" });
|
|
|
|
expect((await app.inject({
|
|
method: "DELETE", url: `/catalog/databases/${database.id}/relationships/${relationshipId}`,
|
|
})).statusCode).toBe(204);
|
|
});
|
|
|
|
test("rejects generated inference while the Catalog schema is not current", async () => {
|
|
const { app, database } = await setup();
|
|
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/relationships/rebuild-generated`,
|
|
});
|
|
|
|
expect(response.statusCode).toBe(409);
|
|
expect(response.json()).toEqual({
|
|
code: "relationship_schema_stale",
|
|
message: "Synchronize the current database schema before managing logical relationships.",
|
|
});
|
|
});
|
|
|
|
test("rebuilds generated relationships and returns the exact summary", async () => {
|
|
const { app, repository, database } = await setup();
|
|
const observed = snapshot();
|
|
observed.relationships = [];
|
|
await seedCatalog(repository, database, observed);
|
|
|
|
const first = await app.inject({
|
|
method: "POST", url: `/catalog/databases/${database.id}/relationships/rebuild-generated`,
|
|
});
|
|
expect(first.statusCode).toBe(200);
|
|
expect(first.json()).toEqual({ added: 1, alreadyPresent: 0, excluded: 0, ambiguous: 0 });
|
|
const metrics = await app.inject({
|
|
method: "GET", url: `/catalog/metrics?databaseId=${database.id}`,
|
|
});
|
|
expect(metrics.statusCode).toBe(200);
|
|
expect(metrics.json()).toMatchObject({ relationships: 1 });
|
|
const generated = (await repository.listLogicalRelationships(database.id))[0]!;
|
|
|
|
await app.inject({
|
|
method: "PATCH",
|
|
url: `/catalog/databases/${database.id}/relationships/${generated.id}`,
|
|
payload: { status: "excluded" },
|
|
});
|
|
const second = await app.inject({
|
|
method: "POST", url: `/catalog/databases/${database.id}/relationships/rebuild-generated`,
|
|
});
|
|
expect(second.json()).toEqual({ added: 0, alreadyPresent: 0, excluded: 1, ambiguous: 0 });
|
|
});
|
|
|
|
test("validates relationship requests and keeps physical relationships read-only", async () => {
|
|
const { app, repository, database } = await setup();
|
|
await seedCatalog(repository, database);
|
|
const physical = (await repository.listRelationships(database.id))[0]!;
|
|
|
|
const invalid = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/relationships`,
|
|
payload: { sourceColumnId: physical.columns[0].sourceColumnId, targetColumnId: physical.columns[0].targetColumnId, generated: true },
|
|
});
|
|
expect(invalid.statusCode).toBe(400);
|
|
expect(invalid.json()).toMatchObject({ code: "relationship_request_invalid" });
|
|
|
|
const readOnly = await app.inject({
|
|
method: "PATCH",
|
|
url: `/catalog/databases/${database.id}/relationships/${physical.id}`,
|
|
payload: { status: "excluded" },
|
|
});
|
|
expect(readOnly.statusCode).toBe(409);
|
|
expect(readOnly.json()).toMatchObject({ code: "relationship_read_only" });
|
|
});
|
|
|
|
test("requires database.manage for relationship map mutations", async () => {
|
|
const { app, repository, database } = await setup({ AUTH_MODE: "upstream" });
|
|
const observed = snapshot();
|
|
observed.relationships = [];
|
|
await seedCatalog(repository, database, observed);
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/relationships/rebuild-generated`,
|
|
headers: {
|
|
"x-thoth-principal-issuer": "portal",
|
|
"x-thoth-principal-subject": "catalog-reader",
|
|
"x-thoth-is-admin": "0",
|
|
},
|
|
});
|
|
expect(response.statusCode).toBe(403);
|
|
expect(response.json()).toEqual({ code: "auth_forbidden", error: "This operation is not permitted" });
|
|
});
|