375 lines
19 KiB
TypeScript
375 lines
19 KiB
TypeScript
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { afterEach, expect, test, vi } from "vitest";
|
|
import { buildApp } from "../src/app.js";
|
|
import { loadConfig } from "../src/config.js";
|
|
import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
|
|
import type { CatalogSchemaIntrospector } from "../src/catalog/schema-introspector.js";
|
|
import type { CatalogSyncRun, ObservedSchemaSnapshot } from "../src/catalog/types.js";
|
|
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
|
import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/registry.js";
|
|
import type { WorkspaceDescriptor } from "../src/workspaces/schema.js";
|
|
|
|
const roots: string[] = [];
|
|
afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
|
|
|
|
const workspace: WorkspaceDescriptor = {
|
|
workspace: { schema_version: 3, id: "psd-clinical", name: "Policlinico San Donato", language: "it" },
|
|
dwh: { engine: "postgres", database: "warehouse", schema: "datawarehouse", port: 5432, supported_transports: ["postgres_direct"] },
|
|
semantic_index: {
|
|
vector_store: { engine: "qdrant", collection: "psd", dimensions: 1024, distance: "cosine" },
|
|
embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
|
|
},
|
|
llm_policy: { allowed: ["zai/glm-5.2"] },
|
|
};
|
|
const revision: WorkspaceRevision = { id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/tmp/psd.yaml" };
|
|
|
|
function snapshot(): ObservedSchemaSnapshot {
|
|
return {
|
|
schemaVersion: 1,
|
|
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
|
tables: [
|
|
{ name: "patients", sourceComment: "Clinical patients" },
|
|
{ name: "visits", sourceComment: "Patient visits" },
|
|
],
|
|
columns: [
|
|
{ tableName: "patients", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: "Patient key" },
|
|
{ tableName: "visits", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
|
{ tableName: "visits", name: "patient_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: "Owning patient" },
|
|
],
|
|
relationships: [{
|
|
constraintName: "visits_patient_id_fkey",
|
|
sourceTableName: "visits",
|
|
targetTableName: "patients",
|
|
updateRule: "NO ACTION",
|
|
deleteRule: "CASCADE",
|
|
deferrable: false,
|
|
initiallyDeferred: false,
|
|
columns: [{ position: 1, sourceColumnName: "patient_id", targetColumnName: "id" }],
|
|
}],
|
|
};
|
|
}
|
|
|
|
function expandedSnapshot(): ObservedSchemaSnapshot {
|
|
const expanded = snapshot();
|
|
expanded.tables.push(
|
|
{ name: "encounters", sourceComment: null },
|
|
{ name: "departments", sourceComment: null },
|
|
);
|
|
expanded.columns.push(
|
|
{ tableName: "encounters", name: "visit_id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
|
{ tableName: "departments", name: "patient_id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
|
);
|
|
expanded.relationships.push(
|
|
{
|
|
constraintName: "encounters_visit_id_fkey",
|
|
sourceTableName: "encounters",
|
|
targetTableName: "visits",
|
|
updateRule: "NO ACTION",
|
|
deleteRule: "CASCADE",
|
|
deferrable: false,
|
|
initiallyDeferred: false,
|
|
columns: [{ position: 1, sourceColumnName: "visit_id", targetColumnName: "id" }],
|
|
},
|
|
{
|
|
constraintName: "departments_patient_id_fkey",
|
|
sourceTableName: "departments",
|
|
targetTableName: "patients",
|
|
updateRule: "NO ACTION",
|
|
deleteRule: "CASCADE",
|
|
deferrable: false,
|
|
initiallyDeferred: false,
|
|
columns: [{ position: 1, sourceColumnName: "patient_id", targetColumnName: "id" }],
|
|
},
|
|
);
|
|
return expanded;
|
|
}
|
|
|
|
async function waitFor(repository: MemoryCatalogRepository, runId: string, state: CatalogSyncRun["state"]): Promise<CatalogSyncRun> {
|
|
for (let attempt = 0; attempt < 100; attempt += 1) {
|
|
const run = await repository.getSyncRun(runId);
|
|
if (run?.state === state) return run;
|
|
await new Promise((resolve) => setTimeout(resolve, 5));
|
|
}
|
|
throw new Error(`Run ${runId} did not reach ${state}`);
|
|
}
|
|
|
|
async function setup() {
|
|
const secretRoot = mkdtempSync(join(tmpdir(), "catalog-schema-secret-"));
|
|
const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-schema-runtime-"));
|
|
roots.push(secretRoot, runtimeRoot);
|
|
const repository = new MemoryCatalogRepository();
|
|
const created = await repository.create({
|
|
workspaceId: "psd-clinical", engine: "postgres", databaseName: "warehouse", schema: "datawarehouse",
|
|
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
|
});
|
|
await repository.recordTest(created.id, created.version, {
|
|
connectionStatus: "reachable", testedVersion: created.version, lastTestedAt: new Date().toISOString(),
|
|
});
|
|
let observed = snapshot();
|
|
const scan = vi.fn(async (_database, _signal, progress) => {
|
|
await progress?.("connecting");
|
|
await progress?.("scanning_tables", { tables: observed.tables.length });
|
|
await progress?.("scanning_columns", { tables: observed.tables.length, columns: observed.columns.length });
|
|
await progress?.("scanning_relationships", { relationships: observed.relationships.length });
|
|
return structuredClone(observed);
|
|
});
|
|
const introspector: CatalogSchemaIntrospector = { scan };
|
|
const registry = {
|
|
list: vi.fn(async () => [revision]),
|
|
listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]),
|
|
read: vi.fn(async () => ({ workspace, revision })),
|
|
} as unknown as WorkspaceRegistry;
|
|
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test" }), {
|
|
thtRunner: {} as never,
|
|
workspaceRegistry: registry,
|
|
workspaceSecretStore: new WorkspaceSecretStore({ root: secretRoot, runtimeRoot, installationId: "test" }),
|
|
catalogRepository: repository,
|
|
catalogSchemaIntrospector: introspector,
|
|
workspaceDiagnoser: vi.fn(),
|
|
});
|
|
return {
|
|
app, repository, database: (await repository.get(created.id))!, scan,
|
|
setObserved(next: ObservedSchemaSnapshot) { observed = next; },
|
|
};
|
|
}
|
|
|
|
async function seedCatalog(
|
|
repository: MemoryCatalogRepository,
|
|
database: { id: string; version: number },
|
|
observed: ObservedSchemaSnapshot = snapshot(),
|
|
): Promise<void> {
|
|
await repository.applySchemaSync(database.id, database.version, "all", [], observed);
|
|
}
|
|
|
|
test("synchronizes a full physical schema and derives primary and foreign key flags", async () => {
|
|
const { app, repository, database } = await setup();
|
|
const started = await app.inject({
|
|
method: "POST", url: `/catalog/databases/${database.id}/sync-runs`,
|
|
payload: { version: database.version, scope: "all", tableIds: [] },
|
|
});
|
|
expect(started.statusCode).toBe(202);
|
|
const completed = await waitFor(repository, started.json().id, "succeeded");
|
|
expect(completed.counts).toMatchObject({ tables: 2, columns: 3, relationships: 1 });
|
|
|
|
const tables = await repository.listTables(database.id);
|
|
const visits = tables.find((table) => table.name === "visits")!;
|
|
const columns = (await app.inject({ method: "GET", url: `/catalog/databases/${database.id}/tables/${visits.id}/columns` })).json();
|
|
expect(columns).toMatchObject([
|
|
{ name: "id", isPrimaryKey: true, primaryKeyPosition: 1, isForeignKey: false },
|
|
{ name: "patient_id", isPrimaryKey: false, isForeignKey: true, foreignKeyCount: 1 },
|
|
]);
|
|
const relationships = (await app.inject({ method: "GET", url: `/catalog/databases/${database.id}/relationships` })).json();
|
|
expect(relationships).toMatchObject([{ constraintName: "visits_patient_id_fkey", columns: [{ sourceColumnName: "patient_id", targetColumnName: "id" }] }]);
|
|
expect((await repository.get(database.id))?.schemaSyncedVersion).toBe(database.version);
|
|
});
|
|
|
|
test("synchronizes columns for every catalog table when no table selection is supplied", async () => {
|
|
const { app, repository, database, setObserved } = await setup();
|
|
const tablesRun = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/sync-runs`,
|
|
payload: { version: database.version, scope: "tables", tableIds: [] },
|
|
});
|
|
expect(tablesRun.statusCode).toBe(202);
|
|
await waitFor(repository, tablesRun.json().id, "succeeded");
|
|
const tables = await repository.listTables(database.id);
|
|
expect(tables.map((table) => table.name)).toEqual(["patients", "visits"]);
|
|
|
|
const columnsRun = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/sync-runs`,
|
|
payload: { version: database.version, scope: "columns", tableIds: [] },
|
|
});
|
|
expect(columnsRun.statusCode).toBe(202);
|
|
await waitFor(repository, columnsRun.json().id, "succeeded");
|
|
const patients = tables.find((table) => table.name === "patients")!;
|
|
const visits = tables.find((table) => table.name === "visits")!;
|
|
expect((await repository.listColumns(database.id, patients.id)).map((column) => column.name)).toEqual(["id"]);
|
|
expect((await repository.listColumns(database.id, visits.id)).map((column) => column.name)).toEqual(["id", "patient_id"]);
|
|
|
|
const next = snapshot();
|
|
next.columns = next.columns.filter((column) => column.name !== "id");
|
|
setObserved(next);
|
|
const selectedDestructiveRun = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/sync-runs`,
|
|
payload: { version: database.version, scope: "columns", tableIds: [patients.id] },
|
|
});
|
|
expect(selectedDestructiveRun.statusCode).toBe(202);
|
|
const selectedWaiting = await waitFor(repository, selectedDestructiveRun.json().id, "awaiting_confirmation");
|
|
expect(selectedWaiting.plannedDiff?.deletedColumns).toEqual([
|
|
{ tableName: "patients", columnName: "id" },
|
|
]);
|
|
expect((await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/sync-runs/${selectedWaiting.id}/cancel`,
|
|
})).statusCode).toBe(200);
|
|
|
|
const destructiveRun = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/sync-runs`,
|
|
payload: { version: database.version, scope: "columns", tableIds: [] },
|
|
});
|
|
expect(destructiveRun.statusCode).toBe(202);
|
|
const waiting = await waitFor(repository, destructiveRun.json().id, "awaiting_confirmation");
|
|
expect(waiting.plannedDiff?.deletedColumns).toEqual([
|
|
{ tableName: "patients", columnName: "id" },
|
|
{ tableName: "visits", columnName: "id" },
|
|
]);
|
|
});
|
|
|
|
test("keeps generated descriptions editable and preserves them across synchronization", async () => {
|
|
const { app, repository, database } = await setup();
|
|
const first = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
|
await waitFor(repository, first.json().id, "succeeded");
|
|
const patients = (await repository.listTables(database.id)).find((table) => table.name === "patients")!;
|
|
const editedTable = await app.inject({
|
|
method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}`,
|
|
payload: { version: patients.version, description: null, generatedDescription: "Generated table draft" },
|
|
});
|
|
expect(editedTable.json()).toMatchObject({ description: null, generatedDescription: "Generated table draft" });
|
|
const idColumn = (await repository.listColumns(database.id, patients.id))[0];
|
|
const editedColumn = await app.inject({
|
|
method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}/columns/${idColumn.id}`,
|
|
payload: { version: idColumn.version, description: "Reviewed key", generatedDescription: "Generated key draft" },
|
|
});
|
|
expect(editedColumn.json()).toMatchObject({ description: "Reviewed key", generatedDescription: "Generated key draft" });
|
|
|
|
const second = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
|
await waitFor(repository, second.json().id, "succeeded");
|
|
expect(await repository.getTable(database.id, patients.id)).toMatchObject({ generatedDescription: "Generated table draft" });
|
|
expect(await repository.getColumn(database.id, patients.id, idColumn.id)).toMatchObject({ description: "Reviewed key", generatedDescription: "Generated key draft" });
|
|
});
|
|
|
|
test("waits for confirmation and rescans before applying destructive changes", async () => {
|
|
const { app, repository, database, scan, setObserved } = await setup();
|
|
const first = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
|
await waitFor(repository, first.json().id, "succeeded");
|
|
const next = snapshot();
|
|
next.tables = next.tables.filter((table) => table.name !== "visits");
|
|
next.columns = next.columns.filter((column) => column.tableName !== "visits");
|
|
next.relationships = [];
|
|
setObserved(next);
|
|
|
|
const destructive = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
|
const waiting = await waitFor(repository, destructive.json().id, "awaiting_confirmation");
|
|
expect(waiting.plannedDiff).toMatchObject({ deletedTables: ["visits"] });
|
|
expect(await repository.listTables(database.id)).toHaveLength(2);
|
|
const confirmed = await app.inject({
|
|
method: "POST", url: `/catalog/sync-runs/${waiting.id}/confirm`, payload: { confirmationToken: waiting.confirmationToken },
|
|
});
|
|
expect(confirmed.statusCode).toBe(200);
|
|
await waitFor(repository, waiting.id, "succeeded");
|
|
expect((await repository.listTables(database.id)).map((table) => table.name)).toEqual(["patients"]);
|
|
expect(scan).toHaveBeenCalledTimes(3);
|
|
});
|
|
|
|
test("deletes every catalog table for multiple selected databases and cascades dependent metadata", async () => {
|
|
const { app, repository, database } = await setup();
|
|
const second = await repository.create({
|
|
workspaceId: "radiology", engine: "postgres", databaseName: "radiology", schema: "public",
|
|
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
|
});
|
|
await seedCatalog(repository, database);
|
|
await seedCatalog(repository, second);
|
|
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: "/catalog/databases/metadata-cleanup",
|
|
payload: { databaseIds: [database.id, second.id], target: "tables" },
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(response.json()).toEqual({ tables: 4, columns: 6, relationships: 2 });
|
|
expect(await repository.get(database.id)).toBeDefined();
|
|
expect(await repository.get(second.id)).toBeDefined();
|
|
expect(await repository.listTables(database.id)).toEqual([]);
|
|
expect(await repository.listTables(second.id)).toEqual([]);
|
|
expect(await repository.listRelationships(database.id)).toEqual([]);
|
|
expect(await repository.listRelationships(second.id)).toEqual([]);
|
|
});
|
|
|
|
test("deletes relationships for selected databases without deleting their tables or columns", async () => {
|
|
const { app, repository, database } = await setup();
|
|
await seedCatalog(repository, database);
|
|
const tables = await repository.listTables(database.id);
|
|
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: "/catalog/databases/metadata-cleanup",
|
|
payload: { databaseIds: [database.id], target: "relationships" },
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(response.json()).toEqual({ tables: 0, columns: 0, relationships: 1 });
|
|
expect(await repository.listTables(database.id)).toHaveLength(2);
|
|
expect(await repository.listColumns(database.id, tables[0]!.id)).not.toEqual([]);
|
|
expect(await repository.listRelationships(database.id)).toEqual([]);
|
|
expect((await repository.get(database.id))?.schemaSyncedVersion).toBeUndefined();
|
|
});
|
|
|
|
test("deletes columns from multiple selected catalog tables only", async () => {
|
|
const { app, repository, database } = await setup();
|
|
await seedCatalog(repository, database, expandedSnapshot());
|
|
const tables = await repository.listTables(database.id);
|
|
const patients = tables.find((table) => table.name === "patients")!;
|
|
const visits = tables.find((table) => table.name === "visits")!;
|
|
const encounters = tables.find((table) => table.name === "encounters")!;
|
|
const departments = tables.find((table) => table.name === "departments")!;
|
|
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/tables/metadata-cleanup`,
|
|
payload: { tableIds: [patients.id, visits.id], target: "columns" },
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(response.json()).toEqual({ tables: 0, columns: 3, relationships: 0 });
|
|
expect(await repository.listColumns(database.id, patients.id)).toEqual([]);
|
|
expect(await repository.listColumns(database.id, visits.id)).toEqual([]);
|
|
expect((await repository.listColumns(database.id, encounters.id)).map((column) => column.name))
|
|
.toEqual(["visit_id"]);
|
|
expect((await repository.listColumns(database.id, departments.id)).map((column) => column.name))
|
|
.toEqual(["patient_id"]);
|
|
expect((await repository.get(database.id))?.schemaSyncedVersion).toBeUndefined();
|
|
});
|
|
|
|
test("deletes incoming and outgoing relationships for multiple selected catalog tables only", async () => {
|
|
const { app, repository, database } = await setup();
|
|
await seedCatalog(repository, database, expandedSnapshot());
|
|
const tables = await repository.listTables(database.id);
|
|
const visits = tables.find((table) => table.name === "visits")!;
|
|
const encounters = tables.find((table) => table.name === "encounters")!;
|
|
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: `/catalog/databases/${database.id}/tables/metadata-cleanup`,
|
|
payload: { tableIds: [visits.id, encounters.id], target: "relationships" },
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(response.json()).toEqual({ tables: 0, columns: 0, relationships: 2 });
|
|
expect((await repository.listRelationships(database.id)).map((relationship) => relationship.constraintName))
|
|
.toEqual(["departments_patient_id_fkey"]);
|
|
expect(await repository.listTables(database.id)).toHaveLength(4);
|
|
});
|
|
|
|
test("rejects a missing database without partially cleaning valid selections", async () => {
|
|
const { app, repository, database } = await setup();
|
|
await seedCatalog(repository, database);
|
|
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: "/catalog/databases/metadata-cleanup",
|
|
payload: {
|
|
databaseIds: [database.id, "99999999-9999-4999-8999-999999999999"],
|
|
target: "tables",
|
|
},
|
|
});
|
|
|
|
expect(response.statusCode).toBe(404);
|
|
expect(await repository.listTables(database.id)).toHaveLength(2);
|
|
});
|