feat: implement metadata catalog database management
This commit is contained in:
Codex
@@ -0,0 +1,224 @@
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import { buildApp } from "../src/app.js";
|
||||
import { loadConfig } from "../src/config.js";
|
||||
import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
|
||||
import type { CatalogSchemaIntrospector } from "../src/catalog/schema-introspector.js";
|
||||
import type { CatalogSyncRun, ObservedSchemaSnapshot } from "../src/catalog/types.js";
|
||||
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||
import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/registry.js";
|
||||
import type { WorkspaceDescriptor } from "../src/workspaces/schema.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
|
||||
|
||||
const workspace: WorkspaceDescriptor = {
|
||||
workspace: { schema_version: 3, id: "psd-clinical", name: "Policlinico San Donato", language: "it" },
|
||||
dwh: { engine: "postgres", database: "warehouse", schema: "datawarehouse", port: 5432, supported_transports: ["postgres_direct"] },
|
||||
semantic_index: {
|
||||
vector_store: { engine: "qdrant", collection: "psd", dimensions: 1024, distance: "cosine" },
|
||||
embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
|
||||
},
|
||||
llm_policy: { allowed: ["zai/glm-5.2"] },
|
||||
};
|
||||
const revision: WorkspaceRevision = { id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/tmp/psd.yaml" };
|
||||
|
||||
function snapshot(): ObservedSchemaSnapshot {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||
tables: [
|
||||
{ name: "patients", sourceComment: "Clinical patients" },
|
||||
{ name: "visits", sourceComment: "Patient visits" },
|
||||
],
|
||||
columns: [
|
||||
{ tableName: "patients", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: "Patient key" },
|
||||
{ tableName: "visits", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
||||
{ tableName: "visits", name: "patient_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: "Owning patient" },
|
||||
],
|
||||
relationships: [{
|
||||
constraintName: "visits_patient_id_fkey",
|
||||
sourceTableName: "visits",
|
||||
targetTableName: "patients",
|
||||
updateRule: "NO ACTION",
|
||||
deleteRule: "CASCADE",
|
||||
deferrable: false,
|
||||
initiallyDeferred: false,
|
||||
columns: [{ position: 1, sourceColumnName: "patient_id", targetColumnName: "id" }],
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
async function waitFor(repository: MemoryCatalogRepository, runId: string, state: CatalogSyncRun["state"]): Promise<CatalogSyncRun> {
|
||||
for (let attempt = 0; attempt < 100; attempt += 1) {
|
||||
const run = await repository.getSyncRun(runId);
|
||||
if (run?.state === state) return run;
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
throw new Error(`Run ${runId} did not reach ${state}`);
|
||||
}
|
||||
|
||||
async function setup() {
|
||||
const secretRoot = mkdtempSync(join(tmpdir(), "catalog-schema-secret-"));
|
||||
const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-schema-runtime-"));
|
||||
roots.push(secretRoot, runtimeRoot);
|
||||
const repository = new MemoryCatalogRepository();
|
||||
const created = await repository.create({
|
||||
workspaceId: "psd-clinical", engine: "postgres", databaseName: "warehouse", schema: "datawarehouse",
|
||||
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
||||
});
|
||||
await repository.recordTest(created.id, created.version, {
|
||||
connectionStatus: "reachable", testedVersion: created.version, lastTestedAt: new Date().toISOString(),
|
||||
});
|
||||
let observed = snapshot();
|
||||
const scan = vi.fn(async (_database, _signal, progress) => {
|
||||
await progress?.("connecting");
|
||||
await progress?.("scanning_tables", { tables: observed.tables.length });
|
||||
await progress?.("scanning_columns", { tables: observed.tables.length, columns: observed.columns.length });
|
||||
await progress?.("scanning_relationships", { relationships: observed.relationships.length });
|
||||
return structuredClone(observed);
|
||||
});
|
||||
const introspector: CatalogSchemaIntrospector = { scan };
|
||||
const registry = {
|
||||
list: vi.fn(async () => [revision]),
|
||||
listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]),
|
||||
read: vi.fn(async () => ({ workspace, revision })),
|
||||
} as unknown as WorkspaceRegistry;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test" }), {
|
||||
thtRunner: {} as never,
|
||||
workspaceRegistry: registry,
|
||||
workspaceSecretStore: new WorkspaceSecretStore({ root: secretRoot, runtimeRoot, installationId: "test" }),
|
||||
catalogRepository: repository,
|
||||
catalogSchemaIntrospector: introspector,
|
||||
workspaceDiagnoser: vi.fn(),
|
||||
});
|
||||
return {
|
||||
app, repository, database: (await repository.get(created.id))!, scan,
|
||||
setObserved(next: ObservedSchemaSnapshot) { observed = next; },
|
||||
};
|
||||
}
|
||||
|
||||
test("synchronizes a full physical schema and derives primary and foreign key flags", async () => {
|
||||
const { app, repository, database } = await setup();
|
||||
const started = await app.inject({
|
||||
method: "POST", url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version, scope: "all", tableIds: [] },
|
||||
});
|
||||
expect(started.statusCode).toBe(202);
|
||||
const completed = await waitFor(repository, started.json().id, "succeeded");
|
||||
expect(completed.counts).toMatchObject({ tables: 2, columns: 3, relationships: 1 });
|
||||
|
||||
const tables = await repository.listTables(database.id);
|
||||
const visits = tables.find((table) => table.name === "visits")!;
|
||||
const columns = (await app.inject({ method: "GET", url: `/catalog/databases/${database.id}/tables/${visits.id}/columns` })).json();
|
||||
expect(columns).toMatchObject([
|
||||
{ name: "id", isPrimaryKey: true, primaryKeyPosition: 1, isForeignKey: false },
|
||||
{ name: "patient_id", isPrimaryKey: false, isForeignKey: true, foreignKeyCount: 1 },
|
||||
]);
|
||||
const relationships = (await app.inject({ method: "GET", url: `/catalog/databases/${database.id}/relationships` })).json();
|
||||
expect(relationships).toMatchObject([{ constraintName: "visits_patient_id_fkey", columns: [{ sourceColumnName: "patient_id", targetColumnName: "id" }] }]);
|
||||
expect((await repository.get(database.id))?.schemaSyncedVersion).toBe(database.version);
|
||||
});
|
||||
|
||||
test("synchronizes columns for every catalog table when no table selection is supplied", async () => {
|
||||
const { app, repository, database, setObserved } = await setup();
|
||||
const tablesRun = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version, scope: "tables", tableIds: [] },
|
||||
});
|
||||
expect(tablesRun.statusCode).toBe(202);
|
||||
await waitFor(repository, tablesRun.json().id, "succeeded");
|
||||
const tables = await repository.listTables(database.id);
|
||||
expect(tables.map((table) => table.name)).toEqual(["patients", "visits"]);
|
||||
|
||||
const columnsRun = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version, scope: "columns", tableIds: [] },
|
||||
});
|
||||
expect(columnsRun.statusCode).toBe(202);
|
||||
await waitFor(repository, columnsRun.json().id, "succeeded");
|
||||
const patients = tables.find((table) => table.name === "patients")!;
|
||||
const visits = tables.find((table) => table.name === "visits")!;
|
||||
expect((await repository.listColumns(database.id, patients.id)).map((column) => column.name)).toEqual(["id"]);
|
||||
expect((await repository.listColumns(database.id, visits.id)).map((column) => column.name)).toEqual(["id", "patient_id"]);
|
||||
|
||||
const next = snapshot();
|
||||
next.columns = next.columns.filter((column) => column.name !== "id");
|
||||
setObserved(next);
|
||||
const selectedDestructiveRun = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version, scope: "columns", tableIds: [patients.id] },
|
||||
});
|
||||
expect(selectedDestructiveRun.statusCode).toBe(202);
|
||||
const selectedWaiting = await waitFor(repository, selectedDestructiveRun.json().id, "awaiting_confirmation");
|
||||
expect(selectedWaiting.plannedDiff?.deletedColumns).toEqual([
|
||||
{ tableName: "patients", columnName: "id" },
|
||||
]);
|
||||
expect((await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/sync-runs/${selectedWaiting.id}/cancel`,
|
||||
})).statusCode).toBe(200);
|
||||
|
||||
const destructiveRun = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version, scope: "columns", tableIds: [] },
|
||||
});
|
||||
expect(destructiveRun.statusCode).toBe(202);
|
||||
const waiting = await waitFor(repository, destructiveRun.json().id, "awaiting_confirmation");
|
||||
expect(waiting.plannedDiff?.deletedColumns).toEqual([
|
||||
{ tableName: "patients", columnName: "id" },
|
||||
{ tableName: "visits", columnName: "id" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("keeps generated descriptions editable and preserves them across synchronization", async () => {
|
||||
const { app, repository, database } = await setup();
|
||||
const first = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
||||
await waitFor(repository, first.json().id, "succeeded");
|
||||
const patients = (await repository.listTables(database.id)).find((table) => table.name === "patients")!;
|
||||
const editedTable = await app.inject({
|
||||
method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}`,
|
||||
payload: { version: patients.version, description: null, generatedDescription: "Generated table draft" },
|
||||
});
|
||||
expect(editedTable.json()).toMatchObject({ description: null, generatedDescription: "Generated table draft" });
|
||||
const idColumn = (await repository.listColumns(database.id, patients.id))[0];
|
||||
const editedColumn = await app.inject({
|
||||
method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}/columns/${idColumn.id}`,
|
||||
payload: { version: idColumn.version, description: "Reviewed key", generatedDescription: "Generated key draft" },
|
||||
});
|
||||
expect(editedColumn.json()).toMatchObject({ description: "Reviewed key", generatedDescription: "Generated key draft" });
|
||||
|
||||
const second = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
||||
await waitFor(repository, second.json().id, "succeeded");
|
||||
expect(await repository.getTable(database.id, patients.id)).toMatchObject({ generatedDescription: "Generated table draft" });
|
||||
expect(await repository.getColumn(database.id, patients.id, idColumn.id)).toMatchObject({ description: "Reviewed key", generatedDescription: "Generated key draft" });
|
||||
});
|
||||
|
||||
test("waits for confirmation and rescans before applying destructive changes", async () => {
|
||||
const { app, repository, database, scan, setObserved } = await setup();
|
||||
const first = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
||||
await waitFor(repository, first.json().id, "succeeded");
|
||||
const next = snapshot();
|
||||
next.tables = next.tables.filter((table) => table.name !== "visits");
|
||||
next.columns = next.columns.filter((column) => column.tableName !== "visits");
|
||||
next.relationships = [];
|
||||
setObserved(next);
|
||||
|
||||
const destructive = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
||||
const waiting = await waitFor(repository, destructive.json().id, "awaiting_confirmation");
|
||||
expect(waiting.plannedDiff).toMatchObject({ deletedTables: ["visits"] });
|
||||
expect(await repository.listTables(database.id)).toHaveLength(2);
|
||||
const confirmed = await app.inject({
|
||||
method: "POST", url: `/catalog/sync-runs/${waiting.id}/confirm`, payload: { confirmationToken: waiting.confirmationToken },
|
||||
});
|
||||
expect(confirmed.statusCode).toBe(200);
|
||||
await waitFor(repository, waiting.id, "succeeded");
|
||||
expect((await repository.listTables(database.id)).map((table) => table.name)).toEqual(["patients"]);
|
||||
expect(scan).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
Reference in New Issue
Block a user