feat: implement metadata catalog database management
This commit is contained in:
Codex
@@ -187,6 +187,7 @@ test("roles collapse duplicates and admin contains all administrative permission
|
||||
"settings.manage",
|
||||
"workspace.manage",
|
||||
"workspace.secrets.manage",
|
||||
"database.manage",
|
||||
"pi.manage",
|
||||
"auth.diagnostics.read",
|
||||
]);
|
||||
|
||||
@@ -114,7 +114,7 @@ test.each([
|
||||
const created = await fixture.app.thothiiAuthSessionStore?.create({
|
||||
principal: {
|
||||
issuer: "local", subject: user.id, displayName: user.username, roles: ["admin"],
|
||||
permissions: ["session.use", "session.read_all", "session.manage_all", "settings.manage", "workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read"],
|
||||
permissions: ["session.use", "session.read_all", "session.manage_all", "settings.manage", "workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read"],
|
||||
isAdmin: true,
|
||||
},
|
||||
method: "local",
|
||||
|
||||
@@ -130,7 +130,7 @@ test("local login sets a non-persistent opaque session cookie and exposes only a
|
||||
roles: ["admin"],
|
||||
permissions: [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
],
|
||||
isAdmin: true,
|
||||
csrfToken: expect.stringMatching(/^[A-Za-z0-9_-]{43}$/),
|
||||
|
||||
@@ -32,7 +32,7 @@ test("local mode resolves a stable local principal", async () => {
|
||||
roles: ["admin"],
|
||||
permissions: [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
],
|
||||
isAdmin: true,
|
||||
});
|
||||
@@ -74,7 +74,7 @@ test("upstream mode accepts only normalized proxy principal headers", async () =
|
||||
issuer: "portal", subject: "42", displayName: "Alice", roles: ["user", "admin"],
|
||||
permissions: [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
],
|
||||
isAdmin: true,
|
||||
});
|
||||
|
||||
@@ -15,14 +15,14 @@ const admin: PrincipalContext = {
|
||||
issuer: "oidc", subject: "admin", roles: ["admin"],
|
||||
permissions: [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
],
|
||||
isAdmin: true,
|
||||
};
|
||||
|
||||
const catalog: readonly Permission[] = [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
];
|
||||
|
||||
test("permission matrix gives role-less identities no access, users session use, and admins every catalog permission", () => {
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import { buildApp } from "../src/app.js";
|
||||
import { loadConfig } from "../src/config.js";
|
||||
import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
|
||||
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||
import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/registry.js";
|
||||
import type { WorkspaceDescriptor } from "../src/workspaces/schema.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
|
||||
|
||||
const workspace: WorkspaceDescriptor = {
|
||||
workspace: { schema_version: 3, id: "psd-clinical", name: "Policlinico San Donato", language: "it" },
|
||||
dwh: {
|
||||
engine: "postgres", database: "warehouse", schema: "datawarehouse", port: 5432,
|
||||
supported_transports: ["postgres_direct", "rest_api"],
|
||||
},
|
||||
semantic_index: {
|
||||
vector_store: { engine: "qdrant", collection: "psd", dimensions: 1024, distance: "cosine" },
|
||||
embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
|
||||
},
|
||||
llm_policy: { allowed: ["zai/glm-5.2"] },
|
||||
diagnostics: { dwh_rest: { method: "GET", path: "/health", auth: "bearer", response: { database: "database", schema: "schema" } } },
|
||||
};
|
||||
const revision: WorkspaceRevision = { id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/tmp/psd.yaml" };
|
||||
|
||||
function setup() {
|
||||
const secretRoot = mkdtempSync(join(tmpdir(), "catalog-secret-"));
|
||||
const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-secret-runtime-"));
|
||||
roots.push(secretRoot, runtimeRoot);
|
||||
const secretStore = new WorkspaceSecretStore({ root: secretRoot, runtimeRoot, installationId: "test" });
|
||||
const repository = new MemoryCatalogRepository();
|
||||
const registry = {
|
||||
list: vi.fn(async () => [revision]),
|
||||
listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]),
|
||||
read: vi.fn(async () => ({ workspace, revision })),
|
||||
} as unknown as WorkspaceRegistry;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test" }), {
|
||||
thtRunner: {} as never,
|
||||
workspaceRegistry: registry,
|
||||
workspaceSecretStore: secretStore,
|
||||
catalogRepository: repository,
|
||||
workspaceDiagnoser: vi.fn(),
|
||||
});
|
||||
return { app, secretStore, repository };
|
||||
}
|
||||
|
||||
const direct = {
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "datawarehouse",
|
||||
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
||||
};
|
||||
|
||||
test("lists every YAML workspace and creates its one database configuration", async () => {
|
||||
const { app } = setup();
|
||||
const initial = await app.inject({ method: "GET", url: "/catalog/databases" });
|
||||
expect(initial.statusCode).toBe(200);
|
||||
expect(initial.json()).toMatchObject([{ workspaceId: "psd-clinical", configured: false, databaseName: "warehouse" }]);
|
||||
|
||||
const created = await app.inject({ method: "POST", url: "/catalog/databases", payload: direct });
|
||||
expect(created.statusCode).toBe(201);
|
||||
expect(created.json()).toMatchObject({ configured: true, workspaceId: "psd-clinical", version: 1 });
|
||||
expect((await app.inject({ method: "POST", url: "/catalog/databases", payload: direct })).statusCode).toBe(409);
|
||||
|
||||
const listed = await app.inject({ method: "GET", url: "/catalog/databases" });
|
||||
expect(listed.json()).toMatchObject([{ configured: true, binding: { transport: "postgres_direct", host: "db.internal" } }]);
|
||||
});
|
||||
|
||||
test("lists orphaned records and takes the REST diagnostic path from workspace YAML", async () => {
|
||||
const { app, repository } = setup();
|
||||
await repository.create({
|
||||
workspaceId: "removed-workspace",
|
||||
engine: "postgres",
|
||||
databaseName: "legacy",
|
||||
schema: "public",
|
||||
binding: { transport: "postgres_direct", host: "legacy.internal", port: 5432, username: "reader" },
|
||||
});
|
||||
const created = await app.inject({
|
||||
method: "POST",
|
||||
url: "/catalog/databases",
|
||||
payload: {
|
||||
...direct,
|
||||
binding: {
|
||||
transport: "rest_api", baseUrl: "https://psd.example/api", restPath: "/client-controlled", restAuth: "bearer",
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(created.statusCode).toBe(201);
|
||||
expect(created.json()).toMatchObject({ binding: { restPath: "/health" } });
|
||||
|
||||
const rows = (await app.inject({ method: "GET", url: "/catalog/databases" })).json();
|
||||
expect(rows).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ workspaceId: "removed-workspace", configured: true, workspaceAvailable: false }),
|
||||
expect.objectContaining({ workspaceId: "psd-clinical", configured: true, workspaceAvailable: true }),
|
||||
]));
|
||||
});
|
||||
|
||||
test("uses optimistic versions, keeps secrets write-only, and hard-deletes only local configuration", async () => {
|
||||
const { app, secretStore } = setup();
|
||||
const created = (await app.inject({ method: "POST", url: "/catalog/databases", payload: direct })).json();
|
||||
const stale = await app.inject({ method: "PATCH", url: `/catalog/databases/${created.id}`, payload: { ...direct, version: 99 } });
|
||||
expect(stale.statusCode).toBe(409);
|
||||
|
||||
const secret = await app.inject({
|
||||
method: "PUT", url: `/catalog/databases/${created.id}/secrets`,
|
||||
payload: { version: 1, values: { password: "do-not-return-this" } },
|
||||
});
|
||||
expect(secret.statusCode).toBe(200);
|
||||
expect(secret.body).not.toContain("do-not-return-this");
|
||||
expect(secret.json()).toMatchObject({ version: 2, secrets: { password: true } });
|
||||
expect(secretStore.has("psd-clinical", "catalog.dwh.password")).toBe(true);
|
||||
|
||||
const removed = await app.inject({ method: "DELETE", url: `/catalog/databases/${created.id}?version=2` });
|
||||
expect(removed.statusCode).toBe(204);
|
||||
expect(secretStore.has("psd-clinical", "catalog.dwh.password")).toBe(false);
|
||||
expect((await app.inject({ method: "GET", url: "/catalog/databases" })).json()).toMatchObject([{ configured: false }]);
|
||||
});
|
||||
@@ -0,0 +1,171 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import { existsSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { PassThrough } from "node:stream";
|
||||
import type { ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
import type { Client, ClientConfig } from "pg";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import {
|
||||
buildSshArguments,
|
||||
ConcreteCatalogPostgresAccess,
|
||||
} from "../src/catalog/postgres-access.js";
|
||||
import { CATALOG_SECRET_IDS } from "../src/catalog/secrets.js";
|
||||
import type { WorkspaceDatabase } from "../src/catalog/types.js";
|
||||
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function secretStore() {
|
||||
const root = mkdtempSync(join(tmpdir(), "catalog-ssh-secrets-"));
|
||||
const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-ssh-runtime-"));
|
||||
roots.push(root, runtimeRoot);
|
||||
return new WorkspaceSecretStore({ root, runtimeRoot, installationId: "test" });
|
||||
}
|
||||
|
||||
function sshDatabase(): WorkspaceDatabase {
|
||||
return {
|
||||
id: "11111111-1111-4111-8111-111111111111",
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "datawarehouse",
|
||||
version: 4,
|
||||
createdAt: "2026-08-27T08:00:00Z",
|
||||
updatedAt: "2026-08-27T09:00:00Z",
|
||||
connectionStatus: "reachable",
|
||||
binding: {
|
||||
transport: "ssh_tunnel",
|
||||
username: "warehouse_reader",
|
||||
sshHost: "bastion.internal",
|
||||
sshPort: 2222,
|
||||
sshUsername: "tunnel_user",
|
||||
sshTargetHost: "postgres.internal",
|
||||
sshTargetPort: 5432,
|
||||
tlsServername: "postgres.internal",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function fakeChild(): ChildProcessWithoutNullStreams {
|
||||
const child = new EventEmitter() as EventEmitter & {
|
||||
stdin: PassThrough;
|
||||
stdout: PassThrough;
|
||||
stderr: PassThrough;
|
||||
exitCode: number | null;
|
||||
signalCode: NodeJS.Signals | null;
|
||||
kill: (signal?: NodeJS.Signals | number) => boolean;
|
||||
};
|
||||
child.stdin = new PassThrough();
|
||||
child.stdout = new PassThrough();
|
||||
child.stderr = new PassThrough();
|
||||
child.exitCode = null;
|
||||
child.signalCode = null;
|
||||
child.kill = vi.fn((signal: NodeJS.Signals | number = "SIGTERM") => {
|
||||
child.signalCode = typeof signal === "string" ? signal : "SIGTERM";
|
||||
child.emit("exit", null, child.signalCode);
|
||||
return true;
|
||||
});
|
||||
return child as unknown as ChildProcessWithoutNullStreams;
|
||||
}
|
||||
|
||||
test("builds a strict host-verified OpenSSH stdio tunnel", () => {
|
||||
const args = buildSshArguments({
|
||||
sshHost: "bastion.internal",
|
||||
sshPort: 2222,
|
||||
sshUsername: "tunnel_user",
|
||||
targetHost: "postgres.internal",
|
||||
targetPort: 5432,
|
||||
privateKeyFile: "/runtime/id",
|
||||
knownHostsFile: "/runtime/known_hosts",
|
||||
passphraseFile: "/runtime/passphrase",
|
||||
connectTimeoutMs: 5_001,
|
||||
});
|
||||
|
||||
expect(args).toEqual(expect.arrayContaining([
|
||||
"-F", "/dev/null",
|
||||
"-o", "BatchMode=no",
|
||||
"-o", "StrictHostKeyChecking=yes",
|
||||
"-o", "UserKnownHostsFile=/runtime/known_hosts",
|
||||
"-o", "GlobalKnownHostsFile=/dev/null",
|
||||
"-o", "IdentitiesOnly=yes",
|
||||
"-o", "IdentityAgent=none",
|
||||
"-o", "PasswordAuthentication=no",
|
||||
"-o", "KbdInteractiveAuthentication=no",
|
||||
"-o", "ConnectTimeout=6",
|
||||
"-W", "postgres.internal:5432",
|
||||
"--", "tunnel_user@bastion.internal",
|
||||
]));
|
||||
});
|
||||
|
||||
test("connects pg through OpenSSH, supplies askpass, and releases all secret leases", async () => {
|
||||
const store = secretStore();
|
||||
store.putMany("psd-clinical", {
|
||||
[CATALOG_SECRET_IDS.password]: "db-password ",
|
||||
[CATALOG_SECRET_IDS.sshPrivateKey]: "PRIVATE KEY\n",
|
||||
[CATALOG_SECRET_IDS.sshPrivateKeyPassphrase]: "key-passphrase",
|
||||
[CATALOG_SECRET_IDS.sshKnownHosts]: "bastion.internal ssh-ed25519 AAAATEST\n",
|
||||
[CATALOG_SECRET_IDS.tlsCa]: "CA CERTIFICATE\n",
|
||||
});
|
||||
const child = fakeChild();
|
||||
let clientConfig: ClientConfig | undefined;
|
||||
let spawnCall: { command: string; args: readonly string[]; env: NodeJS.ProcessEnv } | undefined;
|
||||
const end = vi.fn(async () => undefined);
|
||||
const query = vi.fn(async () => ({ rows: [{ ok: true }] }));
|
||||
const connect = vi.fn(async () => undefined);
|
||||
|
||||
const access = new ConcreteCatalogPostgresAccess(store, {
|
||||
sshBinary: "/usr/bin/ssh",
|
||||
askpassPath: "/app/ssh-askpass.mjs",
|
||||
connectTimeoutMs: 5_000,
|
||||
spawnSsh: (command, args, options) => {
|
||||
spawnCall = { command, args, env: options.env };
|
||||
return child;
|
||||
},
|
||||
createClient: (config) => {
|
||||
clientConfig = config;
|
||||
return { connect, query, end } as unknown as Client;
|
||||
},
|
||||
});
|
||||
|
||||
const client = await access.connect(sshDatabase(), new AbortController().signal);
|
||||
expect(connect).toHaveBeenCalledOnce();
|
||||
expect(clientConfig).toMatchObject({
|
||||
host: "postgres.internal",
|
||||
port: 5432,
|
||||
database: "warehouse",
|
||||
user: "warehouse_reader",
|
||||
password: "db-password ",
|
||||
connectionTimeoutMillis: 5_000,
|
||||
ssl: {
|
||||
ca: "CA CERTIFICATE\n",
|
||||
servername: "postgres.internal",
|
||||
rejectUnauthorized: true,
|
||||
},
|
||||
});
|
||||
expect(clientConfig?.stream).toBeTypeOf("function");
|
||||
expect(spawnCall?.command).toBe("/usr/bin/ssh");
|
||||
expect(spawnCall?.args.some((argument) => argument.startsWith("IdentityFile="))).toBe(true);
|
||||
expect(spawnCall?.args.some((argument) => argument.startsWith("UserKnownHostsFile="))).toBe(true);
|
||||
expect(spawnCall?.env).toMatchObject({
|
||||
DISPLAY: "thothii",
|
||||
SSH_ASKPASS: "/app/ssh-askpass.mjs",
|
||||
SSH_ASKPASS_REQUIRE: "force",
|
||||
});
|
||||
const leasedPaths = spawnCall!.args
|
||||
.filter((argument) => argument.startsWith("IdentityFile=") || argument.startsWith("UserKnownHostsFile="))
|
||||
.map((argument) => argument.slice(argument.indexOf("=") + 1));
|
||||
leasedPaths.push(spawnCall!.env.THT_SSH_PASSPHRASE_FILE!);
|
||||
expect(leasedPaths.every(existsSync)).toBe(true);
|
||||
|
||||
await expect(client.query("SELECT 1", [])).resolves.toEqual({ rows: [{ ok: true }] });
|
||||
await client.end();
|
||||
|
||||
expect(end).toHaveBeenCalledOnce();
|
||||
expect(child.kill).toHaveBeenCalledWith("SIGTERM");
|
||||
expect(leasedPaths.some(existsSync)).toBe(false);
|
||||
});
|
||||
@@ -0,0 +1,125 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { PostgreSqlContainer } from "@testcontainers/postgresql";
|
||||
import { CamelCasePlugin, Kysely, PostgresDialect, sql } from "kysely";
|
||||
import { Pool } from "pg";
|
||||
import { expect, test } from "vitest";
|
||||
import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js";
|
||||
import type { ObservedSchemaSnapshot } from "../src/catalog/types.js";
|
||||
import { up as upDatabases } from "../src/catalog/migrations/001_workspace_databases.js";
|
||||
import { up as upTables } from "../src/catalog/migrations/002_catalog_tables.js";
|
||||
import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema_sync.js";
|
||||
import { up as upRuntimeSequencePrivileges } from "../src/catalog/migrations/004_catalog_runtime_sequence_privileges.js";
|
||||
|
||||
const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
|
||||
|
||||
test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per workspace and optimistic updates", async () => {
|
||||
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
|
||||
const db = new Kysely<CatalogDatabase>({
|
||||
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
|
||||
plugins: [new CamelCasePlugin()],
|
||||
});
|
||||
try {
|
||||
await upDatabases(db);
|
||||
await upTables(db);
|
||||
await upSchemaSync(db);
|
||||
await sql`CREATE ROLE thothii_catalog_runtime`.execute(db);
|
||||
await upRuntimeSequencePrivileges(db);
|
||||
const sequencePrivilege = await sql<{ allowed: boolean }>`
|
||||
SELECT has_sequence_privilege(
|
||||
'thothii_catalog_runtime',
|
||||
'catalog_sync_events_id_seq',
|
||||
'USAGE'
|
||||
) AS allowed
|
||||
`.execute(db);
|
||||
expect(sequencePrivilege.rows[0]?.allowed).toBe(true);
|
||||
const repository = new KyselyCatalogRepository(db);
|
||||
const input = {
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres" as const,
|
||||
databaseName: "warehouse",
|
||||
schema: "datawarehouse",
|
||||
binding: { transport: "rest_api" as const, baseUrl: "https://psd.example/api", restPath: "/health", restAuth: "bearer" as const },
|
||||
};
|
||||
const created = await repository.create(input);
|
||||
expect(created).toMatchObject({ version: 1, connectionStatus: "untested", binding: { transport: "rest_api" } });
|
||||
await expect(repository.create(input)).rejects.toThrow("Workspace database already exists");
|
||||
expect(await repository.update(created.id, 99, input)).toBeUndefined();
|
||||
const synchronized = await repository.reconcileTables(created.id, 1, [
|
||||
{ name: "patients", sourceComment: "Clinical patients" },
|
||||
{ name: "visits", sourceComment: null },
|
||||
], []);
|
||||
expect(synchronized).toMatchObject({ kind: "applied", createdCount: 2, deletedCount: 0 });
|
||||
const patients = (await repository.listTables(created.id))[0];
|
||||
expect(await repository.updateTableDescription(
|
||||
created.id,
|
||||
patients.id,
|
||||
patients.version,
|
||||
"Curated patients",
|
||||
)).toMatchObject({ description: "Curated patients", sourceComment: "Clinical patients", version: 2 });
|
||||
const visits = (await repository.listTables(created.id)).find((table) => table.name === "visits")!;
|
||||
const fullColumnsSnapshot: ObservedSchemaSnapshot = {
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||
tables: [
|
||||
{ name: "patients", sourceComment: "Clinical patients" },
|
||||
{ name: "visits", sourceComment: null },
|
||||
],
|
||||
columns: [
|
||||
{ tableName: "patients", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
||||
{ tableName: "patients", name: "name", ordinalPosition: 2, dataType: "text", isNullable: true, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
||||
{ tableName: "visits", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
||||
{ tableName: "visits", name: "patient_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
||||
],
|
||||
relationships: [],
|
||||
};
|
||||
expect(await repository.applySchemaSync(created.id, 1, "columns", [], fullColumnsSnapshot))
|
||||
.toMatchObject({ created: 4, deleted: 0 });
|
||||
expect((await repository.listColumns(created.id, patients.id)).map((column) => column.name))
|
||||
.toEqual(["id", "name"]);
|
||||
expect((await repository.listColumns(created.id, visits.id)).map((column) => column.name))
|
||||
.toEqual(["id", "patient_id"]);
|
||||
|
||||
const reducedColumnsSnapshot: ObservedSchemaSnapshot = {
|
||||
...fullColumnsSnapshot,
|
||||
columns: fullColumnsSnapshot.columns.filter((column) => column.name === "id"),
|
||||
};
|
||||
expect(await repository.planSchemaSync(created.id, "columns", [], reducedColumnsSnapshot)).toMatchObject({
|
||||
deletedColumns: [
|
||||
{ tableName: "patients", columnName: "name" },
|
||||
{ tableName: "visits", columnName: "patient_id" },
|
||||
],
|
||||
});
|
||||
expect(await repository.planSchemaSync(created.id, "columns", [patients.id], reducedColumnsSnapshot)).toMatchObject({
|
||||
deletedColumns: [{ tableName: "patients", columnName: "name" }],
|
||||
});
|
||||
expect(await repository.applySchemaSync(created.id, 1, "columns", [patients.id], reducedColumnsSnapshot))
|
||||
.toMatchObject({ deleted: 1 });
|
||||
expect((await repository.listColumns(created.id, patients.id)).map((column) => column.name))
|
||||
.toEqual(["id"]);
|
||||
expect((await repository.listColumns(created.id, visits.id)).map((column) => column.name))
|
||||
.toEqual(["id", "patient_id"]);
|
||||
expect(await repository.reconcileTables(created.id, 1, [
|
||||
{ name: "patients", sourceComment: "Updated physical comment" },
|
||||
], [])).toEqual({ kind: "confirmation_required", deletedNames: ["visits"] });
|
||||
expect(await repository.reconcileTables(created.id, 1, [
|
||||
{ name: "patients", sourceComment: "Updated physical comment" },
|
||||
], ["visits"])).toMatchObject({ kind: "applied", updatedCount: 1, deletedCount: 1 });
|
||||
const syncRun = await repository.createSyncRun(created.id, "columns", [patients.id], 1);
|
||||
expect(syncRun).toMatchObject({
|
||||
databaseId: created.id,
|
||||
scope: "columns",
|
||||
tableIds: [patients.id],
|
||||
state: "queued",
|
||||
});
|
||||
expect(await repository.listSyncRuns(created.id)).toEqual([
|
||||
expect.objectContaining({ id: syncRun.id, tableIds: [patients.id] }),
|
||||
]);
|
||||
expect(await repository.update(created.id, 1, { ...input, schema: "public" })).toMatchObject({ version: 2, schema: "public" });
|
||||
expect(await repository.delete(created.id, 2)).toBe(true);
|
||||
expect(await repository.list()).toEqual([]);
|
||||
expect(await repository.listTables(created.id)).toEqual([]);
|
||||
} finally {
|
||||
await db.destroy();
|
||||
await container.stop();
|
||||
}
|
||||
}, 60_000);
|
||||
@@ -0,0 +1,194 @@
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import type { CatalogDatabaseClient, CatalogPostgresAccess } from "../src/catalog/postgres-access.js";
|
||||
import { ConcreteCatalogSchemaIntrospector } from "../src/catalog/schema-introspector.js";
|
||||
import {
|
||||
CatalogSchemaCapabilityUnavailableError,
|
||||
type WorkspaceDatabase,
|
||||
} from "../src/catalog/types.js";
|
||||
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function store() {
|
||||
const root = mkdtempSync(join(tmpdir(), "catalog-schema-introspection-secrets-"));
|
||||
const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-schema-introspection-runtime-"));
|
||||
roots.push(root, runtimeRoot);
|
||||
return new WorkspaceSecretStore({ root, runtimeRoot, installationId: "test" });
|
||||
}
|
||||
|
||||
function database(binding: WorkspaceDatabase["binding"]): WorkspaceDatabase {
|
||||
return {
|
||||
id: "11111111-1111-4111-8111-111111111111",
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "datawarehouse",
|
||||
binding,
|
||||
version: 4,
|
||||
connectionStatus: "reachable",
|
||||
testedVersion: 4,
|
||||
createdAt: "2026-08-27T08:00:00Z",
|
||||
updatedAt: "2026-08-27T09:00:00Z",
|
||||
};
|
||||
}
|
||||
|
||||
test("reads columns, ordered composite keys, and physical relationships from one PostgreSQL connection", async () => {
|
||||
const query = vi.fn()
|
||||
.mockResolvedValueOnce({ rows: [{ present: true }] })
|
||||
.mockResolvedValueOnce({ rows: [{ name: "visits", source_comment: "Visits" }] })
|
||||
.mockResolvedValueOnce({ rows: [
|
||||
{ table_name: "visits", name: "tenant_id", ordinal_position: 1, data_type: "uuid", is_nullable: false, default_expression: null, primary_key_position: 1, source_comment: null },
|
||||
{ table_name: "visits", name: "patient_id", ordinal_position: 2, data_type: "bigint", is_nullable: false, default_expression: null, primary_key_position: 2, source_comment: "Patient" },
|
||||
] })
|
||||
.mockResolvedValueOnce({ rows: [
|
||||
{ constraint_name: "visits_patient_fkey", source_table_name: "visits", target_table_name: "patients", update_action: "a", delete_action: "c", deferrable: true, initially_deferred: false, position: 1, source_column_name: "tenant_id", target_column_name: "tenant_id" },
|
||||
{ constraint_name: "visits_patient_fkey", source_table_name: "visits", target_table_name: "patients", update_action: "a", delete_action: "c", deferrable: true, initially_deferred: false, position: 2, source_column_name: "patient_id", target_column_name: "id" },
|
||||
] });
|
||||
const end = vi.fn(async () => undefined);
|
||||
const client: CatalogDatabaseClient = { query, end };
|
||||
const postgres: CatalogPostgresAccess = { connect: vi.fn(async () => client) };
|
||||
const introspector = new ConcreteCatalogSchemaIntrospector(postgres, store());
|
||||
|
||||
const result = await introspector.scan(database({
|
||||
transport: "ssh_tunnel",
|
||||
username: "reader",
|
||||
sshHost: "bastion.internal",
|
||||
sshPort: 22,
|
||||
sshUsername: "tunnel",
|
||||
sshTargetHost: "db.internal",
|
||||
sshTargetPort: 5432,
|
||||
}), new AbortController().signal);
|
||||
|
||||
expect(result.capabilities).toEqual({ tables: "available", columns: "available", relationships: "available" });
|
||||
expect(result.columns).toMatchObject([
|
||||
{ name: "tenant_id", primaryKeyPosition: 1, isNullable: false },
|
||||
{ name: "patient_id", primaryKeyPosition: 2, sourceComment: "Patient" },
|
||||
]);
|
||||
expect(result.relationships).toEqual([expect.objectContaining({
|
||||
constraintName: "visits_patient_fkey",
|
||||
updateRule: "NO ACTION",
|
||||
deleteRule: "CASCADE",
|
||||
deferrable: true,
|
||||
columns: [
|
||||
{ position: 1, sourceColumnName: "tenant_id", targetColumnName: "tenant_id" },
|
||||
{ position: 2, sourceColumnName: "patient_id", targetColumnName: "id" },
|
||||
],
|
||||
})]);
|
||||
expect(query.mock.calls[2][0]).toContain("format_type");
|
||||
expect(query.mock.calls[3][0]).toContain("WITH ORDINALITY");
|
||||
expect(query.mock.calls.slice(1).every((call) => call[1][0] === "datawarehouse")).toBe(true);
|
||||
expect(end).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
test("uses the typed full REST snapshot RPC and preserves explicit capability unavailability", async () => {
|
||||
const response = {
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "unavailable", relationships: "unavailable" },
|
||||
tables: [{ name: "patients", sourceComment: null }],
|
||||
columns: [],
|
||||
relationships: [],
|
||||
};
|
||||
const fetchMock = vi.fn(async () => new Response(JSON.stringify(response), { status: 200, headers: { "content-type": "application/json" } }));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const postgres: CatalogPostgresAccess = { connect: vi.fn(async () => { throw new Error("wire access must not be used"); }) };
|
||||
const introspector = new ConcreteCatalogSchemaIntrospector(postgres, store());
|
||||
|
||||
const result = await introspector.scan(database({
|
||||
transport: "rest_api", baseUrl: "https://connector.internal/api/", restPath: "/health", restAuth: "none",
|
||||
}), new AbortController().signal);
|
||||
|
||||
expect(result).toEqual(response);
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://connector.internal/api/rpc/schema_snapshot",
|
||||
expect.objectContaining({ method: "POST", body: JSON.stringify({ schema_name: "datawarehouse" }) }),
|
||||
);
|
||||
});
|
||||
|
||||
test("falls back to one read-only REST query when the snapshot RPC is absent", async () => {
|
||||
const response = {
|
||||
schemaVersion: 1 as const,
|
||||
capabilities: { tables: "available" as const, columns: "available" as const, relationships: "available" as const },
|
||||
tables: [
|
||||
{ name: "patients", sourceComment: "Clinical patients" },
|
||||
{ name: "visits", sourceComment: null },
|
||||
],
|
||||
columns: [
|
||||
{ tableName: "patients", name: "tenant_id", ordinalPosition: 1, dataType: "uuid", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: "Tenant" },
|
||||
{ tableName: "patients", name: "id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: "nextval('patients_id_seq'::regclass)", primaryKeyPosition: 2, sourceComment: null },
|
||||
{ tableName: "visits", name: "tenant_id", ordinalPosition: 1, dataType: "uuid", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
||||
{ tableName: "visits", name: "patient_id", ordinalPosition: 2, dataType: "bigint", isNullable: true, defaultExpression: null, primaryKeyPosition: null, sourceComment: "Owning patient" },
|
||||
],
|
||||
relationships: [{
|
||||
constraintName: "visits_patient_fkey",
|
||||
sourceTableName: "visits",
|
||||
targetTableName: "patients",
|
||||
updateRule: "CASCADE",
|
||||
deleteRule: "RESTRICT",
|
||||
deferrable: true,
|
||||
initiallyDeferred: false,
|
||||
columns: [
|
||||
{ position: 1, sourceColumnName: "tenant_id", targetColumnName: "tenant_id" },
|
||||
{ position: 2, sourceColumnName: "patient_id", targetColumnName: "id" },
|
||||
],
|
||||
}],
|
||||
};
|
||||
const fetchMock = vi.fn()
|
||||
.mockResolvedValueOnce(new Response(null, { status: 404 }))
|
||||
.mockResolvedValueOnce(new Response(JSON.stringify([response]), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
}));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const postgres: CatalogPostgresAccess = {
|
||||
connect: vi.fn(async () => { throw new Error("wire access must not be used"); }),
|
||||
};
|
||||
const introspector = new ConcreteCatalogSchemaIntrospector(postgres, store());
|
||||
|
||||
const result = await introspector.scan(database({
|
||||
transport: "rest_api",
|
||||
baseUrl: "https://connector.internal/api/",
|
||||
restPath: "/health",
|
||||
restAuth: "none",
|
||||
}), new AbortController().signal);
|
||||
|
||||
expect(result).toEqual(response);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
expect(fetchMock.mock.calls[0]).toEqual([
|
||||
"https://connector.internal/api/rpc/schema_snapshot",
|
||||
expect.objectContaining({ method: "POST", body: JSON.stringify({ schema_name: "datawarehouse" }) }),
|
||||
]);
|
||||
expect(fetchMock.mock.calls[1][0]).toBe("https://connector.internal/api/rpc/run_query");
|
||||
const fallbackRequest = fetchMock.mock.calls[1][1] as RequestInit;
|
||||
expect(fallbackRequest).toMatchObject({ method: "POST" });
|
||||
const fallbackBody = JSON.parse(String(fallbackRequest.body)) as { query_text: string };
|
||||
expect(Object.keys(fallbackBody)).toEqual(["query_text"]);
|
||||
expect(fallbackBody.query_text).toMatch(/^\s*WITH\b/);
|
||||
expect(fallbackBody.query_text).toContain("pg_catalog.pg_constraint");
|
||||
expect(fallbackBody.query_text).not.toMatch(/\b(INSERT|UPDATE|DROP|ALTER|CREATE|TRUNCATE)\b/i);
|
||||
});
|
||||
|
||||
test("classifies a missing REST snapshot RPC as an explicit binding capability", async () => {
|
||||
vi.stubGlobal("fetch", vi.fn(async () => new Response(null, { status: 404 })));
|
||||
const postgres: CatalogPostgresAccess = {
|
||||
connect: vi.fn(async () => { throw new Error("wire access must not be used"); }),
|
||||
};
|
||||
const introspector = new ConcreteCatalogSchemaIntrospector(postgres, store());
|
||||
|
||||
const scan = introspector.scan(database({
|
||||
transport: "rest_api",
|
||||
baseUrl: "https://connector.internal/api/",
|
||||
restPath: "/health",
|
||||
restAuth: "none",
|
||||
}), new AbortController().signal);
|
||||
|
||||
await expect(scan).rejects.toMatchObject<CatalogSchemaCapabilityUnavailableError>({
|
||||
capability: "schema_snapshot",
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,224 @@
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import { buildApp } from "../src/app.js";
|
||||
import { loadConfig } from "../src/config.js";
|
||||
import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
|
||||
import type { CatalogSchemaIntrospector } from "../src/catalog/schema-introspector.js";
|
||||
import type { CatalogSyncRun, ObservedSchemaSnapshot } from "../src/catalog/types.js";
|
||||
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||
import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/registry.js";
|
||||
import type { WorkspaceDescriptor } from "../src/workspaces/schema.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
|
||||
|
||||
const workspace: WorkspaceDescriptor = {
|
||||
workspace: { schema_version: 3, id: "psd-clinical", name: "Policlinico San Donato", language: "it" },
|
||||
dwh: { engine: "postgres", database: "warehouse", schema: "datawarehouse", port: 5432, supported_transports: ["postgres_direct"] },
|
||||
semantic_index: {
|
||||
vector_store: { engine: "qdrant", collection: "psd", dimensions: 1024, distance: "cosine" },
|
||||
embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
|
||||
},
|
||||
llm_policy: { allowed: ["zai/glm-5.2"] },
|
||||
};
|
||||
const revision: WorkspaceRevision = { id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/tmp/psd.yaml" };
|
||||
|
||||
function snapshot(): ObservedSchemaSnapshot {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||
tables: [
|
||||
{ name: "patients", sourceComment: "Clinical patients" },
|
||||
{ name: "visits", sourceComment: "Patient visits" },
|
||||
],
|
||||
columns: [
|
||||
{ tableName: "patients", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: "Patient key" },
|
||||
{ tableName: "visits", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
||||
{ tableName: "visits", name: "patient_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: "Owning patient" },
|
||||
],
|
||||
relationships: [{
|
||||
constraintName: "visits_patient_id_fkey",
|
||||
sourceTableName: "visits",
|
||||
targetTableName: "patients",
|
||||
updateRule: "NO ACTION",
|
||||
deleteRule: "CASCADE",
|
||||
deferrable: false,
|
||||
initiallyDeferred: false,
|
||||
columns: [{ position: 1, sourceColumnName: "patient_id", targetColumnName: "id" }],
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
async function waitFor(repository: MemoryCatalogRepository, runId: string, state: CatalogSyncRun["state"]): Promise<CatalogSyncRun> {
|
||||
for (let attempt = 0; attempt < 100; attempt += 1) {
|
||||
const run = await repository.getSyncRun(runId);
|
||||
if (run?.state === state) return run;
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
throw new Error(`Run ${runId} did not reach ${state}`);
|
||||
}
|
||||
|
||||
async function setup() {
|
||||
const secretRoot = mkdtempSync(join(tmpdir(), "catalog-schema-secret-"));
|
||||
const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-schema-runtime-"));
|
||||
roots.push(secretRoot, runtimeRoot);
|
||||
const repository = new MemoryCatalogRepository();
|
||||
const created = await repository.create({
|
||||
workspaceId: "psd-clinical", engine: "postgres", databaseName: "warehouse", schema: "datawarehouse",
|
||||
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
||||
});
|
||||
await repository.recordTest(created.id, created.version, {
|
||||
connectionStatus: "reachable", testedVersion: created.version, lastTestedAt: new Date().toISOString(),
|
||||
});
|
||||
let observed = snapshot();
|
||||
const scan = vi.fn(async (_database, _signal, progress) => {
|
||||
await progress?.("connecting");
|
||||
await progress?.("scanning_tables", { tables: observed.tables.length });
|
||||
await progress?.("scanning_columns", { tables: observed.tables.length, columns: observed.columns.length });
|
||||
await progress?.("scanning_relationships", { relationships: observed.relationships.length });
|
||||
return structuredClone(observed);
|
||||
});
|
||||
const introspector: CatalogSchemaIntrospector = { scan };
|
||||
const registry = {
|
||||
list: vi.fn(async () => [revision]),
|
||||
listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]),
|
||||
read: vi.fn(async () => ({ workspace, revision })),
|
||||
} as unknown as WorkspaceRegistry;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test" }), {
|
||||
thtRunner: {} as never,
|
||||
workspaceRegistry: registry,
|
||||
workspaceSecretStore: new WorkspaceSecretStore({ root: secretRoot, runtimeRoot, installationId: "test" }),
|
||||
catalogRepository: repository,
|
||||
catalogSchemaIntrospector: introspector,
|
||||
workspaceDiagnoser: vi.fn(),
|
||||
});
|
||||
return {
|
||||
app, repository, database: (await repository.get(created.id))!, scan,
|
||||
setObserved(next: ObservedSchemaSnapshot) { observed = next; },
|
||||
};
|
||||
}
|
||||
|
||||
test("synchronizes a full physical schema and derives primary and foreign key flags", async () => {
|
||||
const { app, repository, database } = await setup();
|
||||
const started = await app.inject({
|
||||
method: "POST", url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version, scope: "all", tableIds: [] },
|
||||
});
|
||||
expect(started.statusCode).toBe(202);
|
||||
const completed = await waitFor(repository, started.json().id, "succeeded");
|
||||
expect(completed.counts).toMatchObject({ tables: 2, columns: 3, relationships: 1 });
|
||||
|
||||
const tables = await repository.listTables(database.id);
|
||||
const visits = tables.find((table) => table.name === "visits")!;
|
||||
const columns = (await app.inject({ method: "GET", url: `/catalog/databases/${database.id}/tables/${visits.id}/columns` })).json();
|
||||
expect(columns).toMatchObject([
|
||||
{ name: "id", isPrimaryKey: true, primaryKeyPosition: 1, isForeignKey: false },
|
||||
{ name: "patient_id", isPrimaryKey: false, isForeignKey: true, foreignKeyCount: 1 },
|
||||
]);
|
||||
const relationships = (await app.inject({ method: "GET", url: `/catalog/databases/${database.id}/relationships` })).json();
|
||||
expect(relationships).toMatchObject([{ constraintName: "visits_patient_id_fkey", columns: [{ sourceColumnName: "patient_id", targetColumnName: "id" }] }]);
|
||||
expect((await repository.get(database.id))?.schemaSyncedVersion).toBe(database.version);
|
||||
});
|
||||
|
||||
test("synchronizes columns for every catalog table when no table selection is supplied", async () => {
|
||||
const { app, repository, database, setObserved } = await setup();
|
||||
const tablesRun = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version, scope: "tables", tableIds: [] },
|
||||
});
|
||||
expect(tablesRun.statusCode).toBe(202);
|
||||
await waitFor(repository, tablesRun.json().id, "succeeded");
|
||||
const tables = await repository.listTables(database.id);
|
||||
expect(tables.map((table) => table.name)).toEqual(["patients", "visits"]);
|
||||
|
||||
const columnsRun = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version, scope: "columns", tableIds: [] },
|
||||
});
|
||||
expect(columnsRun.statusCode).toBe(202);
|
||||
await waitFor(repository, columnsRun.json().id, "succeeded");
|
||||
const patients = tables.find((table) => table.name === "patients")!;
|
||||
const visits = tables.find((table) => table.name === "visits")!;
|
||||
expect((await repository.listColumns(database.id, patients.id)).map((column) => column.name)).toEqual(["id"]);
|
||||
expect((await repository.listColumns(database.id, visits.id)).map((column) => column.name)).toEqual(["id", "patient_id"]);
|
||||
|
||||
const next = snapshot();
|
||||
next.columns = next.columns.filter((column) => column.name !== "id");
|
||||
setObserved(next);
|
||||
const selectedDestructiveRun = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version, scope: "columns", tableIds: [patients.id] },
|
||||
});
|
||||
expect(selectedDestructiveRun.statusCode).toBe(202);
|
||||
const selectedWaiting = await waitFor(repository, selectedDestructiveRun.json().id, "awaiting_confirmation");
|
||||
expect(selectedWaiting.plannedDiff?.deletedColumns).toEqual([
|
||||
{ tableName: "patients", columnName: "id" },
|
||||
]);
|
||||
expect((await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/sync-runs/${selectedWaiting.id}/cancel`,
|
||||
})).statusCode).toBe(200);
|
||||
|
||||
const destructiveRun = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version, scope: "columns", tableIds: [] },
|
||||
});
|
||||
expect(destructiveRun.statusCode).toBe(202);
|
||||
const waiting = await waitFor(repository, destructiveRun.json().id, "awaiting_confirmation");
|
||||
expect(waiting.plannedDiff?.deletedColumns).toEqual([
|
||||
{ tableName: "patients", columnName: "id" },
|
||||
{ tableName: "visits", columnName: "id" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("keeps generated descriptions editable and preserves them across synchronization", async () => {
|
||||
const { app, repository, database } = await setup();
|
||||
const first = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
||||
await waitFor(repository, first.json().id, "succeeded");
|
||||
const patients = (await repository.listTables(database.id)).find((table) => table.name === "patients")!;
|
||||
const editedTable = await app.inject({
|
||||
method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}`,
|
||||
payload: { version: patients.version, description: null, generatedDescription: "Generated table draft" },
|
||||
});
|
||||
expect(editedTable.json()).toMatchObject({ description: null, generatedDescription: "Generated table draft" });
|
||||
const idColumn = (await repository.listColumns(database.id, patients.id))[0];
|
||||
const editedColumn = await app.inject({
|
||||
method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}/columns/${idColumn.id}`,
|
||||
payload: { version: idColumn.version, description: "Reviewed key", generatedDescription: "Generated key draft" },
|
||||
});
|
||||
expect(editedColumn.json()).toMatchObject({ description: "Reviewed key", generatedDescription: "Generated key draft" });
|
||||
|
||||
const second = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
||||
await waitFor(repository, second.json().id, "succeeded");
|
||||
expect(await repository.getTable(database.id, patients.id)).toMatchObject({ generatedDescription: "Generated table draft" });
|
||||
expect(await repository.getColumn(database.id, patients.id, idColumn.id)).toMatchObject({ description: "Reviewed key", generatedDescription: "Generated key draft" });
|
||||
});
|
||||
|
||||
test("waits for confirmation and rescans before applying destructive changes", async () => {
|
||||
const { app, repository, database, scan, setObserved } = await setup();
|
||||
const first = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
||||
await waitFor(repository, first.json().id, "succeeded");
|
||||
const next = snapshot();
|
||||
next.tables = next.tables.filter((table) => table.name !== "visits");
|
||||
next.columns = next.columns.filter((column) => column.tableName !== "visits");
|
||||
next.relationships = [];
|
||||
setObserved(next);
|
||||
|
||||
const destructive = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
||||
const waiting = await waitFor(repository, destructive.json().id, "awaiting_confirmation");
|
||||
expect(waiting.plannedDiff).toMatchObject({ deletedTables: ["visits"] });
|
||||
expect(await repository.listTables(database.id)).toHaveLength(2);
|
||||
const confirmed = await app.inject({
|
||||
method: "POST", url: `/catalog/sync-runs/${waiting.id}/confirm`, payload: { confirmationToken: waiting.confirmationToken },
|
||||
});
|
||||
expect(confirmed.statusCode).toBe(200);
|
||||
await waitFor(repository, waiting.id, "succeeded");
|
||||
expect((await repository.listTables(database.id)).map((table) => table.name)).toEqual(["patients"]);
|
||||
expect(scan).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
@@ -0,0 +1,124 @@
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import type {
|
||||
CatalogDatabaseClient,
|
||||
CatalogPostgresAccess,
|
||||
} from "../src/catalog/postgres-access.js";
|
||||
import { CATALOG_SECRET_IDS } from "../src/catalog/secrets.js";
|
||||
import { ConcreteCatalogTableIntrospector } from "../src/catalog/table-introspector.js";
|
||||
import type { WorkspaceDatabase } from "../src/catalog/types.js";
|
||||
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function secretStore() {
|
||||
const root = mkdtempSync(join(tmpdir(), "catalog-table-introspection-secrets-"));
|
||||
const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-table-introspection-runtime-"));
|
||||
roots.push(root, runtimeRoot);
|
||||
return new WorkspaceSecretStore({ root, runtimeRoot, installationId: "test" });
|
||||
}
|
||||
|
||||
function database(binding: WorkspaceDatabase["binding"]): WorkspaceDatabase {
|
||||
return {
|
||||
id: "11111111-1111-4111-8111-111111111111",
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "datawarehouse",
|
||||
version: 4,
|
||||
createdAt: "2026-08-27T08:00:00Z",
|
||||
updatedAt: "2026-08-27T09:00:00Z",
|
||||
connectionStatus: "reachable",
|
||||
binding,
|
||||
};
|
||||
}
|
||||
|
||||
test("reads only ordinary and partitioned PostgreSQL tables from the configured schema", async () => {
|
||||
const query = vi.fn()
|
||||
.mockResolvedValueOnce({ rows: [{ present: true }] })
|
||||
.mockResolvedValueOnce({ rows: [
|
||||
{ name: "visits", source_comment: null },
|
||||
{ name: "patients", source_comment: "Clinical patients" },
|
||||
] });
|
||||
const end = vi.fn(async () => undefined);
|
||||
const client: CatalogDatabaseClient = { query, end };
|
||||
const postgres: CatalogPostgresAccess = { connect: vi.fn(async () => client) };
|
||||
const introspector = new ConcreteCatalogTableIntrospector(postgres, secretStore());
|
||||
|
||||
const tables = await introspector.scan(database({
|
||||
transport: "postgres_direct",
|
||||
host: "db.internal",
|
||||
port: 5432,
|
||||
username: "reader",
|
||||
}), new AbortController().signal);
|
||||
|
||||
expect(tables).toEqual([
|
||||
{ name: "patients", sourceComment: "Clinical patients" },
|
||||
{ name: "visits", sourceComment: null },
|
||||
]);
|
||||
expect(query.mock.calls[1][0]).toContain("c.relkind IN ('r', 'p')");
|
||||
expect(query.mock.calls[1][0]).not.toContain("'v'");
|
||||
expect(query.mock.calls[1][1]).toEqual(["datawarehouse"]);
|
||||
expect(end).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
test("uses the typed REST table RPC and ignores non-table objects", async () => {
|
||||
const store = secretStore();
|
||||
store.put("psd-clinical", CATALOG_SECRET_IDS.apiKey, "rest-secret");
|
||||
const fetchMock = vi.fn(async () => new Response(JSON.stringify([
|
||||
{ type: "VIEW", table: "patient_view", comment: "Not a table" },
|
||||
{ type: "TABLE", table: "visits", comment: null },
|
||||
{ type: "TABLE", table: "patients", comment: "Clinical patients" },
|
||||
]), { status: 200, headers: { "content-type": "application/json" } }));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const postgres: CatalogPostgresAccess = {
|
||||
connect: vi.fn(async () => { throw new Error("PostgreSQL wire access must not be used"); }),
|
||||
};
|
||||
const introspector = new ConcreteCatalogTableIntrospector(postgres, store);
|
||||
|
||||
const tables = await introspector.scan(database({
|
||||
transport: "rest_api",
|
||||
baseUrl: "https://connector.internal/api/",
|
||||
restPath: "/health",
|
||||
restAuth: "x-api-key",
|
||||
}), new AbortController().signal);
|
||||
|
||||
expect(tables).toEqual([
|
||||
{ name: "patients", sourceComment: "Clinical patients" },
|
||||
{ name: "visits", sourceComment: null },
|
||||
]);
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://connector.internal/api/rpc/list_tables",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
headers: expect.objectContaining({ "x-api-key": "rest-secret" }),
|
||||
body: JSON.stringify({ schema_name: "datawarehouse" }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
test("fails closed when a REST table row violates the typed contract", async () => {
|
||||
const store = secretStore();
|
||||
const fetchMock = vi.fn(async () => new Response(JSON.stringify([
|
||||
{ type: "TABLE", table_name: "patients", comment: "Wrong field name" },
|
||||
]), { status: 200, headers: { "content-type": "application/json" } }));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const postgres: CatalogPostgresAccess = {
|
||||
connect: vi.fn(async () => { throw new Error("PostgreSQL wire access must not be used"); }),
|
||||
};
|
||||
const introspector = new ConcreteCatalogTableIntrospector(postgres, store);
|
||||
|
||||
await expect(introspector.scan(database({
|
||||
transport: "rest_api",
|
||||
baseUrl: "https://connector.internal/api",
|
||||
restPath: "/health",
|
||||
restAuth: "none",
|
||||
}), new AbortController().signal)).rejects.toThrow("REST schema response is invalid");
|
||||
});
|
||||
@@ -0,0 +1,126 @@
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import { buildApp } from "../src/app.js";
|
||||
import { loadConfig } from "../src/config.js";
|
||||
import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
|
||||
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||
import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/registry.js";
|
||||
import type { WorkspaceDescriptor } from "../src/workspaces/schema.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
|
||||
|
||||
const workspace: WorkspaceDescriptor = {
|
||||
workspace: { schema_version: 3, id: "psd-clinical", name: "Policlinico San Donato", language: "it" },
|
||||
dwh: {
|
||||
engine: "postgres", database: "warehouse", schema: "datawarehouse", port: 5432,
|
||||
supported_transports: ["postgres_direct", "rest_api"],
|
||||
},
|
||||
semantic_index: {
|
||||
vector_store: { engine: "qdrant", collection: "psd", dimensions: 1024, distance: "cosine" },
|
||||
embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
|
||||
},
|
||||
llm_policy: { allowed: ["zai/glm-5.2"] },
|
||||
diagnostics: { dwh_rest: { method: "GET", path: "/health", auth: "bearer", response: { database: "database", schema: "schema" } } },
|
||||
};
|
||||
const revision: WorkspaceRevision = {
|
||||
id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/tmp/psd.yaml",
|
||||
};
|
||||
|
||||
async function setup() {
|
||||
const secretRoot = mkdtempSync(join(tmpdir(), "catalog-table-secret-"));
|
||||
const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-table-runtime-"));
|
||||
roots.push(secretRoot, runtimeRoot);
|
||||
const repository = new MemoryCatalogRepository();
|
||||
const database = await repository.create({
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "datawarehouse",
|
||||
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
||||
});
|
||||
await repository.recordTest(database.id, database.version, {
|
||||
connectionStatus: "reachable",
|
||||
testedVersion: database.version,
|
||||
lastTestedAt: new Date().toISOString(),
|
||||
});
|
||||
const scan = vi.fn(async () => [
|
||||
{ name: "patients", sourceComment: "Clinical patients" },
|
||||
{ name: "visits", sourceComment: null },
|
||||
]);
|
||||
const registry = {
|
||||
list: vi.fn(async () => [revision]),
|
||||
listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]),
|
||||
read: vi.fn(async () => ({ workspace, revision })),
|
||||
} as unknown as WorkspaceRegistry;
|
||||
const secretStore = new WorkspaceSecretStore({ root: secretRoot, runtimeRoot, installationId: "test" });
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test" }), {
|
||||
thtRunner: {} as never,
|
||||
workspaceRegistry: registry,
|
||||
workspaceSecretStore: secretStore,
|
||||
catalogRepository: repository,
|
||||
workspaceDiagnoser: vi.fn(),
|
||||
});
|
||||
return { app, repository, database: (await repository.get(database.id))!, scan };
|
||||
}
|
||||
|
||||
test("lists physical tables and updates only review metadata", async () => {
|
||||
const { app, repository, database, scan } = await setup();
|
||||
const synchronized = await repository.reconcileTables(database.id, database.version, await scan(), []);
|
||||
expect(synchronized).toMatchObject({ kind: "applied", createdCount: 2, deletedCount: 0 });
|
||||
expect(scan).toHaveBeenCalledOnce();
|
||||
|
||||
const tables = (await app.inject({
|
||||
method: "GET", url: `/catalog/databases/${database.id}/tables`,
|
||||
})).json();
|
||||
expect(tables.map((table: { name: string }) => table.name)).toEqual(["patients", "visits"]);
|
||||
const patients = tables[0];
|
||||
const edited = await app.inject({
|
||||
method: "PATCH",
|
||||
url: `/catalog/databases/${database.id}/tables/${patients.id}`,
|
||||
payload: { version: patients.version, description: "Curated patient registry" },
|
||||
});
|
||||
expect(edited.statusCode).toBe(200);
|
||||
expect(edited.json()).toMatchObject({
|
||||
name: "patients",
|
||||
sourceComment: "Clinical patients",
|
||||
description: "Curated patient registry",
|
||||
version: 2,
|
||||
});
|
||||
});
|
||||
|
||||
test("requires an exact deletion confirmation before applying the atomic diff", async () => {
|
||||
const { app, repository, database, scan } = await setup();
|
||||
await repository.reconcileTables(database.id, database.version, await scan(), []);
|
||||
scan.mockResolvedValue([{ name: "patients", sourceComment: "Clinical patients" }]);
|
||||
|
||||
const preview = await repository.reconcileTables(database.id, database.version, await scan(), []);
|
||||
expect(preview).toEqual({ kind: "confirmation_required", deletedNames: ["visits"] });
|
||||
expect((await app.inject({ method: "GET", url: `/catalog/databases/${database.id}/tables` })).json()).toHaveLength(2);
|
||||
|
||||
const applied = await repository.reconcileTables(database.id, database.version, await scan(), ["visits"]);
|
||||
expect(applied).toMatchObject({ kind: "applied", deletedCount: 1 });
|
||||
expect((await app.inject({ method: "GET", url: `/catalog/databases/${database.id}/tables` })).json()).toMatchObject([
|
||||
{ name: "patients" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("refuses synchronization until the current binding has passed its connection test", async () => {
|
||||
const { app, repository, database } = await setup();
|
||||
await repository.update(database.id, database.version, {
|
||||
workspaceId: database.workspaceId,
|
||||
engine: database.engine,
|
||||
databaseName: database.databaseName,
|
||||
schema: database.schema,
|
||||
binding: database.binding,
|
||||
});
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version + 1, scope: "tables", tableIds: [] },
|
||||
});
|
||||
expect(response.statusCode).toBe(409);
|
||||
expect(response.json()).toMatchObject({ code: "schema_sync_conflict" });
|
||||
});
|
||||
@@ -289,3 +289,22 @@ test("loadConfig accepts only an absolute generic model key file", () => {
|
||||
expect(() => loadConfig({ THT_MODEL_API_KEY_FILE: " /run/secrets/key" }))
|
||||
.toThrow(/model credential configuration is invalid/);
|
||||
});
|
||||
|
||||
test("loadConfig accepts a file-backed catalog role and rejects partial catalog configuration", () => {
|
||||
expect(loadConfig({
|
||||
THT_CATALOG_DB_HOST: "catalog-db",
|
||||
THT_CATALOG_DB_NAME: "thothii_catalog",
|
||||
THT_CATALOG_RUNTIME_USER: "thothii_catalog_runtime",
|
||||
THT_CATALOG_RUNTIME_PASSWORD_FILE: "/run/secrets/catalog_runtime_password",
|
||||
}).catalogDatabase).toEqual({
|
||||
host: "catalog-db",
|
||||
port: 5432,
|
||||
database: "thothii_catalog",
|
||||
user: "thothii_catalog_runtime",
|
||||
passwordFile: "/run/secrets/catalog_runtime_password",
|
||||
});
|
||||
expect(() => loadConfig({ THT_CATALOG_DB_HOST: "catalog-db" }))
|
||||
.toThrow(/catalog database configuration is invalid/);
|
||||
expect(() => loadConfig({ THT_CATALOG_DATABASE_URL: "https://catalog.invalid/db" }))
|
||||
.toThrow(/catalog database configuration is invalid/);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user