feat: implement metadata catalog database management

This commit is contained in:
Codex
2026-08-27 22:43:54 +02:00
parent 705af3aeb2
commit 79c4c925b5
86 changed files with 12566 additions and 135 deletions
+218
View File
@@ -0,0 +1,218 @@
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
import { z } from "zod";
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
import { CatalogService, type CatalogSecretName } from "../catalog/service.js";
import { WorkspaceRegistryError } from "../workspaces/git-repository.js";
import {
CatalogConflictError,
CatalogOperationInProgressError,
CatalogUnavailableError,
DATABASE_TRANSPORTS,
type CatalogRepository,
type DatabaseConfigurationInput,
} from "../catalog/types.js";
import type { CatalogOperationCoordinator } from "../catalog/operation-coordinator.js";
const idSchema = z.uuid();
const workspaceIdSchema = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/);
const identifier = z.string().trim().min(1).max(128).regex(/^[A-Za-z_][A-Za-z0-9_$-]*$/);
const nonEmpty = z.string().trim().min(1).max(512);
const port = z.number().int().min(1).max(65_535);
const optionalText = nonEmpty.optional();
const sshHost = z.string().trim().min(1).max(255).regex(/^[A-Za-z0-9_.:\[\]-]+$/).optional();
const sshUsername = z.string().trim().min(1).max(128).regex(/^[A-Za-z0-9._-]+$/).optional();
const bindingSchema = z.object({
transport: z.enum(DATABASE_TRANSPORTS),
host: optionalText,
port: port.optional(),
username: optionalText,
baseUrl: z.url().max(2048).optional(),
restPath: z.string().regex(/^\/(?!\/)[^?#\\\u0000-\u001f]*$/).max(512).optional(),
restAuth: z.enum(["none", "bearer", "x-api-key"]).optional(),
tlsServername: optionalText,
sshHost,
sshPort: port.optional(),
sshUsername,
sshTargetHost: sshHost,
sshTargetPort: port.optional(),
}).strict().superRefine((binding, context) => {
const required = binding.transport === "postgres_direct"
? ["host", "port", "username"] as const
: binding.transport === "rest_api"
? ["baseUrl", "restPath", "restAuth"] as const
: ["username", "sshHost", "sshPort", "sshUsername", "sshTargetHost", "sshTargetPort"] as const;
for (const field of required) {
if (binding[field] === undefined) context.addIssue({ code: "custom", path: [field], message: "Required" });
}
});
const configSchema = z.object({
workspaceId: workspaceIdSchema,
engine: z.literal("postgres"),
databaseName: identifier,
schema: identifier,
binding: bindingSchema,
}).strict();
const updateSchema = configSchema.extend({ version: z.number().int().positive() });
const secretNames = [
"password",
"apiKey",
"sshPrivateKey",
"sshPrivateKeyPassphrase",
"sshKnownHosts",
"tlsCa",
] as const;
const secretsSchema = z.object({
version: z.number().int().positive(),
values: z.partialRecord(z.enum(secretNames), z.string().min(1).max(65_536)).refine((values) => Object.keys(values).length > 0),
}).strict();
const versionQuery = z.object({ version: z.coerce.number().int().positive() });
function safeError(reply: FastifyReply, error: unknown) {
if (error instanceof CatalogUnavailableError) {
return reply.code(503).send({ code: "catalog_unavailable", message: "Database catalog is unavailable." });
}
if (error instanceof CatalogConflictError) {
return reply.code(409).send({ code: "database_conflict", message: "This workspace already has a database configuration." });
}
if (error instanceof CatalogOperationInProgressError) {
return reply.code(409).send({ code: "database_operation_in_progress", message: "A database operation is already in progress." });
}
if (error instanceof z.ZodError) {
return reply.code(400).send({ code: "database_invalid", message: "Database configuration is invalid." });
}
if (error instanceof WorkspaceRegistryError) {
return reply.code(400).send({ code: "database_invalid", message: "Database configuration is invalid." });
}
return reply.code(500).send({ code: "database_operation_failed", message: "Database operation failed." });
}
function manage(request: FastifyRequest, reply: FastifyReply) {
return isPrincipalContext(requirePermission(request, reply, "database.manage"));
}
export function catalogDatabaseRoutes(
app: FastifyInstance,
deps: { repository: CatalogRepository; service: CatalogService; operations?: CatalogOperationCoordinator },
): void {
const mutate = async <T>(databaseId: string, operation: () => Promise<T>): Promise<T> => (
deps.operations ? await deps.operations.run(databaseId, operation) : await operation()
);
const activeSyncRun = async (databaseId: string) => {
const run = (await deps.repository.listSyncRuns(databaseId, 5)).find((candidate) =>
["queued", "running", "awaiting_confirmation", "applying"].includes(candidate.state));
if (!run) return undefined;
const {
observedSnapshot: _snapshot,
leaseOwner: _leaseOwner,
leaseExpiresAt: _leaseExpiresAt,
...summary
} = run;
return summary;
};
app.get("/catalog/status", async (request, reply) => {
if (!manage(request, reply)) return reply;
return { available: await deps.repository.available() };
});
app.get("/catalog/databases", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const rows = await deps.service.list();
return await Promise.all(rows.map(async (row) => (
row.id ? { ...row, activeSyncRun: await activeSyncRun(row.id) } : row
)));
} catch (error) { return safeError(reply, error); }
});
app.get("/catalog/databases/:id", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const id = idSchema.parse((request.params as { id?: unknown }).id);
const database = await deps.repository.get(id);
if (!database) return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
return {
...database,
configured: true,
secrets: deps.service.configuredSecrets(database.workspaceId),
activeSyncRun: await activeSyncRun(database.id),
};
} catch (error) { return safeError(reply, error); }
});
app.post("/catalog/databases", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const input = configSchema.parse(request.body) as DatabaseConfigurationInput;
const created = await deps.repository.create(await deps.service.normalizeInput(input));
return reply.code(201).send({ ...created, configured: true, secrets: deps.service.configuredSecrets(created.workspaceId) });
} catch (error) { return safeError(reply, error); }
});
app.patch("/catalog/databases/:id", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const id = idSchema.parse((request.params as { id?: unknown }).id);
const { version, ...input } = updateSchema.parse(request.body);
const current = await deps.repository.get(id);
if (!current) return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
if (current.workspaceId !== input.workspaceId) {
return reply.code(400).send({ code: "database_invalid", message: "Database configuration is invalid." });
}
const updated = await mutate(id, async () => await deps.repository.update(
id, version, await deps.service.normalizeInput(input as DatabaseConfigurationInput),
));
if (!updated) return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." });
return { ...updated, configured: true, secrets: deps.service.configuredSecrets(updated.workspaceId) };
} catch (error) { return safeError(reply, error); }
});
app.put("/catalog/databases/:id/secrets", async (request, reply) => {
if (!manage(request, reply)) return reply;
if (!isPrincipalContext(requirePermission(request, reply, "workspace.secrets.manage"))) return reply;
try {
const id = idSchema.parse((request.params as { id?: unknown }).id);
const { version, values } = secretsSchema.parse(request.body);
const database = await deps.repository.get(id);
if (!database) return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
if (database.version !== version) return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." });
const updated = await mutate(id, async () => {
const touched = await deps.repository.touch(id, version);
if (touched) deps.service.replaceSecrets(database.workspaceId, values as Partial<Record<CatalogSecretName, string>>);
return touched;
});
if (!updated) return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." });
return { ...updated, configured: true, secrets: deps.service.configuredSecrets(updated.workspaceId) };
} catch (error) { return safeError(reply, error); }
});
app.post("/catalog/databases/:id/test", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const id = idSchema.parse((request.params as { id?: unknown }).id);
const { version } = z.object({ version: z.number().int().positive() }).strict().parse(request.body);
const database = await deps.repository.get(id);
if (!database) return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
if (database.version !== version) return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." });
const tested = await deps.repository.recordTest(id, version, await deps.service.test(database));
if (!tested) return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." });
return { ...tested, configured: true, secrets: deps.service.configuredSecrets(tested.workspaceId) };
} catch (error) { return safeError(reply, error); }
});
app.delete("/catalog/databases/:id", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const id = idSchema.parse((request.params as { id?: unknown }).id);
const { version } = versionQuery.parse(request.query);
const database = await deps.repository.get(id);
if (!database) return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
const deleted = await mutate(id, async () => {
const removed = await deps.repository.delete(id, version);
if (removed) deps.service.forgetSecrets(database.workspaceId);
return removed;
});
if (!deleted) return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." });
return reply.code(204).send();
} catch (error) { return safeError(reply, error); }
});
}
+230
View File
@@ -0,0 +1,230 @@
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
import { setTimeout as delay } from "node:timers/promises";
import { z } from "zod";
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
import type { CatalogSyncWorker } from "../catalog/sync-worker.js";
import {
CatalogConflictError,
CatalogConnectorError,
CatalogOperationInProgressError,
CatalogUnavailableError,
type CatalogRepository,
type CatalogSyncRun,
} from "../catalog/types.js";
const idSchema = z.uuid();
const metadataSchema = z.object({
version: z.number().int().positive(),
description: z.string().max(20_000).nullable(),
generatedDescription: z.string().max(20_000).nullable(),
}).strict();
const createRunSchema = z.object({
version: z.number().int().positive(),
scope: z.enum(["tables", "columns", "relationships", "all"]),
tableIds: z.array(idSchema).max(10_000).default([]),
}).strict();
const confirmationSchema = z.object({ confirmationToken: z.string().uuid() }).strict();
const eventQuerySchema = z.object({ after: z.coerce.number().int().nonnegative().default(0) });
function manage(request: FastifyRequest, reply: FastifyReply) {
return isPrincipalContext(requirePermission(request, reply, "database.manage"));
}
function safeError(reply: FastifyReply, error: unknown) {
if (error instanceof CatalogUnavailableError) {
return reply.code(503).send({ code: "catalog_unavailable", message: "Database catalog is unavailable." });
}
if (error instanceof CatalogConflictError || error instanceof CatalogOperationInProgressError) {
return reply.code(409).send({ code: "schema_sync_conflict", message: error.message });
}
if (error instanceof CatalogConnectorError) {
return reply.code(502).send({ code: "schema_introspection_failed", message: "The database schema could not be read safely." });
}
if (error instanceof z.ZodError) {
return reply.code(400).send({ code: "schema_request_invalid", message: "Schema request is invalid." });
}
return reply.code(500).send({ code: "schema_operation_failed", message: "Schema operation failed." });
}
function normalized(value: string | null): string | null {
return value?.trim() || null;
}
function publicRun(run: CatalogSyncRun) {
const {
observedSnapshot: _snapshot,
leaseOwner: _leaseOwner,
leaseExpiresAt: _leaseExpiresAt,
...result
} = run;
return result;
}
export function catalogSchemaRoutes(
app: FastifyInstance,
deps: { repository: CatalogRepository; worker: CatalogSyncWorker },
): void {
app.get("/catalog/databases/:databaseId/tables/:tableId/columns", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const params = request.params as { databaseId?: unknown; tableId?: unknown };
const databaseId = idSchema.parse(params.databaseId);
const tableId = idSchema.parse(params.tableId);
if (!(await deps.repository.getTable(databaseId, tableId))) {
return reply.code(404).send({ code: "table_not_found", message: "Catalog table was not found." });
}
return await deps.repository.listColumns(databaseId, tableId);
} catch (error) { return safeError(reply, error); }
});
app.patch("/catalog/databases/:databaseId/tables/:tableId/columns/:columnId", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const params = request.params as { databaseId?: unknown; tableId?: unknown; columnId?: unknown };
const databaseId = idSchema.parse(params.databaseId);
const tableId = idSchema.parse(params.tableId);
const columnId = idSchema.parse(params.columnId);
const input = metadataSchema.parse(request.body);
const current = await deps.repository.getColumn(databaseId, tableId, columnId);
if (!current) return reply.code(404).send({ code: "column_not_found", message: "Catalog column was not found." });
if (current.version !== input.version) {
return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
}
const updated = await deps.repository.updateColumnMetadata(
databaseId,
tableId,
columnId,
input.version,
normalized(input.description),
normalized(input.generatedDescription),
);
if (!updated) return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
return updated;
} catch (error) { return safeError(reply, error); }
});
app.get("/catalog/databases/:databaseId/relationships", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
if (!(await deps.repository.get(databaseId))) {
return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
}
return await deps.repository.listRelationships(databaseId);
} catch (error) { return safeError(reply, error); }
});
app.post("/catalog/databases/:databaseId/sync-runs", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
const input = createRunSchema.parse(request.body);
const database = await deps.repository.get(databaseId);
if (!database) return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
if (database.version !== input.version) {
return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." });
}
return reply.code(202).send(publicRun(await deps.worker.start(database, input.scope, input.tableIds)));
} catch (error) { return safeError(reply, error); }
});
app.get("/catalog/databases/:databaseId/sync-runs", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
if (!(await deps.repository.get(databaseId))) {
return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
}
return (await deps.repository.listSyncRuns(databaseId)).map(publicRun);
} catch (error) { return safeError(reply, error); }
});
app.get("/catalog/sync-runs/:runId", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
const run = await deps.repository.getSyncRun(runId);
return run ? publicRun(run) : reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." });
} catch (error) { return safeError(reply, error); }
});
app.post("/catalog/sync-runs/:runId/confirm", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
const { confirmationToken } = confirmationSchema.parse(request.body);
const run = await deps.worker.confirm(runId, confirmationToken);
return run ? publicRun(run) : reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." });
} catch (error) { return safeError(reply, error); }
});
app.post("/catalog/sync-runs/:runId/cancel", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
const run = await deps.worker.cancel(runId);
return run ? publicRun(run) : reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." });
} catch (error) { return safeError(reply, error); }
});
app.post("/catalog/sync-runs/:runId/retry", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
const run = await deps.worker.retry(runId);
return run ? reply.code(202).send(publicRun(run)) : reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." });
} catch (error) { return safeError(reply, error); }
});
app.get("/catalog/sync-runs/:runId/events", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
let after = eventQuerySchema.parse(request.query).after;
const headerCursor = Number(request.headers["last-event-id"]);
if (Number.isInteger(headerCursor) && headerCursor >= 0) after = Math.max(after, headerCursor);
if (!(await deps.repository.getSyncRun(runId))) {
return reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." });
}
reply.hijack();
reply.raw.writeHead(200, {
"content-type": "text/event-stream; charset=utf-8",
"cache-control": "no-cache, no-transform",
connection: "keep-alive",
"x-accel-buffering": "no",
});
const controller = new AbortController();
request.raw.once("close", () => controller.abort());
let lastRunUpdate = "";
while (!controller.signal.aborted) {
const events = await deps.repository.listSyncEvents(runId, after);
for (const event of events) {
after = event.sequence;
reply.raw.write(`id: ${event.sequence}\nevent: log\ndata: ${JSON.stringify(event)}\n\n`);
}
const run = await deps.repository.getSyncRun(runId);
if (!run) break;
if (run.updatedAt !== lastRunUpdate) {
lastRunUpdate = run.updatedAt;
reply.raw.write(`event: run\ndata: ${JSON.stringify(publicRun(run))}\n\n`);
}
if (["succeeded", "failed", "cancelled", "interrupted"].includes(run.state)) break;
await delay(500, undefined, { signal: controller.signal }).catch(() => undefined);
}
reply.raw.end();
return reply;
} catch (error) { return safeError(reply, error); }
});
app.get("/catalog/sync-runs/:runId/events-list", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
const after = eventQuerySchema.parse(request.query).after;
if (!(await deps.repository.getSyncRun(runId))) {
return reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." });
}
return await deps.repository.listSyncEvents(runId, after);
} catch (error) { return safeError(reply, error); }
});
}
+79
View File
@@ -0,0 +1,79 @@
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
import { z } from "zod";
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
import type { CatalogTableService } from "../catalog/table-service.js";
import {
CatalogConnectorError,
CatalogOperationInProgressError,
CatalogUnavailableError,
type CatalogRepository,
} from "../catalog/types.js";
const idSchema = z.uuid();
const updateSchema = z.object({
version: z.number().int().positive(),
description: z.string().max(20_000).nullable(),
generatedDescription: z.string().max(20_000).nullable().optional(),
}).strict();
function manage(request: FastifyRequest, reply: FastifyReply) {
return isPrincipalContext(requirePermission(request, reply, "database.manage"));
}
function safeError(reply: FastifyReply, error: unknown) {
if (error instanceof CatalogUnavailableError) {
return reply.code(503).send({ code: "catalog_unavailable", message: "Database catalog is unavailable." });
}
if (error instanceof CatalogOperationInProgressError) {
return reply.code(409).send({ code: "database_operation_in_progress", message: "A database operation is already in progress." });
}
if (error instanceof CatalogConnectorError) {
return reply.code(502).send({ code: "table_introspection_failed", message: "Database tables could not be read." });
}
if (error instanceof z.ZodError) {
return reply.code(400).send({ code: "table_invalid", message: "Table request is invalid." });
}
return reply.code(500).send({ code: "table_operation_failed", message: "Table operation failed." });
}
export function catalogTableRoutes(
app: FastifyInstance,
deps: { repository: CatalogRepository; service: CatalogTableService },
): void {
app.get("/catalog/databases/:databaseId/tables", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
if (!(await deps.repository.get(databaseId))) {
return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
}
return await deps.service.list(databaseId);
} catch (error) { return safeError(reply, error); }
});
app.patch("/catalog/databases/:databaseId/tables/:tableId", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const params = request.params as { databaseId?: unknown; tableId?: unknown };
const databaseId = idSchema.parse(params.databaseId);
const tableId = idSchema.parse(params.tableId);
const input = updateSchema.parse(request.body);
const { version, description } = input;
const current = await deps.repository.getTable(databaseId, tableId);
if (!current) return reply.code(404).send({ code: "table_not_found", message: "Catalog table was not found." });
if (current.version !== version) {
return reply.code(409).send({ code: "table_stale", message: "Table description changed. Reload and try again." });
}
const updated = await deps.service.updateMetadata(
databaseId,
tableId,
version,
description,
input.generatedDescription === undefined ? current.generatedDescription : input.generatedDescription,
);
if (!updated) return reply.code(409).send({ code: "table_stale", message: "Table description changed. Reload and try again." });
return updated;
} catch (error) { return safeError(reply, error); }
});
}