feat: implement metadata catalog database management
This commit is contained in:
Codex
@@ -0,0 +1,230 @@
|
||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { z } from "zod";
|
||||
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
|
||||
import type { CatalogSyncWorker } from "../catalog/sync-worker.js";
|
||||
import {
|
||||
CatalogConflictError,
|
||||
CatalogConnectorError,
|
||||
CatalogOperationInProgressError,
|
||||
CatalogUnavailableError,
|
||||
type CatalogRepository,
|
||||
type CatalogSyncRun,
|
||||
} from "../catalog/types.js";
|
||||
|
||||
const idSchema = z.uuid();
|
||||
const metadataSchema = z.object({
|
||||
version: z.number().int().positive(),
|
||||
description: z.string().max(20_000).nullable(),
|
||||
generatedDescription: z.string().max(20_000).nullable(),
|
||||
}).strict();
|
||||
const createRunSchema = z.object({
|
||||
version: z.number().int().positive(),
|
||||
scope: z.enum(["tables", "columns", "relationships", "all"]),
|
||||
tableIds: z.array(idSchema).max(10_000).default([]),
|
||||
}).strict();
|
||||
const confirmationSchema = z.object({ confirmationToken: z.string().uuid() }).strict();
|
||||
const eventQuerySchema = z.object({ after: z.coerce.number().int().nonnegative().default(0) });
|
||||
|
||||
function manage(request: FastifyRequest, reply: FastifyReply) {
|
||||
return isPrincipalContext(requirePermission(request, reply, "database.manage"));
|
||||
}
|
||||
|
||||
function safeError(reply: FastifyReply, error: unknown) {
|
||||
if (error instanceof CatalogUnavailableError) {
|
||||
return reply.code(503).send({ code: "catalog_unavailable", message: "Database catalog is unavailable." });
|
||||
}
|
||||
if (error instanceof CatalogConflictError || error instanceof CatalogOperationInProgressError) {
|
||||
return reply.code(409).send({ code: "schema_sync_conflict", message: error.message });
|
||||
}
|
||||
if (error instanceof CatalogConnectorError) {
|
||||
return reply.code(502).send({ code: "schema_introspection_failed", message: "The database schema could not be read safely." });
|
||||
}
|
||||
if (error instanceof z.ZodError) {
|
||||
return reply.code(400).send({ code: "schema_request_invalid", message: "Schema request is invalid." });
|
||||
}
|
||||
return reply.code(500).send({ code: "schema_operation_failed", message: "Schema operation failed." });
|
||||
}
|
||||
|
||||
function normalized(value: string | null): string | null {
|
||||
return value?.trim() || null;
|
||||
}
|
||||
|
||||
function publicRun(run: CatalogSyncRun) {
|
||||
const {
|
||||
observedSnapshot: _snapshot,
|
||||
leaseOwner: _leaseOwner,
|
||||
leaseExpiresAt: _leaseExpiresAt,
|
||||
...result
|
||||
} = run;
|
||||
return result;
|
||||
}
|
||||
|
||||
export function catalogSchemaRoutes(
|
||||
app: FastifyInstance,
|
||||
deps: { repository: CatalogRepository; worker: CatalogSyncWorker },
|
||||
): void {
|
||||
app.get("/catalog/databases/:databaseId/tables/:tableId/columns", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const params = request.params as { databaseId?: unknown; tableId?: unknown };
|
||||
const databaseId = idSchema.parse(params.databaseId);
|
||||
const tableId = idSchema.parse(params.tableId);
|
||||
if (!(await deps.repository.getTable(databaseId, tableId))) {
|
||||
return reply.code(404).send({ code: "table_not_found", message: "Catalog table was not found." });
|
||||
}
|
||||
return await deps.repository.listColumns(databaseId, tableId);
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.patch("/catalog/databases/:databaseId/tables/:tableId/columns/:columnId", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const params = request.params as { databaseId?: unknown; tableId?: unknown; columnId?: unknown };
|
||||
const databaseId = idSchema.parse(params.databaseId);
|
||||
const tableId = idSchema.parse(params.tableId);
|
||||
const columnId = idSchema.parse(params.columnId);
|
||||
const input = metadataSchema.parse(request.body);
|
||||
const current = await deps.repository.getColumn(databaseId, tableId, columnId);
|
||||
if (!current) return reply.code(404).send({ code: "column_not_found", message: "Catalog column was not found." });
|
||||
if (current.version !== input.version) {
|
||||
return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
|
||||
}
|
||||
const updated = await deps.repository.updateColumnMetadata(
|
||||
databaseId,
|
||||
tableId,
|
||||
columnId,
|
||||
input.version,
|
||||
normalized(input.description),
|
||||
normalized(input.generatedDescription),
|
||||
);
|
||||
if (!updated) return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
|
||||
return updated;
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.get("/catalog/databases/:databaseId/relationships", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
|
||||
if (!(await deps.repository.get(databaseId))) {
|
||||
return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
|
||||
}
|
||||
return await deps.repository.listRelationships(databaseId);
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.post("/catalog/databases/:databaseId/sync-runs", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
|
||||
const input = createRunSchema.parse(request.body);
|
||||
const database = await deps.repository.get(databaseId);
|
||||
if (!database) return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
|
||||
if (database.version !== input.version) {
|
||||
return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." });
|
||||
}
|
||||
return reply.code(202).send(publicRun(await deps.worker.start(database, input.scope, input.tableIds)));
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.get("/catalog/databases/:databaseId/sync-runs", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
|
||||
if (!(await deps.repository.get(databaseId))) {
|
||||
return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
|
||||
}
|
||||
return (await deps.repository.listSyncRuns(databaseId)).map(publicRun);
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.get("/catalog/sync-runs/:runId", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
|
||||
const run = await deps.repository.getSyncRun(runId);
|
||||
return run ? publicRun(run) : reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." });
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.post("/catalog/sync-runs/:runId/confirm", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
|
||||
const { confirmationToken } = confirmationSchema.parse(request.body);
|
||||
const run = await deps.worker.confirm(runId, confirmationToken);
|
||||
return run ? publicRun(run) : reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." });
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.post("/catalog/sync-runs/:runId/cancel", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
|
||||
const run = await deps.worker.cancel(runId);
|
||||
return run ? publicRun(run) : reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." });
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.post("/catalog/sync-runs/:runId/retry", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
|
||||
const run = await deps.worker.retry(runId);
|
||||
return run ? reply.code(202).send(publicRun(run)) : reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." });
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.get("/catalog/sync-runs/:runId/events", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
|
||||
let after = eventQuerySchema.parse(request.query).after;
|
||||
const headerCursor = Number(request.headers["last-event-id"]);
|
||||
if (Number.isInteger(headerCursor) && headerCursor >= 0) after = Math.max(after, headerCursor);
|
||||
if (!(await deps.repository.getSyncRun(runId))) {
|
||||
return reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." });
|
||||
}
|
||||
reply.hijack();
|
||||
reply.raw.writeHead(200, {
|
||||
"content-type": "text/event-stream; charset=utf-8",
|
||||
"cache-control": "no-cache, no-transform",
|
||||
connection: "keep-alive",
|
||||
"x-accel-buffering": "no",
|
||||
});
|
||||
const controller = new AbortController();
|
||||
request.raw.once("close", () => controller.abort());
|
||||
let lastRunUpdate = "";
|
||||
while (!controller.signal.aborted) {
|
||||
const events = await deps.repository.listSyncEvents(runId, after);
|
||||
for (const event of events) {
|
||||
after = event.sequence;
|
||||
reply.raw.write(`id: ${event.sequence}\nevent: log\ndata: ${JSON.stringify(event)}\n\n`);
|
||||
}
|
||||
const run = await deps.repository.getSyncRun(runId);
|
||||
if (!run) break;
|
||||
if (run.updatedAt !== lastRunUpdate) {
|
||||
lastRunUpdate = run.updatedAt;
|
||||
reply.raw.write(`event: run\ndata: ${JSON.stringify(publicRun(run))}\n\n`);
|
||||
}
|
||||
if (["succeeded", "failed", "cancelled", "interrupted"].includes(run.state)) break;
|
||||
await delay(500, undefined, { signal: controller.signal }).catch(() => undefined);
|
||||
}
|
||||
reply.raw.end();
|
||||
return reply;
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.get("/catalog/sync-runs/:runId/events-list", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
|
||||
const after = eventQuerySchema.parse(request.query).after;
|
||||
if (!(await deps.repository.getSyncRun(runId))) {
|
||||
return reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." });
|
||||
}
|
||||
return await deps.repository.listSyncEvents(runId, after);
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user