feat: add AI catalog description generation

2026-08-29 16:42:56 +02:00
parent b0afba81ca
commit 376dd5a09d
76 changed files with 14860 additions and 102 deletions
@@ -0,0 +1,72 @@
+import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
+import { z } from "zod";
+import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
+import type { CatalogOperationCoordinator } from "../catalog/operation-coordinator.js";
+import {
+  CatalogOperationInProgressError,
+  CatalogUnavailableError,
+  type CatalogRepository,
+} from "../catalog/types.js";
+
+const idSchema = z.uuid();
+const consolidationSchema = z.object({
+  target: z.enum(["tables", "columns"]),
+  targetIds: z.array(idSchema).min(1).max(10_000),
+}).strict();
+
+function manage(request: FastifyRequest, reply: FastifyReply) {
+  return isPrincipalContext(requirePermission(request, reply, "database.manage"));
+}
+
+function safeError(reply: FastifyReply, error: unknown) {
+  if (error instanceof CatalogUnavailableError) {
+    return reply.code(503).send({ code: "catalog_unavailable", message: "Database catalog is unavailable." });
+  }
+  if (error instanceof CatalogOperationInProgressError) {
+    return reply.code(409).send({
+      code: "database_operation_in_progress",
+      message: "A database operation is already in progress.",
+    });
+  }
+  if (error instanceof z.ZodError) {
+    return reply.code(400).send({
+      code: "description_consolidation_invalid",
+      message: "Description consolidation request is invalid.",
+    });
+  }
+  return reply.code(500).send({
+    code: "description_consolidation_failed",
+    message: "Description consolidation failed.",
+  });
+}
+
+export function catalogDescriptionConsolidationRoutes(
+  app: FastifyInstance,
+  deps: { repository: CatalogRepository; operations: CatalogOperationCoordinator },
+): void {
+  app.post("/catalog/databases/:databaseId/descriptions/consolidate", async (request, reply) => {
+    if (!manage(request, reply)) return reply;
+    try {
+      const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
+      const input = consolidationSchema.parse(request.body);
+      const targetIds = [...new Set(input.targetIds)];
+      const result = await deps.operations.run(
+        databaseId,
+        async () => await deps.repository.consolidateGeneratedDescriptions(
+          databaseId,
+          input.target,
+          targetIds,
+        ),
+      );
+      if (!result) {
+        return reply.code(404).send({
+          code: "catalog_target_not_found",
+          message: "The database or one or more selected catalog targets were not found.",
+        });
+      }
+      return result;
+    } catch (error) {
+      return safeError(reply, error);
+    }
+  });
+}
@@ -0,0 +1,347 @@
+import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
+import { z } from "zod";
+import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
+import {
+  DescriptionGenerationDuplicateTargetIdsError,
+  DescriptionGenerationNoEligibleTargetsError,
+  DescriptionGenerationRunLiveError,
+  DescriptionGenerationTargetIdsRequiredError,
+  DescriptionGenerationTargetNotFoundError,
+  DescriptionGenerationWorkspaceUnavailableError,
+  type DescriptionGenerationWorker,
+} from "../catalog/description-generation-worker.js";
+import { MetadataGenerationModelUnavailableError } from "../catalog/metadata-generation-models.js";
+import {
+  CatalogOperationInProgressError,
+  CatalogUnavailableError,
+  DescriptionGenerationRunActiveError,
+  type CatalogRepository,
+  type DescriptionGenerationEvent,
+  type DescriptionGenerationRun,
+} from "../catalog/types.js";
+
+const idSchema = z.uuid();
+const modelIdSchema = z.string().regex(/^[a-z][a-z0-9._-]{0,63}$/);
+const selectedTargetIdsSchema = z.array(idSchema).min(1);
+const startSchema = z.discriminatedUnion("scope", [
+  z.object({
+    modelId: modelIdSchema,
+    scope: z.literal("selected_columns"),
+    targetIds: selectedTargetIdsSchema,
+  }).strict(),
+  z.object({
+    modelId: modelIdSchema,
+    scope: z.literal("selected_tables"),
+    targetIds: selectedTargetIdsSchema,
+  }).strict(),
+  z.object({ modelId: modelIdSchema, scope: z.literal("all") }).strict(),
+  z.object({ modelId: modelIdSchema, scope: z.literal("missing") }).strict(),
+]);
+const eventQuerySchema = z.object({
+  after: z.coerce.number().int().nonnegative().default(0),
+}).strict();
+const historyQuerySchema = z.object({
+  limit: z.coerce.number().int().min(1).max(100).default(50),
+}).strict();
+const terminalStatuses = new Set<DescriptionGenerationRun["status"]>([
+  "completed",
+  "completed_with_errors",
+  "cancelled",
+  "failed",
+  "interrupted",
+]);
+
+function manage(request: FastifyRequest, reply: FastifyReply) {
+  return isPrincipalContext(requirePermission(request, reply, "database.manage"));
+}
+
+function publicEvent(event: DescriptionGenerationEvent) {
+  return {
+    sequence: event.sequence,
+    level: event.level,
+    message: event.message,
+    createdAt: event.createdAt,
+  };
+}
+
+function publicRun(run: DescriptionGenerationRun) {
+  return {
+    id: run.id,
+    databaseId: run.databaseId,
+    scope: run.scope,
+    modelId: run.modelId,
+    language: run.language,
+    status: run.status,
+    total: run.total,
+    processed: run.processed,
+    generated: run.generated,
+    nonGeneratable: run.nonGeneratable,
+    failed: run.failed,
+    createdAt: run.createdAt,
+    startedAt: run.startedAt,
+    updatedAt: run.updatedAt,
+    finishedAt: run.finishedAt,
+    errorSummary: run.errorSummary,
+  };
+}
+
+function safeError(reply: FastifyReply, error: unknown) {
+  if (error instanceof CatalogUnavailableError) {
+    return reply.code(503).send({
+      code: "catalog_unavailable",
+      message: "Database catalog is unavailable.",
+    });
+  }
+  if (error instanceof DescriptionGenerationRunActiveError) {
+    return reply.code(409).send({
+      code: "description_generation_run_active",
+      message: "A Description Generation Run is already active.",
+    });
+  }
+  if (error instanceof DescriptionGenerationRunLiveError) {
+    return reply.code(409).send({
+      code: "description_generation_run_live",
+      message: "A local Description Generation worker or helper is still running.",
+    });
+  }
+  if (error instanceof CatalogOperationInProgressError) {
+    return reply.code(409).send({
+      code: "database_operation_in_progress",
+      message: "A database operation is already in progress.",
+    });
+  }
+  if (error instanceof MetadataGenerationModelUnavailableError) {
+    return reply.code(409).send({
+      code: "metadata_generation_model_unavailable",
+      message: "The selected metadata-generation model is unavailable.",
+    });
+  }
+  if (error instanceof DescriptionGenerationDuplicateTargetIdsError) {
+    return reply.code(400).send({
+      code: "description_generation_target_ids_duplicate",
+      message: "Description generation target IDs must be unique.",
+    });
+  }
+  if (error instanceof DescriptionGenerationTargetIdsRequiredError) {
+    return reply.code(400).send({
+      code: "description_generation_request_invalid",
+      message: "At least one description generation target ID is required.",
+    });
+  }
+  if (error instanceof DescriptionGenerationNoEligibleTargetsError) {
+    return reply.code(409).send({
+      code: "description_generation_no_eligible_targets",
+      message: error.scope === "all"
+        ? "No Catalog Tables or Catalog Columns are available for description generation."
+        : "No Catalog Tables or Catalog Columns have a missing Generated Description.",
+    });
+  }
+  if (error instanceof DescriptionGenerationTargetNotFoundError) {
+    const code = error.target === "database"
+      ? "database_not_found"
+      : error.target === "table"
+        ? "catalog_table_not_found"
+        : "catalog_column_not_found";
+    const message = error.target === "database"
+      ? "Database configuration was not found."
+      : error.target === "table"
+        ? "One or more selected Catalog Tables were not found."
+        : "One or more selected Catalog Columns were not found.";
+    return reply.code(404).send({
+      code,
+      message,
+    });
+  }
+  if (error instanceof DescriptionGenerationWorkspaceUnavailableError) {
+    return reply.code(409).send({
+      code: "workspace_configuration_unavailable",
+      message: "The database workspace configuration is unavailable.",
+    });
+  }
+  if (error instanceof z.ZodError) {
+    return reply.code(400).send({
+      code: "description_generation_request_invalid",
+      message: "Description generation request is invalid.",
+    });
+  }
+  return reply.code(500).send({
+    code: "description_generation_failed",
+    message: "Description generation failed.",
+  });
+}
+
+export function catalogDescriptionGenerationRoutes(
+  app: FastifyInstance,
+  deps: { repository: CatalogRepository; worker: DescriptionGenerationWorker },
+): void {
+  app.post("/catalog/databases/:databaseId/description-generation-runs", async (request, reply) => {
+    if (!manage(request, reply)) return reply;
+    try {
+      const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
+      const input = startSchema.parse(request.body);
+      const targetIds = "targetIds" in input ? input.targetIds : [];
+      const run = await deps.worker.start(databaseId, input.modelId, input.scope, targetIds);
+      return reply.code(202).send(publicRun(run));
+    } catch (error) {
+      return safeError(reply, error);
+    }
+  });
+
+  app.get("/catalog/description-generation-runs", async (request, reply) => {
+    if (!manage(request, reply)) return reply;
+    try {
+      const { limit } = historyQuerySchema.parse(request.query);
+      return (await deps.repository.listDescriptionGenerationRuns(limit)).map(publicRun);
+    } catch (error) {
+      return safeError(reply, error);
+    }
+  });
+
+  app.get("/catalog/description-generation-runs/:runId", async (request, reply) => {
+    if (!manage(request, reply)) return reply;
+    try {
+      const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
+      const run = await deps.repository.getDescriptionGenerationRun(runId);
+      return run ? publicRun(run) : reply.code(404).send({
+        code: "description_generation_run_not_found",
+        message: "Description Generation Run was not found.",
+      });
+    } catch (error) {
+      return safeError(reply, error);
+    }
+  });
+
+  app.post("/catalog/description-generation-runs/:runId/cancel", async (request, reply) => {
+    if (!manage(request, reply)) return reply;
+    try {
+      const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
+      const run = await deps.worker.cancel(runId);
+      return run ? publicRun(run) : reply.code(404).send({
+        code: "description_generation_run_not_found",
+        message: "Description Generation Run was not found.",
+      });
+    } catch (error) {
+      return safeError(reply, error);
+    }
+  });
+
+  app.post("/catalog/description-generation-runs/unlock", async (request, reply) => {
+    if (!manage(request, reply)) return reply;
+    try {
+      const run = await deps.worker.unlock();
+      return run ? publicRun(run) : reply.code(404).send({
+        code: "description_generation_run_not_found",
+        message: "No stale active Description Generation Run was found.",
+      });
+    } catch (error) {
+      return safeError(reply, error);
+    }
+  });
+
+  app.get("/catalog/description-generation-runs/:runId/events", async (request, reply) => {
+    if (!manage(request, reply)) return reply;
+    let unsubscribe: (() => void) | undefined;
+    let hijacked = false;
+    try {
+      const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
+      let after = eventQuerySchema.parse(request.query).after;
+      const headerCursor = Number(request.headers["last-event-id"]);
+      if (Number.isInteger(headerCursor) && headerCursor >= 0) after = Math.max(after, headerCursor);
+      if (!(await deps.repository.getDescriptionGenerationRun(runId))) {
+        return reply.code(404).send({
+          code: "description_generation_run_not_found",
+          message: "Description Generation Run was not found.",
+        });
+      }
+
+      const buffered: DescriptionGenerationEvent[] = [];
+      let ready = false;
+      let closed = false;
+      let lastRunSnapshot = "";
+      let delivery = Promise.resolve();
+      const close = () => {
+        if (closed) return;
+        closed = true;
+        unsubscribe?.();
+        if (!reply.raw.destroyed) reply.raw.end();
+      };
+      const writeRun = (run: DescriptionGenerationRun) => {
+        if (closed) return;
+        const snapshot = JSON.stringify(publicRun(run));
+        if (snapshot === lastRunSnapshot) return;
+        lastRunSnapshot = snapshot;
+        reply.raw.write(`event: run\ndata: ${snapshot}\n\n`);
+        if (terminalStatuses.has(run.status)) close();
+      };
+      const enqueue = (event: DescriptionGenerationEvent) => {
+        delivery = delivery.then(async () => {
+          if (closed || event.sequence <= after) return;
+          after = event.sequence;
+          reply.raw.write(
+            `id: ${event.sequence}\nevent: log\ndata: ${JSON.stringify(publicEvent(event))}\n\n`,
+          );
+          const run = await deps.repository.getDescriptionGenerationRun(runId);
+          if (run) writeRun(run);
+        }).catch(close);
+      };
+      unsubscribe = deps.worker.subscribeEvents(runId, (event) => {
+        if (ready) enqueue(event);
+        else buffered.push(event);
+      });
+      const persisted = await deps.repository.listDescriptionGenerationEvents(runId, after);
+
+      reply.hijack();
+      hijacked = true;
+      reply.raw.writeHead(200, {
+        "content-type": "text/event-stream; charset=utf-8",
+        "cache-control": "no-cache, no-transform",
+        connection: "keep-alive",
+        "x-accel-buffering": "no",
+      });
+      reply.raw.once("close", close);
+      request.raw.once("aborted", close);
+      for (const event of persisted) {
+        if (event.sequence <= after) continue;
+        after = event.sequence;
+        reply.raw.write(
+          `id: ${event.sequence}\nevent: log\ndata: ${JSON.stringify(publicEvent(event))}\n\n`,
+        );
+      }
+      ready = true;
+      buffered.sort((a, b) => a.sequence - b.sequence).forEach(enqueue);
+      let pending = delivery;
+      await pending;
+      while (pending !== delivery) {
+        pending = delivery;
+        await pending;
+      }
+      const current = await deps.repository.getDescriptionGenerationRun(runId);
+      if (current) writeRun(current);
+      return reply;
+    } catch (error) {
+      unsubscribe?.();
+      if (hijacked) {
+        if (!reply.raw.destroyed) reply.raw.end();
+        return reply;
+      }
+      return safeError(reply, error);
+    }
+  });
+
+  app.get("/catalog/description-generation-runs/:runId/events-list", async (request, reply) => {
+    if (!manage(request, reply)) return reply;
+    try {
+      const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
+      const { after } = eventQuerySchema.parse(request.query);
+      if (!(await deps.repository.getDescriptionGenerationRun(runId))) {
+        return reply.code(404).send({
+          code: "description_generation_run_not_found",
+          message: "Description Generation Run was not found.",
+        });
+      }
+      return (await deps.repository.listDescriptionGenerationEvents(runId, after)).map(publicEvent);
+    } catch (error) {
+      return safeError(reply, error);
+    }
+  });
+}
@@ -43,7 +43,13 @@ function safeError(reply: FastifyReply, error: unknown) {
  if (error instanceof CatalogUnavailableError) {
    return reply.code(503).send({ code: "catalog_unavailable", message: "Database catalog is unavailable." });
  }
-  if (error instanceof CatalogConflictError || error instanceof CatalogOperationInProgressError) {
+  if (error instanceof CatalogOperationInProgressError) {
+    return reply.code(409).send({
+      code: "database_operation_in_progress",
+      message: "A database operation is already in progress.",
+    });
+  }
+  if (error instanceof CatalogConflictError) {
    return reply.code(409).send({ code: "schema_sync_conflict", message: error.message });
  }
  if (error instanceof CatalogConnectorError) {
@@ -98,21 +104,24 @@ export function catalogSchemaRoutes(
      const tableId = idSchema.parse(params.tableId);
      const columnId = idSchema.parse(params.columnId);
      const input = metadataSchema.parse(request.body);
-      const current = await deps.repository.getColumn(databaseId, tableId, columnId);
-      if (!current) return reply.code(404).send({ code: "column_not_found", message: "Catalog column was not found." });
-      if (current.version !== input.version) {
-        return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
-      }
-      const updated = await deps.repository.updateColumnMetadata(
-        databaseId,
-        tableId,
-        columnId,
-        input.version,
-        normalized(input.description),
-        normalized(input.generatedDescription),
-      );
-      if (!updated) return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
-      return updated;
+      const operation = async () => {
+        const current = await deps.repository.getColumn(databaseId, tableId, columnId);
+        if (!current) return reply.code(404).send({ code: "column_not_found", message: "Catalog column was not found." });
+        if (current.version !== input.version) {
+          return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
+        }
+        const updated = await deps.repository.updateColumnMetadata(
+          databaseId,
+          tableId,
+          columnId,
+          input.version,
+          normalized(input.description),
+          normalized(input.generatedDescription),
+        );
+        if (!updated) return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
+        return updated;
+      };
+      return deps.operations ? await deps.operations.run(databaseId, operation) : await operation();
    } catch (error) { return safeError(reply, error); }
  });

@@ -2,6 +2,7 @@ import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
 import { z } from "zod";
 import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
 import type { CatalogTableService } from "../catalog/table-service.js";
+import type { CatalogOperationCoordinator } from "../catalog/operation-coordinator.js";
 import {
  CatalogConnectorError,
  CatalogOperationInProgressError,
@@ -38,7 +39,11 @@ function safeError(reply: FastifyReply, error: unknown) {

 export function catalogTableRoutes(
  app: FastifyInstance,
-  deps: { repository: CatalogRepository; service: CatalogTableService },
+  deps: {
+    repository: CatalogRepository;
+    service: CatalogTableService;
+    operations: CatalogOperationCoordinator;
+  },
 ): void {
  app.get("/catalog/databases/:databaseId/tables", async (request, reply) => {
    if (!manage(request, reply)) return reply;
@@ -59,20 +64,22 @@ export function catalogTableRoutes(
      const tableId = idSchema.parse(params.tableId);
      const input = updateSchema.parse(request.body);
      const { version, description } = input;
-      const current = await deps.repository.getTable(databaseId, tableId);
-      if (!current) return reply.code(404).send({ code: "table_not_found", message: "Catalog table was not found." });
-      if (current.version !== version) {
-        return reply.code(409).send({ code: "table_stale", message: "Table description changed. Reload and try again." });
-      }
-      const updated = await deps.service.updateMetadata(
-        databaseId,
-        tableId,
-        version,
-        description,
-        input.generatedDescription === undefined ? current.generatedDescription : input.generatedDescription,
-      );
-      if (!updated) return reply.code(409).send({ code: "table_stale", message: "Table description changed. Reload and try again." });
-      return updated;
+      return await deps.operations.run(databaseId, async () => {
+        const current = await deps.repository.getTable(databaseId, tableId);
+        if (!current) return reply.code(404).send({ code: "table_not_found", message: "Catalog table was not found." });
+        if (current.version !== version) {
+          return reply.code(409).send({ code: "table_stale", message: "Table description changed. Reload and try again." });
+        }
+        const updated = await deps.service.updateMetadata(
+          databaseId,
+          tableId,
+          version,
+          description,
+          input.generatedDescription === undefined ? current.generatedDescription : input.generatedDescription,
+        );
+        if (!updated) return reply.code(409).send({ code: "table_stale", message: "Table description changed. Reload and try again." });
+        return updated;
+      });
    } catch (error) { return safeError(reply, error); }
  });

@@ -0,0 +1,13 @@
+import type { FastifyInstance } from "fastify";
+import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
+import type { MetadataGenerationModels } from "../catalog/metadata-generation-models.js";
+
+export function metadataGenerationModelRoutes(
+  app: FastifyInstance,
+  models: MetadataGenerationModels,
+): void {
+  app.get("/catalog/metadata-generation/models", async (request, reply) => {
+    if (!isPrincipalContext(requirePermission(request, reply, "database.manage"))) return reply;
+    return models.catalog();
+  });
+}