feat: complete catalog fleet management workflow

backend/src/routes/catalog-databases.ts

@@ -66,6 +66,7 @@ const secretsSchema = z.object({
   values: z.partialRecord(z.enum(secretNames), z.string().min(1).max(65_536)).refine((values) => Object.keys(values).length > 0),
 }).strict();
 const versionQuery = z.object({ version: z.coerce.number().int().positive() });
+const metricsQuery = z.object({ databaseId: z.uuid().optional() }).strict();
 
 function safeError(reply: FastifyReply, error: unknown) {
   if (error instanceof CatalogUnavailableError) {
@@ -114,6 +115,21 @@ export function catalogDatabaseRoutes(
     return { available: await deps.repository.available() };
   });
 
+  app.get("/catalog/metrics", async (request, reply) => {
+    if (!manage(request, reply)) return reply;
+    try {
+      const { databaseId } = metricsQuery.parse(request.query);
+      const metrics = await deps.repository.getCatalogMetrics(databaseId);
+      if (!metrics) {
+        return reply.code(404).send({
+          code: "database_not_found",
+          message: "Database configuration was not found.",
+        });
+      }
+      return metrics;
+    } catch (error) { return safeError(reply, error); }
+  });
+
   app.get("/catalog/databases", async (request, reply) => {
     if (!manage(request, reply)) return reply;
     try {

backend/src/routes/catalog-description-generation.ts

@@ -13,13 +13,13 @@ import {
 import { MetadataGenerationModelUnavailableError } from "../catalog/metadata-generation-models.js";
 import { ModelCompletionProviderError } from "../catalog/model-completer.js";
 import {
-  SensitiveDataSuggester,
   SensitiveDataSuggestionDuplicateTargetIdsError,
   SensitiveDataSuggestionInvalidResponseError,
   SensitiveDataSuggestionNoEligibleColumnsError,
   SensitiveDataSuggestionPayloadTooLargeError,
   SensitiveDataSuggestionTargetNotFoundError,
 } from "../catalog/sensitive-data-suggester.js";
+import type { SensitiveDataSuggestionRunner } from "../catalog/sensitive-data-suggestion-runner.js";
 import {
   CatalogOperationInProgressError,
   CatalogUnavailableError,
@@ -27,6 +27,8 @@ import {
   type CatalogRepository,
   type DescriptionGenerationEvent,
   type DescriptionGenerationRun,
+  type SensitiveDataSuggestionEvent,
+  type SensitiveDataSuggestionRun,
 } from "../catalog/types.js";
 
 const idSchema = z.uuid();
@@ -107,6 +109,34 @@ function publicRun(run: DescriptionGenerationRun) {
   };
 }
 
+function publicSensitiveDataSuggestionEvent(event: SensitiveDataSuggestionEvent) {
+  return {
+    runId: event.runId,
+    sequence: event.sequence,
+    level: event.level,
+    message: event.message,
+    createdAt: event.createdAt,
+  };
+}
+
+function publicSensitiveDataSuggestionRun(run: SensitiveDataSuggestionRun) {
+  return {
+    id: run.id,
+    databaseId: run.databaseId,
+    scope: run.scope,
+    modelId: run.modelId,
+    status: run.status,
+    total: run.total,
+    suggestedSensitive: run.suggestedSensitive,
+    suggestedNonSensitive: run.suggestedNonSensitive,
+    createdAt: run.createdAt,
+    startedAt: run.startedAt,
+    updatedAt: run.updatedAt,
+    finishedAt: run.finishedAt,
+    errorSummary: run.errorSummary,
+  };
+}
+
 function safeError(reply: FastifyReply, error: unknown) {
   if (error instanceof CatalogUnavailableError) {
     return reply.code(503).send({
@@ -260,12 +290,31 @@ function safeSuggestionError(reply: FastifyReply, error: unknown) {
   });
 }
 
+function safeSuggestionHistoryError(reply: FastifyReply, error: unknown) {
+  if (error instanceof CatalogUnavailableError) {
+    return reply.code(503).send({
+      code: "catalog_unavailable",
+      message: "Sensitive Data Suggestion history is unavailable because the database catalog is unavailable.",
+    });
+  }
+  if (error instanceof z.ZodError) {
+    return reply.code(400).send({
+      code: "sensitive_data_suggestion_history_request_invalid",
+      message: "Sensitive Data Suggestion history parameters are invalid.",
+    });
+  }
+  return reply.code(500).send({
+    code: "sensitive_data_suggestion_history_failed",
+    message: "Sensitive Data Suggestion history could not be loaded.",
+  });
+}
+
 export function catalogDescriptionGenerationRoutes(
   app: FastifyInstance,
   deps: {
     repository: CatalogRepository;
     worker: DescriptionGenerationWorker;
-    sensitiveDataSuggester: SensitiveDataSuggester;
+    sensitiveDataSuggestionRunner: SensitiveDataSuggestionRunner;
   },
 ): void {
   app.post("/catalog/databases/:databaseId/sensitive-data-suggestions", async (request, reply) => {
@@ -273,19 +322,66 @@ export function catalogDescriptionGenerationRoutes(
     try {
       const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
       const input = suggestionSchema.parse(request.body);
-      const suggestions = await deps.sensitiveDataSuggester.suggest(
+      const result = await deps.sensitiveDataSuggestionRunner.run(
         databaseId,
         input.modelId,
         input.scope,
         "targetIds" in input ? input.targetIds : [],
         new AbortController().signal,
       );
-      return { suggestions };
+      return {
+        suggestions: result.suggestions,
+        run: publicSensitiveDataSuggestionRun(result.run),
+      };
     } catch (error) {
       return safeSuggestionError(reply, error);
     }
   });
 
+  app.get("/catalog/sensitive-data-suggestion-runs", async (request, reply) => {
+    if (!manage(request, reply)) return reply;
+    try {
+      const { limit } = historyQuerySchema.parse(request.query);
+      return (await deps.repository.listSensitiveDataSuggestionRuns(limit))
+        .map(publicSensitiveDataSuggestionRun);
+    } catch (error) {
+      return safeSuggestionHistoryError(reply, error);
+    }
+  });
+
+  app.get("/catalog/sensitive-data-suggestion-runs/:runId", async (request, reply) => {
+    if (!manage(request, reply)) return reply;
+    try {
+      const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
+      const run = await deps.repository.getSensitiveDataSuggestionRun(runId);
+      if (!run) return reply.code(404).send({
+        code: "sensitive_data_suggestion_run_not_found",
+        message: "Sensitive Data Suggestion Run was not found.",
+      });
+      return publicSensitiveDataSuggestionRun(run);
+    } catch (error) {
+      return safeSuggestionHistoryError(reply, error);
+    }
+  });
+
+  app.get("/catalog/sensitive-data-suggestion-runs/:runId/events-list", async (request, reply) => {
+    if (!manage(request, reply)) return reply;
+    try {
+      const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
+      const { after } = eventQuerySchema.parse(request.query);
+      if (!(await deps.repository.getSensitiveDataSuggestionRun(runId))) {
+        return reply.code(404).send({
+          code: "sensitive_data_suggestion_run_not_found",
+          message: "Sensitive Data Suggestion Run was not found.",
+        });
+      }
+      return (await deps.repository.listSensitiveDataSuggestionEvents(runId, after))
+        .map(publicSensitiveDataSuggestionEvent);
+    } catch (error) {
+      return safeSuggestionHistoryError(reply, error);
+    }
+  });
+
   app.post("/catalog/databases/:databaseId/description-generation-runs", async (request, reply) => {
     if (!manage(request, reply)) return reply;
     try {