feat: complete catalog fleet management workflow

2026-08-31 15:58:43 +02:00
parent 866aee4249
commit 9c697dc062
56 changed files with 6692 additions and 611 deletions
@@ -0,0 +1,60 @@
+import { http, HttpResponse } from "msw";
+import { expect, test } from "vitest";
+import { server } from "../test/msw";
+import { getCatalogMetrics, type CatalogMetrics } from "./catalog-databases";
+
+const databaseId = "d4baf0f5-b9c3-4dc5-aad2-2f5676a36e64";
+
+const globalMetrics: CatalogMetrics = {
+  scope: "global",
+  databaseId: null,
+  tables: 163,
+  columns: 2_275,
+  sensitiveColumns: 5,
+  relationships: 21,
+  descriptionTargets: 2_438,
+  describedTargets: 1_829,
+  descriptionCoverage: 75.02,
+  updatedAt: "2026-08-31T08:45:00.000Z",
+};
+
+test("loads global catalog metrics without a database filter", async () => {
+  let requestedUrl: URL | undefined;
+  server.use(http.get("/api/catalog/metrics", ({ request }) => {
+    requestedUrl = new URL(request.url);
+    return HttpResponse.json(globalMetrics);
+  }));
+
+  await expect(getCatalogMetrics()).resolves.toEqual(globalMetrics);
+  expect(requestedUrl?.pathname).toBe("/api/catalog/metrics");
+  expect(requestedUrl?.searchParams.has("databaseId")).toBe(false);
+});
+
+test("loads metrics scoped to the requested database", async () => {
+  const databaseMetrics: CatalogMetrics = {
+    ...globalMetrics,
+    scope: "database",
+    databaseId,
+    tables: 42,
+    columns: 508,
+    sensitiveColumns: 3,
+    relationships: 8,
+    descriptionTargets: 550,
+    describedTargets: 413,
+    descriptionCoverage: 75.09,
+  };
+  let requestedDatabaseId: string | null = null;
+  server.use(http.get("/api/catalog/metrics", ({ request }) => {
+    requestedDatabaseId = new URL(request.url).searchParams.get("databaseId");
+    return HttpResponse.json(databaseMetrics);
+  }));
+
+  await expect(getCatalogMetrics(databaseId)).resolves.toEqual(databaseMetrics);
+  expect(requestedDatabaseId).toBe(databaseId);
+});
+
+test("propagates an unsuccessful catalog metrics response", async () => {
+  server.use(http.get("/api/catalog/metrics", () => new HttpResponse(null, { status: 503 })));
+
+  await expect(getCatalogMetrics()).rejects.toMatchObject({ status: 503 });
+});
@@ -52,6 +52,19 @@ export interface CatalogDatabase {
  secrets: Record<CatalogSecretName, boolean>;
 }

+export interface CatalogMetrics {
+  scope: "global" | "database";
+  databaseId: string | null;
+  tables: number;
+  columns: number;
+  sensitiveColumns: number;
+  relationships: number;
+  descriptionTargets: number;
+  describedTargets: number;
+  descriptionCoverage: number;
+  updatedAt: string | null;
+}
+
 export interface DatabaseConfiguration {
  workspaceId: string;
  engine: "postgres";
@@ -109,6 +122,7 @@ export interface SensitiveDataSuggestion {

 export interface SensitiveDataSuggestions {
  suggestions: SensitiveDataSuggestion[];
+  run: SensitiveDataSuggestionRun;
 }

 export type SensitiveDataSuggestionRequest =
@@ -116,6 +130,32 @@ export type SensitiveDataSuggestionRequest =
  | { scope: "selected_tables"; targetIds: string[] }
  | { scope: "selected_columns"; targetIds: string[] };

+export type SensitiveDataSuggestionStatus = "running" | "completed" | "failed" | "interrupted";
+
+export interface SensitiveDataSuggestionRun {
+  id: string;
+  databaseId: string;
+  scope: SensitiveDataSuggestionRequest["scope"];
+  modelId: string;
+  status: SensitiveDataSuggestionStatus;
+  total: number;
+  suggestedSensitive: number;
+  suggestedNonSensitive: number;
+  createdAt: string;
+  startedAt: string | null;
+  updatedAt: string;
+  finishedAt: string | null;
+  errorSummary: string | null;
+}
+
+export interface SensitiveDataSuggestionEvent {
+  runId: string;
+  sequence: number;
+  level: "info" | "warning" | "error";
+  message: string;
+  createdAt: string;
+}
+
 export interface CatalogRelationshipColumn {
  position: number;
  sourceColumnId: string;
@@ -245,6 +285,12 @@ export interface DescriptionGenerationEvent {

 export const listCatalogDatabases = () => apiFetch<CatalogDatabase[]>("/catalog/databases");

+export const getCatalogMetrics = (databaseId?: string) => apiFetch<CatalogMetrics>(
+  databaseId
+    ? `/catalog/metrics?databaseId=${encodeURIComponent(databaseId)}`
+    : "/catalog/metrics",
+);
+
 export const listMetadataGenerationModels = () =>
  apiFetch<MetadataGenerationModels>("/catalog/metadata-generation/models");

@@ -388,6 +434,21 @@ export const suggestSensitiveFields = (
    { method: "POST", body: JSON.stringify({ modelId, ...selection }) },
  );

+export const getSensitiveDataSuggestionRun = (runId: string) =>
+  apiFetch<SensitiveDataSuggestionRun>(
+    `/catalog/sensitive-data-suggestion-runs/${encodeURIComponent(runId)}`,
+  );
+
+export const listSensitiveDataSuggestionRuns = (limit = 50) =>
+  apiFetch<SensitiveDataSuggestionRun[]>(
+    `/catalog/sensitive-data-suggestion-runs?limit=${encodeURIComponent(String(limit))}`,
+  );
+
+export const listSensitiveDataSuggestionEvents = (runId: string, after = 0) =>
+  apiFetch<SensitiveDataSuggestionEvent[]>(
+    `/catalog/sensitive-data-suggestion-runs/${encodeURIComponent(runId)}/events-list?after=${after}`,
+  );
+
 export const listCatalogRelationships = (databaseId: string) =>
  apiFetch<CatalogRelationship[]>(`/catalog/databases/${encodeURIComponent(databaseId)}/relationships`);

@@ -3,9 +3,13 @@ import { server } from "../test/msw";
 import {
  cancelDescriptionGenerationRun,
  descriptionGenerationEventsUrl,
+  getSensitiveDataSuggestionRun,
  listDescriptionGenerationRuns,
+  listSensitiveDataSuggestionEvents,
+  listSensitiveDataSuggestionRuns,
  unlockDescriptionGenerationRun,
  type DescriptionGenerationRun,
+  type SensitiveDataSuggestionRun,
 } from "./catalog-databases";

 const historicalRun: DescriptionGenerationRun = {
@@ -27,6 +31,22 @@ const historicalRun: DescriptionGenerationRun = {
  errorSummary: null,
 };

+const sensitiveSuggestionRun: SensitiveDataSuggestionRun = {
+  id: "99999999-9999-4999-8999-999999999999",
+  databaseId: "11111111-1111-4111-8111-111111111111",
+  scope: "selected_columns",
+  modelId: "local-qwen",
+  status: "completed",
+  total: 4,
+  suggestedSensitive: 2,
+  suggestedNonSensitive: 2,
+  createdAt: "2026-08-28T09:00:00Z",
+  startedAt: "2026-08-28T09:00:00Z",
+  updatedAt: "2026-08-28T09:00:01Z",
+  finishedAt: "2026-08-28T09:00:01Z",
+  errorSummary: null,
+};
+
 test("lists newest-first persisted description-generation history with a bounded limit", async () => {
  let requestedLimit: string | null = null;
  server.use(http.get("/api/catalog/description-generation-runs", ({ request }) => {
@@ -74,3 +94,34 @@ test("builds the same-origin description-generation SSE replay URL from an event
    `/api/catalog/description-generation-runs/${historicalRun.id}/events?after=17`,
  );
 });
+
+test("lists, reads, and replays persisted sensitive-suggestion history", async () => {
+  let requestedLimit: string | null = null;
+  let requestedAfter: string | null = null;
+  const event = {
+    runId: sensitiveSuggestionRun.id,
+    sequence: 3,
+    level: "info" as const,
+    message: "Classified 4 columns",
+    createdAt: "2026-08-28T09:00:01Z",
+  };
+  server.use(
+    http.get("/api/catalog/sensitive-data-suggestion-runs", ({ request }) => {
+      requestedLimit = new URL(request.url).searchParams.get("limit");
+      return HttpResponse.json([sensitiveSuggestionRun]);
+    }),
+    http.get("/api/catalog/sensitive-data-suggestion-runs/:runId", () => (
+      HttpResponse.json(sensitiveSuggestionRun)
+    )),
+    http.get("/api/catalog/sensitive-data-suggestion-runs/:runId/events-list", ({ request }) => {
+      requestedAfter = new URL(request.url).searchParams.get("after");
+      return HttpResponse.json([event]);
+    }),
+  );
+
+  await expect(listSensitiveDataSuggestionRuns(50)).resolves.toEqual([sensitiveSuggestionRun]);
+  await expect(getSensitiveDataSuggestionRun(sensitiveSuggestionRun.id)).resolves.toEqual(sensitiveSuggestionRun);
+  await expect(listSensitiveDataSuggestionEvents(sensitiveSuggestionRun.id, 2)).resolves.toEqual([event]);
+  expect(requestedLimit).toBe("50");
+  expect(requestedAfter).toBe("2");
+});
@@ -149,6 +149,9 @@ test.each([
  ["catalog_table_not_found", "One or more selected catalog tables were not found."],
  ["sensitive_data_suggestion_invalid_response", "The model returned an incomplete or invalid classification. No suggestions were applied."],
  ["sensitive_data_suggestion_provider_unavailable", "The selected model could not complete the request. No suggestions were applied."],
+  ["sensitive_data_suggestion_history_request_invalid", "Sensitive suggestion history parameters are invalid."],
+  ["sensitive_data_suggestion_history_failed", "Sensitive suggestion history could not be loaded."],
+  ["sensitive_data_suggestion_run_not_found", "The sensitive suggestion run was not found."],
 ])("maps the catalog error code %s to safe local copy", async (code, message) => {
  const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue(
    new Response(JSON.stringify({ code, message: "provider detail must not be trusted" }), {
@@ -30,6 +30,9 @@ const safeErrorCodes = new Set([
  "sensitive_data_suggestion_invalid_response",
  "sensitive_data_suggestion_provider_unavailable",
  "sensitive_data_suggestion_failed",
+  "sensitive_data_suggestion_history_request_invalid",
+  "sensitive_data_suggestion_history_failed",
+  "sensitive_data_suggestion_run_not_found",
  "schema_sync_conflict", "schema_introspection_failed", "schema_request_invalid",
  "schema_operation_failed", "sync_run_not_found", "table_stale", "column_stale",
 ]);
@@ -91,6 +94,9 @@ const localCodeMessages: Record<string, string> = {
  sensitive_data_suggestion_invalid_response: "The model returned an incomplete or invalid classification. No suggestions were applied.",
  sensitive_data_suggestion_provider_unavailable: "The selected model could not complete the request. No suggestions were applied.",
  sensitive_data_suggestion_failed: "Sensitive-field suggestions failed before review. No changes were applied.",
+  sensitive_data_suggestion_history_request_invalid: "Sensitive suggestion history parameters are invalid.",
+  sensitive_data_suggestion_history_failed: "Sensitive suggestion history could not be loaded.",
+  sensitive_data_suggestion_run_not_found: "The sensitive suggestion run was not found.",
  schema_sync_conflict: "A schema synchronization is already active or no longer current.",
  schema_introspection_failed: "The database schema could not be read safely.",
  schema_request_invalid: "The schema request is invalid.",