feat: complete catalog fleet management workflow

backend/src/catalog/memory-repository.ts

@@ -1,4 +1,9 @@
 import { randomUUID } from "node:crypto";
+import {
+  createCatalogMetrics,
+  hasCatalogDescription,
+  latestCatalogTimestamp,
+} from "./metrics.js";
 import {
   CatalogConflictError,
   CatalogConnectorError,
@@ -8,6 +13,7 @@ import {
   type CatalogDescriptionTarget,
   type CatalogDatabaseMetadataDeleteTarget,
   type CatalogMetadataDeleteCounts,
+  type CatalogMetrics,
   type CatalogRelationship,
   type CatalogSchemaDiff,
   type CatalogSyncCounts,
@@ -26,6 +32,10 @@ import {
   type DescriptionGenerationRun,
   type DescriptionGenerationRunUpdate,
   type DescriptionGenerationScope,
+  type SensitiveDataSuggestionEvent,
+  type SensitiveDataSuggestionRun,
+  type SensitiveDataSuggestionRunUpdate,
+  type SensitiveDataSuggestionScope,
   type TableSyncRepositoryResult,
   type WorkspaceDatabase,
 } from "./types.js";
@@ -42,6 +52,8 @@ export class MemoryCatalogRepository implements CatalogRepository {
   private readonly relationships = new Map<string, CatalogRelationship>();
   private readonly descriptionGenerationRuns = new Map<string, DescriptionGenerationRun>();
   private readonly descriptionGenerationEvents = new Map<string, DescriptionGenerationEvent[]>();
+  private readonly sensitiveDataSuggestionRuns = new Map<string, SensitiveDataSuggestionRun>();
+  private readonly sensitiveDataSuggestionEvents = new Map<string, SensitiveDataSuggestionEvent[]>();
   private readonly syncRuns = new Map<string, CatalogSyncRun>();
   private readonly syncEvents = new Map<string, CatalogSyncEvent[]>();
 
@@ -56,6 +68,34 @@ export class MemoryCatalogRepository implements CatalogRepository {
     const value = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
     return value ? clone(value) : undefined;
   }
+  async getCatalogMetrics(databaseId?: string): Promise<CatalogMetrics | undefined> {
+    if (databaseId !== undefined && !this.records.has(databaseId)) return undefined;
+
+    const selectedDatabaseIds = new Set(
+      databaseId === undefined ? this.records.keys() : [databaseId],
+    );
+    const tables = [...this.tables.values()]
+      .filter((table) => selectedDatabaseIds.has(table.databaseId));
+    const tableIds = new Set(tables.map((table) => table.id));
+    const columns = [...this.columns.values()]
+      .filter((column) => tableIds.has(column.tableId));
+    const relationships = [...this.relationships.values()]
+      .filter((relationship) => selectedDatabaseIds.has(relationship.databaseId));
+
+    return createCatalogMetrics(databaseId, {
+      tables: tables.length,
+      columns: columns.length,
+      sensitiveColumns: columns.filter((column) => column.sensitive).length,
+      relationships: relationships.length,
+      describedTables: tables.filter(hasCatalogDescription).length,
+      describedColumns: columns.filter(hasCatalogDescription).length,
+    }, latestCatalogTimestamp([
+      ...[...selectedDatabaseIds].map((id) => this.records.get(id)?.schemaSyncedAt),
+      ...tables.map((table) => table.updatedAt),
+      ...columns.map((column) => column.updatedAt),
+      ...relationships.map((relationship) => relationship.updatedAt),
+    ]));
+  }
   async create(input: DatabaseConfigurationInput): Promise<WorkspaceDatabase> {
     if ([...this.records.values()].some((record) => record.workspaceId === input.workspaceId)) {
       throw new CatalogConflictError("Workspace database already exists");
@@ -136,6 +176,11 @@ export class MemoryCatalogRepository implements CatalogRepository {
       this.descriptionGenerationRuns.delete(runId);
       this.descriptionGenerationEvents.delete(runId);
     }
+    for (const [runId, run] of this.sensitiveDataSuggestionRuns) {
+      if (run.databaseId !== id) continue;
+      this.sensitiveDataSuggestionRuns.delete(runId);
+      this.sensitiveDataSuggestionEvents.delete(runId);
+    }
     return this.records.delete(id);
   }
   async listTables(databaseId: string): Promise<CatalogTable[]> {
@@ -378,6 +423,110 @@ export class MemoryCatalogRepository implements CatalogRepository {
       .map((event) => structuredClone(event));
   }
 
+  async createSensitiveDataSuggestionRun(
+    databaseId: string,
+    scope: SensitiveDataSuggestionScope,
+    modelId: string,
+  ): Promise<SensitiveDataSuggestionRun> {
+    const now = new Date().toISOString();
+    const run: SensitiveDataSuggestionRun = {
+      id: randomUUID(),
+      databaseId,
+      scope,
+      modelId,
+      status: "running",
+      total: 0,
+      suggestedSensitive: 0,
+      suggestedNonSensitive: 0,
+      createdAt: now,
+      startedAt: now,
+      updatedAt: now,
+      finishedAt: null,
+      errorSummary: null,
+    };
+    this.sensitiveDataSuggestionRuns.set(run.id, run);
+    return structuredClone(run);
+  }
+
+  async getSensitiveDataSuggestionRun(
+    runId: string,
+  ): Promise<SensitiveDataSuggestionRun | undefined> {
+    const run = this.sensitiveDataSuggestionRuns.get(runId);
+    return run ? structuredClone(run) : undefined;
+  }
+
+  async listSensitiveDataSuggestionRuns(limit = 50): Promise<SensitiveDataSuggestionRun[]> {
+    return [...this.sensitiveDataSuggestionRuns.values()]
+      .sort((a, b) => b.createdAt.localeCompare(a.createdAt) || b.id.localeCompare(a.id))
+      .slice(0, limit)
+      .map((run) => structuredClone(run));
+  }
+
+  async interruptActiveSensitiveDataSuggestionRuns(
+    errorSummary: string,
+  ): Promise<SensitiveDataSuggestionRun[]> {
+    const interrupted: SensitiveDataSuggestionRun[] = [];
+    for (const run of this.sensitiveDataSuggestionRuns.values()) {
+      if (run.status !== "running") continue;
+      const now = new Date().toISOString();
+      const updated: SensitiveDataSuggestionRun = {
+        ...run,
+        status: "interrupted",
+        updatedAt: now,
+        finishedAt: now,
+        errorSummary,
+      };
+      this.sensitiveDataSuggestionRuns.set(run.id, updated);
+      interrupted.push(structuredClone(updated));
+    }
+    return interrupted;
+  }
+
+  async updateSensitiveDataSuggestionRun(
+    runId: string,
+    update: SensitiveDataSuggestionRunUpdate,
+  ): Promise<SensitiveDataSuggestionRun | undefined> {
+    const current = this.sensitiveDataSuggestionRuns.get(runId);
+    if (!current) return undefined;
+    const updated = {
+      ...current,
+      ...structuredClone(update),
+      updatedAt: new Date().toISOString(),
+    };
+    this.sensitiveDataSuggestionRuns.set(runId, updated);
+    return structuredClone(updated);
+  }
+
+  async appendSensitiveDataSuggestionEvent(
+    runId: string,
+    level: SensitiveDataSuggestionEvent["level"],
+    message: string,
+  ): Promise<SensitiveDataSuggestionEvent> {
+    if (!this.sensitiveDataSuggestionRuns.has(runId)) {
+      throw new CatalogConflictError("Sensitive Data Suggestion Run does not exist");
+    }
+    const events = this.sensitiveDataSuggestionEvents.get(runId) ?? [];
+    const event: SensitiveDataSuggestionEvent = {
+      runId,
+      sequence: events.length + 1,
+      level,
+      message,
+      createdAt: new Date().toISOString(),
+    };
+    events.push(event);
+    this.sensitiveDataSuggestionEvents.set(runId, events);
+    return structuredClone(event);
+  }
+
+  async listSensitiveDataSuggestionEvents(
+    runId: string,
+    afterSequence = 0,
+  ): Promise<SensitiveDataSuggestionEvent[]> {
+    return (this.sensitiveDataSuggestionEvents.get(runId) ?? [])
+      .filter((event) => event.sequence > afterSequence)
+      .map((event) => structuredClone(event));
+  }
+
   async listRelationships(databaseId: string): Promise<CatalogRelationship[]> {
     return [...this.relationships.values()].filter((relationship) => relationship.databaseId === databaseId)
       .sort((a, b) => `${a.sourceTableName}.${a.constraintName}`.localeCompare(`${b.sourceTableName}.${b.constraintName}`))

backend/src/catalog/metrics.ts

@@ -0,0 +1,53 @@
+import type { CatalogMetrics } from "./types.js";
+
+export interface CatalogMetricCounts {
+  tables: number;
+  columns: number;
+  sensitiveColumns: number;
+  relationships: number;
+  describedTables: number;
+  describedColumns: number;
+}
+
+export function hasCatalogDescription(target: {
+  description: string | null;
+  generatedDescription: string | null;
+}): boolean {
+  return Boolean(target.description?.trim() || target.generatedDescription?.trim());
+}
+
+export function latestCatalogTimestamp(
+  values: readonly (string | null | undefined)[],
+): string | null {
+  let latest: number | undefined;
+  for (const value of values) {
+    if (!value) continue;
+    const timestamp = Date.parse(value);
+    if (!Number.isFinite(timestamp)) continue;
+    latest = latest === undefined ? timestamp : Math.max(latest, timestamp);
+  }
+  return latest === undefined ? null : new Date(latest).toISOString();
+}
+
+export function createCatalogMetrics(
+  databaseId: string | undefined,
+  counts: CatalogMetricCounts,
+  updatedAt: string | null,
+): CatalogMetrics {
+  const descriptionTargets = counts.tables + counts.columns;
+  const describedTargets = counts.describedTables + counts.describedColumns;
+  return {
+    scope: databaseId === undefined ? "global" : "database",
+    databaseId: databaseId ?? null,
+    tables: counts.tables,
+    columns: counts.columns,
+    sensitiveColumns: counts.sensitiveColumns,
+    relationships: counts.relationships,
+    descriptionTargets,
+    describedTargets,
+    descriptionCoverage: descriptionTargets === 0
+      ? 0
+      : Math.round((describedTargets / descriptionTargets) * 100),
+    updatedAt,
+  };
+}

backend/src/catalog/migrate.ts

@@ -9,6 +9,7 @@ import * as catalogSchemaSyncMigration from "./migrations/003_catalog_schema_syn
 import * as catalogRuntimeSequencePrivilegesMigration from "./migrations/004_catalog_runtime_sequence_privileges.js";
 import * as descriptionGenerationRunsMigration from "./migrations/005_description_generation_runs.js";
 import * as sensitiveDataFlagMigration from "./migrations/006_sensitive_data_flag.js";
+import * as sensitiveDataSuggestionRunsMigration from "./migrations/007_sensitive_data_suggestion_runs.js";
 
 const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL;
 const host = process.env.THT_CATALOG_DB_HOST;
@@ -40,6 +41,7 @@ const provider: MigrationProvider = {
       "004_catalog_runtime_sequence_privileges": catalogRuntimeSequencePrivilegesMigration,
       "005_description_generation_runs": descriptionGenerationRunsMigration,
       "006_sensitive_data_flag": sensitiveDataFlagMigration,
+      "007_sensitive_data_suggestion_runs": sensitiveDataSuggestionRunsMigration,
     };
   },
 };

backend/src/catalog/migrations/007_sensitive_data_suggestion_runs.ts

@@ -0,0 +1,73 @@
+import { type Kysely, sql } from "kysely";
+import type { CatalogDatabase } from "../repository.js";
+
+export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
+  await db.schema.createTable("sensitive_data_suggestion_runs")
+    .addColumn("id", "uuid", (column) => column.primaryKey())
+    .addColumn("database_id", "uuid", (column) => column.notNull()
+      .references("workspace_databases.id").onDelete("cascade"))
+    .addColumn("scope", "text", (column) => column.notNull())
+    .addColumn("model_id", "text", (column) => column.notNull())
+    .addColumn("status", "text", (column) => column.notNull())
+    .addColumn("total", "integer", (column) => column.notNull().defaultTo(0))
+    .addColumn("suggested_sensitive", "integer", (column) => column.notNull().defaultTo(0))
+    .addColumn("suggested_non_sensitive", "integer", (column) => column.notNull().defaultTo(0))
+    .addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
+    .addColumn("started_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
+    .addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
+    .addColumn("finished_at", "timestamptz")
+    .addColumn("error_summary", "text")
+    .addCheckConstraint(
+      "sensitive_data_suggestion_runs_scope_check",
+      sql`scope in ('selected_columns', 'selected_tables', 'all')`,
+    )
+    .addCheckConstraint(
+      "sensitive_data_suggestion_runs_model_id_check",
+      sql`model_id ~ '^[a-z][a-z0-9._-]{0,63}$'`,
+    )
+    .addCheckConstraint(
+      "sensitive_data_suggestion_runs_status_check",
+      sql`status in ('running', 'completed', 'failed', 'interrupted')`,
+    )
+    .addCheckConstraint(
+      "sensitive_data_suggestion_runs_counters_check",
+      sql`total >= 0
+        and suggested_sensitive >= 0
+        and suggested_non_sensitive >= 0
+        and suggested_sensitive + suggested_non_sensitive <= total`,
+    )
+    .addCheckConstraint(
+      "sensitive_data_suggestion_runs_error_summary_check",
+      sql`error_summary is null or char_length(error_summary) between 1 and 2000`,
+    )
+    .execute();
+
+  await db.schema.createIndex("sensitive_data_suggestion_runs_database_created_idx")
+    .on("sensitive_data_suggestion_runs")
+    .columns(["database_id", "created_at"])
+    .execute();
+
+  await db.schema.createTable("sensitive_data_suggestion_events")
+    .addColumn("run_id", "uuid", (column) => column.notNull()
+      .references("sensitive_data_suggestion_runs.id").onDelete("cascade"))
+    .addColumn("sequence", "integer", (column) => column.notNull())
+    .addColumn("level", "text", (column) => column.notNull())
+    .addColumn("message", "text", (column) => column.notNull())
+    .addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
+    .addPrimaryKeyConstraint("sensitive_data_suggestion_events_pkey", ["run_id", "sequence"])
+    .addCheckConstraint("sensitive_data_suggestion_events_sequence_check", sql`sequence > 0`)
+    .addCheckConstraint(
+      "sensitive_data_suggestion_events_level_check",
+      sql`level in ('info', 'warning', 'error')`,
+    )
+    .addCheckConstraint(
+      "sensitive_data_suggestion_events_message_check",
+      sql`char_length(message) between 1 and 2000`,
+    )
+    .execute();
+}
+
+export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
+  await db.schema.dropTable("sensitive_data_suggestion_events").execute();
+  await db.schema.dropTable("sensitive_data_suggestion_runs").execute();
+}

backend/src/catalog/repository.ts

@@ -11,6 +11,7 @@ import {
   type Transaction,
 } from "kysely";
 import { Pool } from "pg";
+import { createCatalogMetrics } from "./metrics.js";
 import {
   CatalogConflictError,
   CatalogConnectorError,
@@ -21,6 +22,7 @@ import {
   type CatalogDescriptionTarget,
   type CatalogDatabaseMetadataDeleteTarget,
   type CatalogMetadataDeleteCounts,
+  type CatalogMetrics,
   type CatalogRelationship,
   type CatalogSchemaDiff,
   type CatalogSyncCounts,
@@ -40,6 +42,10 @@ import {
   type DescriptionGenerationScope,
   type ObservedCatalogTable,
   type ObservedSchemaSnapshot,
+  type SensitiveDataSuggestionEvent,
+  type SensitiveDataSuggestionRun,
+  type SensitiveDataSuggestionRunUpdate,
+  type SensitiveDataSuggestionScope,
   type TableSyncRepositoryResult,
   type WorkspaceDatabase,
 } from "./types.js";
@@ -166,6 +172,30 @@ interface DescriptionGenerationEventTable {
   createdAt: Timestamp;
 }
 
+interface SensitiveDataSuggestionRunTable {
+  id: string;
+  databaseId: string;
+  scope: SensitiveDataSuggestionScope;
+  modelId: string;
+  status: SensitiveDataSuggestionRun["status"];
+  total: number;
+  suggestedSensitive: number;
+  suggestedNonSensitive: number;
+  createdAt: Timestamp;
+  startedAt: Timestamp;
+  updatedAt: Timestamp;
+  finishedAt: Timestamp | null;
+  errorSummary: string | null;
+}
+
+interface SensitiveDataSuggestionEventTable {
+  runId: string;
+  sequence: number;
+  level: SensitiveDataSuggestionEvent["level"];
+  message: string;
+  createdAt: Timestamp;
+}
+
 interface CatalogSyncRunTable {
   id: string;
   databaseId: string;
@@ -213,6 +243,8 @@ export interface CatalogDatabase {
   catalogRelationshipColumns: CatalogRelationshipColumnTable;
   descriptionGenerationRuns: DescriptionGenerationRunTable;
   descriptionGenerationEvents: DescriptionGenerationEventTable;
+  sensitiveDataSuggestionRuns: SensitiveDataSuggestionRunTable;
+  sensitiveDataSuggestionEvents: SensitiveDataSuggestionEventTable;
   catalogSyncRuns: CatalogSyncRunTable;
   catalogSyncEvents: CatalogSyncEventTable;
 }
@@ -220,6 +252,17 @@ export interface CatalogDatabase {
 type DbOrTransaction = Kysely<CatalogDatabase> | Transaction<CatalogDatabase>;
 type JoinedRow = Selectable<WorkspaceDatabaseTable> & Selectable<DatabaseBindingTable>;
 
+interface CatalogMetricsRow {
+  databaseCount: number;
+  tables: number;
+  columns: number;
+  sensitiveColumns: number;
+  relationships: number;
+  describedTables: number;
+  describedColumns: number;
+  updatedAt: Date | string | null;
+}
+
 function present<T>(value: T | null): T | undefined {
   return value === null ? undefined : value;
 }
@@ -338,6 +381,25 @@ function serializeDescriptionGenerationEvent(
   return { ...row, createdAt: new Date(row.createdAt).toISOString() };
 }
 
+function serializeSensitiveDataSuggestionRun(
+  row: Selectable<SensitiveDataSuggestionRunTable>,
+): SensitiveDataSuggestionRun {
+  const stamp = (value: Date | string | null) => value === null ? null : new Date(value).toISOString();
+  return {
+    ...row,
+    createdAt: new Date(row.createdAt).toISOString(),
+    startedAt: new Date(row.startedAt).toISOString(),
+    updatedAt: new Date(row.updatedAt).toISOString(),
+    finishedAt: stamp(row.finishedAt),
+  };
+}
+
+function serializeSensitiveDataSuggestionEvent(
+  row: Selectable<SensitiveDataSuggestionEventTable>,
+): SensitiveDataSuggestionEvent {
+  return { ...row, createdAt: new Date(row.createdAt).toISOString() };
+}
+
 function bindingValues(databaseId: string, binding: DatabaseBinding) {
   return {
     databaseId,
@@ -388,6 +450,83 @@ export class KyselyCatalogRepository implements CatalogRepository {
     return id ? await this.get(id.id) : undefined;
   }
 
+  async getCatalogMetrics(databaseId?: string): Promise<CatalogMetrics | undefined> {
+    const result = await sql<CatalogMetricsRow>`
+      WITH requested_database AS (
+        SELECT ${databaseId ?? null}::uuid AS id
+      ),
+      selected_databases AS (
+        SELECT workspace_databases.id, workspace_databases.schema_synced_at
+        FROM workspace_databases
+        CROSS JOIN requested_database
+        WHERE requested_database.id IS NULL
+          OR workspace_databases.id = requested_database.id
+      ),
+      table_metrics AS (
+        SELECT
+          count(*)::int AS tables,
+          count(*) FILTER (
+            WHERE nullif(btrim(catalog_tables.description), '') IS NOT NULL
+              OR nullif(btrim(catalog_tables.generated_description), '') IS NOT NULL
+          )::int AS "describedTables",
+          max(catalog_tables.updated_at) AS updated_at
+        FROM catalog_tables
+        INNER JOIN selected_databases
+          ON selected_databases.id = catalog_tables.database_id
+      ),
+      column_metrics AS (
+        SELECT
+          count(*)::int AS columns,
+          count(*) FILTER (WHERE catalog_columns.sensitive)::int AS "sensitiveColumns",
+          count(*) FILTER (
+            WHERE nullif(btrim(catalog_columns.description), '') IS NOT NULL
+              OR nullif(btrim(catalog_columns.generated_description), '') IS NOT NULL
+          )::int AS "describedColumns",
+          max(catalog_columns.updated_at) AS updated_at
+        FROM catalog_columns
+        INNER JOIN catalog_tables ON catalog_tables.id = catalog_columns.table_id
+        INNER JOIN selected_databases
+          ON selected_databases.id = catalog_tables.database_id
+      ),
+      relationship_metrics AS (
+        SELECT
+          count(*)::int AS relationships,
+          max(catalog_relationships.updated_at) AS updated_at
+        FROM catalog_relationships
+        INNER JOIN selected_databases
+          ON selected_databases.id = catalog_relationships.database_id
+      )
+      SELECT
+        (SELECT count(*)::int FROM selected_databases) AS "databaseCount",
+        table_metrics.tables,
+        column_metrics.columns,
+        column_metrics."sensitiveColumns",
+        relationship_metrics.relationships,
+        table_metrics."describedTables",
+        column_metrics."describedColumns",
+        greatest(
+          (SELECT max(schema_synced_at) FROM selected_databases),
+          table_metrics.updated_at,
+          column_metrics.updated_at,
+          relationship_metrics.updated_at
+        ) AS "updatedAt"
+      FROM table_metrics
+      CROSS JOIN column_metrics
+      CROSS JOIN relationship_metrics
+    `.execute(this.db);
+    const row = result.rows[0];
+    if (!row || (databaseId !== undefined && Number(row.databaseCount) === 0)) return undefined;
+
+    return createCatalogMetrics(databaseId, {
+      tables: Number(row.tables),
+      columns: Number(row.columns),
+      sensitiveColumns: Number(row.sensitiveColumns),
+      relationships: Number(row.relationships),
+      describedTables: Number(row.describedTables),
+      describedColumns: Number(row.describedColumns),
+    }, row.updatedAt === null ? null : new Date(row.updatedAt).toISOString());
+  }
+
   async create(input: DatabaseConfigurationInput): Promise<WorkspaceDatabase> {
     try {
       return await this.db.transaction().execute(async (trx) => {
@@ -775,6 +914,114 @@ export class KyselyCatalogRepository implements CatalogRepository {
     return rows.map(serializeDescriptionGenerationEvent);
   }
 
+  async createSensitiveDataSuggestionRun(
+    databaseId: string,
+    scope: SensitiveDataSuggestionScope,
+    modelId: string,
+  ): Promise<SensitiveDataSuggestionRun> {
+    const row = await this.db.insertInto("sensitiveDataSuggestionRuns").values({
+      id: randomUUID(),
+      databaseId,
+      scope,
+      modelId,
+      status: "running",
+      total: 0,
+      suggestedSensitive: 0,
+      suggestedNonSensitive: 0,
+      finishedAt: null,
+      errorSummary: null,
+    }).returningAll().executeTakeFirstOrThrow();
+    return serializeSensitiveDataSuggestionRun(row);
+  }
+
+  async getSensitiveDataSuggestionRun(
+    runId: string,
+  ): Promise<SensitiveDataSuggestionRun | undefined> {
+    const row = await this.db.selectFrom("sensitiveDataSuggestionRuns")
+      .selectAll()
+      .where("id", "=", runId)
+      .executeTakeFirst();
+    return row ? serializeSensitiveDataSuggestionRun(row) : undefined;
+  }
+
+  async listSensitiveDataSuggestionRuns(limit = 50): Promise<SensitiveDataSuggestionRun[]> {
+    const rows = await this.db.selectFrom("sensitiveDataSuggestionRuns")
+      .selectAll()
+      .orderBy("createdAt", "desc")
+      .orderBy("id", "desc")
+      .limit(limit)
+      .execute();
+    return rows.map(serializeSensitiveDataSuggestionRun);
+  }
+
+  async interruptActiveSensitiveDataSuggestionRuns(
+    errorSummary: string,
+  ): Promise<SensitiveDataSuggestionRun[]> {
+    const rows = await this.db.updateTable("sensitiveDataSuggestionRuns")
+      .set({
+        status: "interrupted",
+        finishedAt: sql`now()`,
+        updatedAt: sql`now()`,
+        errorSummary,
+      })
+      .where("status", "=", "running")
+      .returningAll()
+      .execute();
+    return rows.map(serializeSensitiveDataSuggestionRun);
+  }
+
+  async updateSensitiveDataSuggestionRun(
+    runId: string,
+    update: SensitiveDataSuggestionRunUpdate,
+  ): Promise<SensitiveDataSuggestionRun | undefined> {
+    const values: any = { ...update, updatedAt: sql`now()` };
+    const row = await this.db.updateTable("sensitiveDataSuggestionRuns")
+      .set(values)
+      .where("id", "=", runId)
+      .returningAll()
+      .executeTakeFirst();
+    return row ? serializeSensitiveDataSuggestionRun(row) : undefined;
+  }
+
+  async appendSensitiveDataSuggestionEvent(
+    runId: string,
+    level: SensitiveDataSuggestionEvent["level"],
+    message: string,
+  ): Promise<SensitiveDataSuggestionEvent> {
+    return await this.db.transaction().execute(async (trx) => {
+      const run = await trx.selectFrom("sensitiveDataSuggestionRuns")
+        .select("id")
+        .where("id", "=", runId)
+        .forUpdate()
+        .executeTakeFirst();
+      if (!run) throw new CatalogConflictError("Sensitive Data Suggestion Run does not exist");
+      const current = await trx.selectFrom("sensitiveDataSuggestionEvents")
+        .select(sql<number>`coalesce(max(sequence), 0)::int`.as("sequence"))
+        .where("runId", "=", runId)
+        .executeTakeFirst();
+      const row = await trx.insertInto("sensitiveDataSuggestionEvents").values({
+        runId,
+        sequence: Number(current?.sequence ?? 0) + 1,
+        level,
+        message,
+      }).returningAll().executeTakeFirstOrThrow();
+      return serializeSensitiveDataSuggestionEvent(row);
+    });
+  }
+
+  async listSensitiveDataSuggestionEvents(
+    runId: string,
+    afterSequence = 0,
+  ): Promise<SensitiveDataSuggestionEvent[]> {
+    const rows = await this.db.selectFrom("sensitiveDataSuggestionEvents")
+      .selectAll()
+      .where("runId", "=", runId)
+      .where("sequence", ">", afterSequence)
+      .orderBy("sequence")
+      .execute();
+    return rows.map(serializeSensitiveDataSuggestionEvent);
+  }
+
   async listRelationships(databaseId: string): Promise<CatalogRelationship[]> {
     const rows = await this.db.selectFrom("catalogRelationships as relationship")
       .innerJoin("catalogTables as sourceTable", "sourceTable.id", "relationship.sourceTableId")
@@ -1345,6 +1592,7 @@ export class UnavailableCatalogRepository implements CatalogRepository {
   async list(): Promise<WorkspaceDatabase[]> { return this.fail(); }
   async get(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
   async getByWorkspace(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
+  async getCatalogMetrics(): Promise<CatalogMetrics | undefined> { return this.fail(); }
   async create(): Promise<WorkspaceDatabase> { return this.fail(); }
   async update(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
   async recordTest(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
@@ -1366,6 +1614,13 @@ export class UnavailableCatalogRepository implements CatalogRepository {
   async updateDescriptionGenerationRun(): Promise<DescriptionGenerationRun | undefined> { return this.fail(); }
   async appendDescriptionGenerationEvent(): Promise<DescriptionGenerationEvent> { return this.fail(); }
   async listDescriptionGenerationEvents(): Promise<DescriptionGenerationEvent[]> { return this.fail(); }
+  async createSensitiveDataSuggestionRun(): Promise<SensitiveDataSuggestionRun> { return this.fail(); }
+  async getSensitiveDataSuggestionRun(): Promise<SensitiveDataSuggestionRun | undefined> { return this.fail(); }
+  async listSensitiveDataSuggestionRuns(): Promise<SensitiveDataSuggestionRun[]> { return this.fail(); }
+  async interruptActiveSensitiveDataSuggestionRuns(): Promise<SensitiveDataSuggestionRun[]> { return this.fail(); }
+  async updateSensitiveDataSuggestionRun(): Promise<SensitiveDataSuggestionRun | undefined> { return this.fail(); }
+  async appendSensitiveDataSuggestionEvent(): Promise<SensitiveDataSuggestionEvent> { return this.fail(); }
+  async listSensitiveDataSuggestionEvents(): Promise<SensitiveDataSuggestionEvent[]> { return this.fail(); }
   async listRelationships(): Promise<CatalogRelationship[]> { return this.fail(); }
   async deleteDatabaseMetadata(): Promise<CatalogMetadataDeleteCounts | undefined> { return this.fail(); }
   async deleteTableMetadata(): Promise<CatalogMetadataDeleteCounts | undefined> { return this.fail(); }

backend/src/catalog/sensitive-data-suggester.ts

@@ -1,7 +1,14 @@
 import { z } from "zod";
 import type { MetadataGenerationModels } from "./metadata-generation-models.js";
 import type { ModelCompleter, ModelCompletionMessage } from "./model-completer.js";
-import type { CatalogColumn, CatalogRepository, CatalogTable } from "./types.js";
+import type {
+  CatalogColumn,
+  CatalogRepository,
+  CatalogTable,
+  SensitiveDataSuggestionScope,
+} from "./types.js";
+
+export type { SensitiveDataSuggestionScope } from "./types.js";
 
 // The helper accepts at most 64 KiB per message. Keep the same safety margin used by
 // Description Generation so UTF-8 structural metadata never reaches that hard limit.
@@ -16,8 +23,6 @@ const responseSchema = z.object({
   }).strict()),
 }).strict();
 
-export type SensitiveDataSuggestionScope = "all" | "selected_tables" | "selected_columns";
-
 interface StructuralColumn {
   columnId: string;
   tableId: string;
@@ -195,10 +200,12 @@ export class SensitiveDataSuggester {
     scope: SensitiveDataSuggestionScope,
     targetIds: readonly string[],
     signal: AbortSignal,
+    onPrepared?: (total: number) => void | Promise<void>,
   ): Promise<readonly SensitiveDataSuggestion[]> {
     const database = await this.repository.get(databaseId);
     if (!database) throw new SensitiveDataSuggestionTargetNotFoundError("database");
     const columns = await this.selectColumns(databaseId, scope, targetIds);
+    await onPrepared?.(columns.length);
     const model = this.models.resolve(modelId);
     const suggestions: SensitiveDataSuggestion[] = [];
 

backend/src/catalog/sensitive-data-suggestion-runner.ts

@@ -0,0 +1,110 @@
+import type {
+  SensitiveDataSuggestion,
+} from "./sensitive-data-suggester.js";
+import {
+  SensitiveDataSuggester,
+  SensitiveDataSuggestionTargetNotFoundError,
+} from "./sensitive-data-suggester.js";
+import type {
+  CatalogRepository,
+  SensitiveDataSuggestionRun,
+  SensitiveDataSuggestionScope,
+} from "./types.js";
+
+const interruptedMessage = "Sensitive-field suggestion generation was interrupted by backend restart.";
+const failedMessage = "Sensitive-field suggestion generation failed.";
+
+export interface SensitiveDataSuggestionRunResult {
+  suggestions: readonly SensitiveDataSuggestion[];
+  run: SensitiveDataSuggestionRun;
+}
+
+export class SensitiveDataSuggestionRunner {
+  constructor(
+    private readonly repository: CatalogRepository,
+    private readonly suggester: SensitiveDataSuggester,
+  ) {}
+
+  async initialize(): Promise<void> {
+    if (!(await this.repository.available())) return;
+    const interrupted = await this.repository.interruptActiveSensitiveDataSuggestionRuns(
+      interruptedMessage,
+    );
+    for (const run of interrupted) {
+      await this.repository.appendSensitiveDataSuggestionEvent(
+        run.id,
+        "warning",
+        interruptedMessage,
+      );
+    }
+  }
+
+  async run(
+    databaseId: string,
+    modelId: string,
+    scope: SensitiveDataSuggestionScope,
+    targetIds: readonly string[],
+    signal: AbortSignal,
+  ): Promise<SensitiveDataSuggestionRunResult> {
+    if (!(await this.repository.get(databaseId))) {
+      throw new SensitiveDataSuggestionTargetNotFoundError("database");
+    }
+    const started = await this.repository.createSensitiveDataSuggestionRun(
+      databaseId,
+      scope,
+      modelId,
+    );
+
+    try {
+      await this.repository.appendSensitiveDataSuggestionEvent(
+        started.id,
+        "info",
+        "Sensitive-field suggestion generation started.",
+      );
+      const suggestions = await this.suggester.suggest(
+        databaseId,
+        modelId,
+        scope,
+        targetIds,
+        signal,
+        async (total) => {
+          const prepared = await this.repository.updateSensitiveDataSuggestionRun(started.id, {
+            total,
+          });
+          if (!prepared) throw new Error("Sensitive Data Suggestion Run disappeared");
+        },
+      );
+      const suggestedSensitive = suggestions.filter((suggestion) => suggestion.sensitive).length;
+      const suggestedNonSensitive = suggestions.length - suggestedSensitive;
+      await this.repository.appendSensitiveDataSuggestionEvent(
+        started.id,
+        "info",
+        `Sensitive-field suggestion generation completed for ${suggestions.length} column${
+          suggestions.length === 1 ? "" : "s"
+        }.`,
+      );
+      const completed = await this.repository.updateSensitiveDataSuggestionRun(started.id, {
+        status: "completed",
+        total: suggestions.length,
+        suggestedSensitive,
+        suggestedNonSensitive,
+        finishedAt: new Date().toISOString(),
+        errorSummary: null,
+      });
+      if (!completed) throw new Error("Sensitive Data Suggestion Run disappeared");
+      return { suggestions, run: completed };
+    } catch (error) {
+      await this.repository.updateSensitiveDataSuggestionRun(started.id, {
+        status: "failed",
+        finishedAt: new Date().toISOString(),
+        errorSummary: failedMessage,
+      }).catch(() => undefined);
+      await this.repository.appendSensitiveDataSuggestionEvent(
+        started.id,
+        "error",
+        failedMessage,
+      ).catch(() => undefined);
+      throw error;
+    }
+  }
+}

backend/src/catalog/types.ts

@@ -38,6 +38,19 @@ export interface WorkspaceDatabase {
   schemaSyncedAt?: string;
 }
 
+export interface CatalogMetrics {
+  scope: "global" | "database";
+  databaseId: string | null;
+  tables: number;
+  columns: number;
+  sensitiveColumns: number;
+  relationships: number;
+  descriptionTargets: number;
+  describedTargets: number;
+  descriptionCoverage: number;
+  updatedAt: string | null;
+}
+
 export interface DatabaseConfigurationInput {
   workspaceId: string;
   engine: "postgres";
@@ -201,6 +214,42 @@ export interface DescriptionGenerationEvent {
   createdAt: string;
 }
 
+export type SensitiveDataSuggestionScope = "all" | "selected_tables" | "selected_columns";
+export type SensitiveDataSuggestionStatus = "running" | "completed" | "failed" | "interrupted";
+
+export interface SensitiveDataSuggestionRun {
+  id: string;
+  databaseId: string;
+  scope: SensitiveDataSuggestionScope;
+  modelId: string;
+  status: SensitiveDataSuggestionStatus;
+  total: number;
+  suggestedSensitive: number;
+  suggestedNonSensitive: number;
+  createdAt: string;
+  startedAt: string;
+  updatedAt: string;
+  finishedAt: string | null;
+  errorSummary: string | null;
+}
+
+export interface SensitiveDataSuggestionRunUpdate {
+  status?: SensitiveDataSuggestionStatus;
+  total?: number;
+  suggestedSensitive?: number;
+  suggestedNonSensitive?: number;
+  finishedAt?: string | null;
+  errorSummary?: string | null;
+}
+
+export interface SensitiveDataSuggestionEvent {
+  runId: string;
+  sequence: number;
+  level: "info" | "warning" | "error";
+  message: string;
+  createdAt: string;
+}
+
 export interface ObservedRelationshipColumn {
   position: number;
   sourceColumnName: string;
@@ -321,6 +370,7 @@ export interface CatalogRepository {
   list(): Promise<WorkspaceDatabase[]>;
   get(id: string): Promise<WorkspaceDatabase | undefined>;
   getByWorkspace(workspaceId: string): Promise<WorkspaceDatabase | undefined>;
+  getCatalogMetrics(databaseId?: string): Promise<CatalogMetrics | undefined>;
   create(input: DatabaseConfigurationInput): Promise<WorkspaceDatabase>;
   update(id: string, expectedVersion: number, input: DatabaseConfigurationInput): Promise<WorkspaceDatabase | undefined>;
   recordTest(id: string, expectedVersion: number, result: DatabaseTestResult): Promise<WorkspaceDatabase | undefined>;
@@ -383,6 +433,29 @@ export interface CatalogRepository {
     runId: string,
     afterSequence?: number,
   ): Promise<DescriptionGenerationEvent[]>;
+  createSensitiveDataSuggestionRun(
+    databaseId: string,
+    scope: SensitiveDataSuggestionScope,
+    modelId: string,
+  ): Promise<SensitiveDataSuggestionRun>;
+  getSensitiveDataSuggestionRun(runId: string): Promise<SensitiveDataSuggestionRun | undefined>;
+  listSensitiveDataSuggestionRuns(limit?: number): Promise<SensitiveDataSuggestionRun[]>;
+  interruptActiveSensitiveDataSuggestionRuns(
+    errorSummary: string,
+  ): Promise<SensitiveDataSuggestionRun[]>;
+  updateSensitiveDataSuggestionRun(
+    runId: string,
+    update: SensitiveDataSuggestionRunUpdate,
+  ): Promise<SensitiveDataSuggestionRun | undefined>;
+  appendSensitiveDataSuggestionEvent(
+    runId: string,
+    level: SensitiveDataSuggestionEvent["level"],
+    message: string,
+  ): Promise<SensitiveDataSuggestionEvent>;
+  listSensitiveDataSuggestionEvents(
+    runId: string,
+    afterSequence?: number,
+  ): Promise<SensitiveDataSuggestionEvent[]>;
   listRelationships(databaseId: string): Promise<CatalogRelationship[]>;
   deleteDatabaseMetadata(
     databaseIds: readonly string[],