feat: complete catalog-driven preprocessing

backend/src/catalog/repository.ts

@@ -27,6 +27,8 @@ import {
   type CatalogLogicalRelationshipCandidate,
   type CatalogLogicalRelationshipContext,
   type CatalogPhysicalRelationship,
+  type CatalogPreprocessingStartResult,
+  type CatalogPreprocessingClearResult,
   type CatalogSchemaDiff,
   type CatalogSyncCounts,
   type CatalogSyncEvent,
@@ -54,6 +56,11 @@ import {
 } from "./types.js";
 
 type Timestamp = ColumnType<Date, Date | string | undefined, Date | string>;
+type NullableTimestamp = ColumnType<
+  Date | null,
+  Date | string | null | undefined,
+  Date | string | null
+>;
 
 interface WorkspaceDatabaseTable {
   id: string;
@@ -66,6 +73,13 @@ interface WorkspaceDatabaseTable {
   updatedAt: Timestamp;
   schemaSyncedVersion: number | null;
   schemaSyncedAt: Timestamp | null;
+  metadataContentRevision: Generated<number>;
+  preprocessingStatus: Generated<WorkspaceDatabase["preprocessingStatus"]>;
+  preprocessingInputFingerprint: Generated<string | null>;
+  preprocessedMetadataRevision: Generated<number | null>;
+  preprocessingStartedAt: NullableTimestamp;
+  preprocessingFinishedAt: NullableTimestamp;
+  preprocessingErrorCode: Generated<string | null>;
 }
 
 interface DatabaseBindingTable {
@@ -325,6 +339,19 @@ function serialize(row: JoinedRow): WorkspaceDatabase {
     lastErrorMessage: present(row.lastErrorMessage),
     schemaSyncedVersion: present(row.schemaSyncedVersion),
     schemaSyncedAt: row.schemaSyncedAt == null ? undefined : new Date(row.schemaSyncedAt).toISOString(),
+    metadataContentRevision: Number(row.metadataContentRevision),
+    preprocessingStatus: row.preprocessingStatus,
+    preprocessingInputFingerprint: present(row.preprocessingInputFingerprint),
+    preprocessedMetadataRevision: row.preprocessedMetadataRevision == null
+      ? undefined
+      : Number(row.preprocessedMetadataRevision),
+    preprocessingStartedAt: row.preprocessingStartedAt == null
+      ? undefined
+      : new Date(row.preprocessingStartedAt).toISOString(),
+    preprocessingFinishedAt: row.preprocessingFinishedAt == null
+      ? undefined
+      : new Date(row.preprocessingFinishedAt).toISOString(),
+    preprocessingErrorCode: present(row.preprocessingErrorCode),
   };
 }
 
@@ -477,6 +504,98 @@ export class KyselyCatalogRepository implements CatalogRepository {
     return id ? await this.get(id.id) : undefined;
   }
 
+  async beginPreprocessing(
+    workspaceId: string,
+    inputFingerprint: string,
+  ): Promise<CatalogPreprocessingStartResult> {
+    return await this.db.transaction().execute(async (trx) => {
+      const database = await trx.selectFrom("workspaceDatabases")
+        .selectAll()
+        .where("workspaceId", "=", workspaceId)
+        .forUpdate()
+        .executeTakeFirst();
+      if (!database) return { kind: "not_found" };
+      if (database.preprocessingStatus === "running") return { kind: "already_running" };
+      if (database.schemaSyncedVersion !== database.version) return { kind: "schema_stale" };
+
+      const activeSync = await trx.selectFrom("catalogSyncRuns")
+        .select("id")
+        .where("databaseId", "=", database.id)
+        .where("state", "in", ["queued", "running", "awaiting_confirmation", "applying"])
+        .executeTakeFirst();
+      const activeDescriptions = await trx.selectFrom("descriptionGenerationRuns")
+        .select("id")
+        .where("databaseId", "=", database.id)
+        .where("status", "in", ["queued", "running"])
+        .executeTakeFirst();
+      const activeSensitivity = await trx.selectFrom("sensitiveDataSuggestionRuns")
+        .select("id")
+        .where("databaseId", "=", database.id)
+        .where("status", "=", "running")
+        .executeTakeFirst();
+      if (activeSync || activeDescriptions || activeSensitivity) return { kind: "catalog_busy" };
+
+      await trx.updateTable("workspaceDatabases")
+        .set({
+          preprocessingStatus: "running",
+          preprocessingInputFingerprint: inputFingerprint,
+          preprocessedMetadataRevision: null,
+          preprocessingStartedAt: sql`now()`,
+          preprocessingFinishedAt: null,
+          preprocessingErrorCode: null,
+          updatedAt: sql`now()`,
+        })
+        .where("id", "=", database.id)
+        .execute();
+      return { kind: "started", database: (await selectOne(trx, database.id))! };
+    });
+  }
+
+  async finishPreprocessing(
+    workspaceId: string,
+    metadataContentRevision: number,
+    inputFingerprint: string,
+    outcome: { status: "succeeded" } | { status: "failed"; errorCode: string },
+  ): Promise<WorkspaceDatabase | undefined> {
+    const result = await this.db.updateTable("workspaceDatabases")
+      .set({
+        preprocessingStatus: outcome.status,
+        preprocessedMetadataRevision: outcome.status === "succeeded" ? metadataContentRevision : null,
+        preprocessingFinishedAt: sql`now()`,
+        preprocessingErrorCode: outcome.status === "failed" ? outcome.errorCode : null,
+        updatedAt: sql`now()`,
+      })
+      .where("workspaceId", "=", workspaceId)
+      .where("preprocessingStatus", "=", "running")
+      .where("metadataContentRevision", "=", metadataContentRevision)
+      .where("preprocessingInputFingerprint", "=", inputFingerprint)
+      .returning("id")
+      .executeTakeFirst();
+    return result ? await this.get(result.id) : undefined;
+  }
+
+  async clearPreprocessing(workspaceId: string): Promise<CatalogPreprocessingClearResult> {
+    return await this.db.transaction().execute(async (trx) => {
+      const database = await trx.selectFrom("workspaceDatabases")
+        .select(["id", "preprocessingStatus"])
+        .where("workspaceId", "=", workspaceId)
+        .forUpdate()
+        .executeTakeFirst();
+      if (!database) return { kind: "not_found" };
+      if (database.preprocessingStatus === "running") return { kind: "already_running" };
+      await trx.updateTable("workspaceDatabases").set({
+        preprocessingStatus: "failed",
+        preprocessingInputFingerprint: null,
+        preprocessedMetadataRevision: null,
+        preprocessingStartedAt: null,
+        preprocessingFinishedAt: sql`now()`,
+        preprocessingErrorCode: "derived_data_cleared",
+        updatedAt: sql`now()`,
+      }).where("id", "=", database.id).execute();
+      return { kind: "cleared", database: (await selectOne(trx, database.id))! };
+    });
+  }
+
   async getCatalogMetrics(databaseId?: string): Promise<CatalogMetrics | undefined> {
     const result = await sql<CatalogMetricsRow>`
       WITH requested_database AS (
@@ -763,7 +882,9 @@ export class KyselyCatalogRepository implements CatalogRepository {
     targetIds: readonly string[],
   ): Promise<CatalogDescriptionConsolidationCounts | undefined> {
     const selectedTargetIds = [...new Set(targetIds)];
-    if (target !== "database_columns" && selectedTargetIds.length === 0) return undefined;
+    if (target !== "database" && target !== "database_columns" && selectedTargetIds.length === 0) {
+      return undefined;
+    }
     return await this.db.transaction().execute(async (trx) => {
       const database = await trx.selectFrom("workspaceDatabases").select("id")
         .where("id", "=", databaseId).forUpdate().executeTakeFirst();
@@ -793,8 +914,23 @@ export class KyselyCatalogRepository implements CatalogRepository {
         };
       }
 
-      const tableRows = await trx.selectFrom("catalogTables").select("id")
-        .where("databaseId", "=", databaseId).execute();
+      const tableRows = await trx.selectFrom("catalogTables").select(["id", "generatedDescription"])
+        .where("databaseId", "=", databaseId)
+        .orderBy("id")
+        .forUpdate()
+        .execute();
+      const copiedTableIds = target === "database"
+        ? tableRows
+          .filter((row) => Boolean(row.generatedDescription?.trim()))
+          .map((row) => row.id)
+        : [];
+      if (copiedTableIds.length > 0) {
+        await trx.updateTable("catalogTables").set({
+          description: sql`generated_description`,
+          version: sql`version + 1`,
+          updatedAt: sql`now()`,
+        }).where("id", "in", copiedTableIds).execute();
+      }
       const rows = tableRows.length === 0 ? [] : await trx.selectFrom("catalogColumns")
         .select(["id", "generatedDescription"])
         .where("tableId", "in", tableRows.map((table) => table.id))
@@ -812,7 +948,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
           version: sql`version + 1`,
           updatedAt: sql`now()`,
         });
-        update = target === "database_columns"
+        update = target === "database" || target === "database_columns"
           ? update
             .where("tableId", "in", tableRows.map((table) => table.id))
             .where(sql<boolean>`nullif(btrim(generated_description), '') is not null`)
@@ -820,8 +956,9 @@ export class KyselyCatalogRepository implements CatalogRepository {
         await update.execute();
       }
       return {
-        copied: copiedIds.length,
-        skipped: rows.length - copiedIds.length,
+        copied: copiedTableIds.length + copiedIds.length,
+        skipped: (target === "database" ? tableRows.length : 0) - copiedTableIds.length
+          + rows.length - copiedIds.length,
       };
     });
   }
@@ -1835,6 +1972,9 @@ export class UnavailableCatalogRepository implements CatalogRepository {
   async list(): Promise<WorkspaceDatabase[]> { return this.fail(); }
   async get(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
   async getByWorkspace(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
+  async beginPreprocessing(): Promise<CatalogPreprocessingStartResult> { return this.fail(); }
+  async finishPreprocessing(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
+  async clearPreprocessing(): Promise<CatalogPreprocessingClearResult> { return this.fail(); }
   async getCatalogMetrics(): Promise<CatalogMetrics | undefined> { return this.fail(); }
   async create(): Promise<WorkspaceDatabase> { return this.fail(); }
   async update(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }