feat: add AI catalog description generation

2026-08-29 16:42:56 +02:00
parent b0afba81ca
commit 376dd5a09d
76 changed files with 14860 additions and 102 deletions
@@ -15,7 +15,10 @@ import {
  CatalogConflictError,
  CatalogConnectorError,
  CatalogUnavailableError,
+  DescriptionGenerationRunActiveError,
  type CatalogColumn,
+  type CatalogDescriptionConsolidationCounts,
+  type CatalogDescriptionTarget,
  type CatalogDatabaseMetadataDeleteTarget,
  type CatalogMetadataDeleteCounts,
  type CatalogRelationship,
@@ -31,6 +34,10 @@ import {
  type DatabaseBinding,
  type DatabaseConfigurationInput,
  type DatabaseTestResult,
+  type DescriptionGenerationEvent,
+  type DescriptionGenerationRun,
+  type DescriptionGenerationRunUpdate,
+  type DescriptionGenerationScope,
  type ObservedCatalogTable,
  type ObservedSchemaSnapshot,
  type TableSyncRepositoryResult,
@@ -131,6 +138,33 @@ interface CatalogRelationshipColumnTable {
  targetColumnId: string;
 }

+interface DescriptionGenerationRunTable {
+  id: string;
+  databaseId: string;
+  scope: DescriptionGenerationScope;
+  modelId: string;
+  language: DescriptionGenerationRun["language"];
+  status: DescriptionGenerationRun["status"];
+  total: number;
+  processed: number;
+  generated: number;
+  nonGeneratable: number;
+  failed: number;
+  createdAt: Timestamp;
+  startedAt: Timestamp | null;
+  updatedAt: Timestamp;
+  finishedAt: Timestamp | null;
+  errorSummary: string | null;
+}
+
+interface DescriptionGenerationEventTable {
+  runId: string;
+  sequence: number;
+  level: DescriptionGenerationEvent["level"];
+  message: string;
+  createdAt: Timestamp;
+}
+
 interface CatalogSyncRunTable {
  id: string;
  databaseId: string;
@@ -176,6 +210,8 @@ export interface CatalogDatabase {
  catalogColumns: CatalogColumnTable;
  catalogRelationships: CatalogRelationshipTable;
  catalogRelationshipColumns: CatalogRelationshipColumnTable;
+  descriptionGenerationRuns: DescriptionGenerationRunTable;
+  descriptionGenerationEvents: DescriptionGenerationEventTable;
  catalogSyncRuns: CatalogSyncRunTable;
  catalogSyncEvents: CatalogSyncEventTable;
 }
@@ -281,6 +317,25 @@ function serializeSyncEvent(row: Selectable<CatalogSyncEventTable>): CatalogSync
  return { ...row, id: Number(row.id), data: row.data ?? {}, createdAt: new Date(row.createdAt).toISOString() };
 }

+function serializeDescriptionGenerationRun(
+  row: Selectable<DescriptionGenerationRunTable>,
+): DescriptionGenerationRun {
+  const stamp = (value: Date | string | null) => value === null ? null : new Date(value).toISOString();
+  return {
+    ...row,
+    createdAt: new Date(row.createdAt).toISOString(),
+    startedAt: stamp(row.startedAt),
+    updatedAt: new Date(row.updatedAt).toISOString(),
+    finishedAt: stamp(row.finishedAt),
+  };
+}
+
+function serializeDescriptionGenerationEvent(
+  row: Selectable<DescriptionGenerationEventTable>,
+): DescriptionGenerationEvent {
+  return { ...row, createdAt: new Date(row.createdAt).toISOString() };
+}
+
 function bindingValues(databaseId: string, binding: DatabaseBinding) {
  return {
    databaseId,
@@ -520,6 +575,202 @@ export class KyselyCatalogRepository implements CatalogRepository {
    return row ? await this.getColumn(databaseId, tableId, row.id) : undefined;
  }

+  async consolidateGeneratedDescriptions(
+    databaseId: string,
+    target: CatalogDescriptionTarget,
+    targetIds: readonly string[],
+  ): Promise<CatalogDescriptionConsolidationCounts | undefined> {
+    const selectedTargetIds = [...new Set(targetIds)];
+    if (selectedTargetIds.length === 0) return undefined;
+    return await this.db.transaction().execute(async (trx) => {
+      const database = await trx.selectFrom("workspaceDatabases").select("id")
+        .where("id", "=", databaseId).forUpdate().executeTakeFirst();
+      if (!database) return undefined;
+      if (target === "tables") {
+        const rows = await trx.selectFrom("catalogTables")
+          .select(["id", "generatedDescription"])
+          .where("databaseId", "=", databaseId)
+          .where("id", "in", selectedTargetIds)
+          .orderBy("id")
+          .forUpdate()
+          .execute();
+        if (rows.length !== selectedTargetIds.length) return undefined;
+        const copiedIds = rows
+          .filter((row) => Boolean(row.generatedDescription?.trim()))
+          .map((row) => row.id);
+        if (copiedIds.length > 0) {
+          await trx.updateTable("catalogTables").set({
+            description: sql`generated_description`,
+            version: sql`version + 1`,
+            updatedAt: sql`now()`,
+          }).where("id", "in", copiedIds).execute();
+        }
+        return {
+          copied: copiedIds.length,
+          skipped: selectedTargetIds.length - copiedIds.length,
+        };
+      }
+
+      const tableRows = await trx.selectFrom("catalogTables").select("id")
+        .where("databaseId", "=", databaseId).execute();
+      const rows = tableRows.length === 0 ? [] : await trx.selectFrom("catalogColumns")
+        .select(["id", "generatedDescription"])
+        .where("tableId", "in", tableRows.map((table) => table.id))
+        .where("id", "in", selectedTargetIds)
+        .orderBy("id")
+        .forUpdate()
+        .execute();
+      if (rows.length !== selectedTargetIds.length) return undefined;
+      const copiedIds = rows
+        .filter((row) => Boolean(row.generatedDescription?.trim()))
+        .map((row) => row.id);
+      if (copiedIds.length > 0) {
+        await trx.updateTable("catalogColumns").set({
+          description: sql`generated_description`,
+          version: sql`version + 1`,
+          updatedAt: sql`now()`,
+        }).where("id", "in", copiedIds).execute();
+      }
+      return {
+        copied: copiedIds.length,
+        skipped: selectedTargetIds.length - copiedIds.length,
+      };
+    });
+  }
+
+  async createDescriptionGenerationRun(
+    databaseId: string,
+    scope: DescriptionGenerationScope,
+    modelId: string,
+    language: DescriptionGenerationRun["language"],
+    total: number,
+  ): Promise<DescriptionGenerationRun> {
+    try {
+      const row = await this.db.insertInto("descriptionGenerationRuns").values({
+        id: randomUUID(),
+        databaseId,
+        scope,
+        modelId,
+        language,
+        status: "queued",
+        total,
+        processed: 0,
+        generated: 0,
+        nonGeneratable: 0,
+        failed: 0,
+        startedAt: null,
+        finishedAt: null,
+        errorSummary: null,
+      }).returningAll().executeTakeFirstOrThrow();
+      return serializeDescriptionGenerationRun(row);
+    } catch (error: any) {
+      if (error?.code === "23505"
+        && error?.constraint === "description_generation_runs_one_active") {
+        throw new DescriptionGenerationRunActiveError(
+          "A description generation run is already active",
+        );
+      }
+      throw error;
+    }
+  }
+
+  async getDescriptionGenerationRun(
+    runId: string,
+  ): Promise<DescriptionGenerationRun | undefined> {
+    const row = await this.db.selectFrom("descriptionGenerationRuns")
+      .selectAll()
+      .where("id", "=", runId)
+      .executeTakeFirst();
+    return row ? serializeDescriptionGenerationRun(row) : undefined;
+  }
+
+  async listDescriptionGenerationRuns(limit = 50): Promise<DescriptionGenerationRun[]> {
+    const rows = await this.db.selectFrom("descriptionGenerationRuns")
+      .selectAll()
+      .orderBy("createdAt", "desc")
+      .orderBy("id", "desc")
+      .limit(limit)
+      .execute();
+    return rows.map(serializeDescriptionGenerationRun);
+  }
+
+  async getActiveDescriptionGenerationRun(): Promise<DescriptionGenerationRun | undefined> {
+    const row = await this.db.selectFrom("descriptionGenerationRuns")
+      .selectAll()
+      .where("status", "in", ["queued", "running"])
+      .orderBy("createdAt", "desc")
+      .executeTakeFirst();
+    return row ? serializeDescriptionGenerationRun(row) : undefined;
+  }
+
+  async interruptActiveDescriptionGenerationRuns(
+    errorSummary: string,
+  ): Promise<DescriptionGenerationRun[]> {
+    const rows = await this.db.updateTable("descriptionGenerationRuns")
+      .set({
+        status: "interrupted",
+        finishedAt: sql`now()`,
+        updatedAt: sql`now()`,
+        errorSummary,
+      })
+      .where("status", "in", ["queued", "running"])
+      .returningAll()
+      .execute();
+    return rows.map(serializeDescriptionGenerationRun);
+  }
+
+  async updateDescriptionGenerationRun(
+    runId: string,
+    update: DescriptionGenerationRunUpdate,
+  ): Promise<DescriptionGenerationRun | undefined> {
+    const values: any = { ...update, updatedAt: sql`now()` };
+    const row = await this.db.updateTable("descriptionGenerationRuns")
+      .set(values)
+      .where("id", "=", runId)
+      .returningAll()
+      .executeTakeFirst();
+    return row ? serializeDescriptionGenerationRun(row) : undefined;
+  }
+
+  async appendDescriptionGenerationEvent(
+    runId: string,
+    level: DescriptionGenerationEvent["level"],
+    message: string,
+  ): Promise<DescriptionGenerationEvent> {
+    return await this.db.transaction().execute(async (trx) => {
+      const run = await trx.selectFrom("descriptionGenerationRuns")
+        .select("id")
+        .where("id", "=", runId)
+        .forUpdate()
+        .executeTakeFirst();
+      if (!run) throw new CatalogConflictError("Description Generation Run does not exist");
+      const current = await trx.selectFrom("descriptionGenerationEvents")
+        .select(sql<number>`coalesce(max(sequence), 0)::int`.as("sequence"))
+        .where("runId", "=", runId)
+        .executeTakeFirst();
+      const row = await trx.insertInto("descriptionGenerationEvents").values({
+        runId,
+        sequence: Number(current?.sequence ?? 0) + 1,
+        level,
+        message,
+      }).returningAll().executeTakeFirstOrThrow();
+      return serializeDescriptionGenerationEvent(row);
+    });
+  }
+
+  async listDescriptionGenerationEvents(
+    runId: string,
+    afterSequence = 0,
+  ): Promise<DescriptionGenerationEvent[]> {
+    const rows = await this.db.selectFrom("descriptionGenerationEvents")
+      .selectAll()
+      .where("runId", "=", runId)
+      .where("sequence", ">", afterSequence)
+      .orderBy("sequence")
+      .execute();
+    return rows.map(serializeDescriptionGenerationEvent);
+  }
+
  async listRelationships(databaseId: string): Promise<CatalogRelationship[]> {
    const rows = await this.db.selectFrom("catalogRelationships as relationship")
      .innerJoin("catalogTables as sourceTable", "sourceTable.id", "relationship.sourceTableId")
@@ -1102,6 +1353,15 @@ export class UnavailableCatalogRepository implements CatalogRepository {
  async listColumns(): Promise<CatalogColumn[]> { return this.fail(); }
  async getColumn(): Promise<CatalogColumn | undefined> { return this.fail(); }
  async updateColumnMetadata(): Promise<CatalogColumn | undefined> { return this.fail(); }
+  async consolidateGeneratedDescriptions(): Promise<CatalogDescriptionConsolidationCounts | undefined> { return this.fail(); }
+  async createDescriptionGenerationRun(): Promise<DescriptionGenerationRun> { return this.fail(); }
+  async getDescriptionGenerationRun(): Promise<DescriptionGenerationRun | undefined> { return this.fail(); }
+  async listDescriptionGenerationRuns(): Promise<DescriptionGenerationRun[]> { return this.fail(); }
+  async getActiveDescriptionGenerationRun(): Promise<DescriptionGenerationRun | undefined> { return this.fail(); }
+  async interruptActiveDescriptionGenerationRuns(): Promise<DescriptionGenerationRun[]> { return this.fail(); }
+  async updateDescriptionGenerationRun(): Promise<DescriptionGenerationRun | undefined> { return this.fail(); }
+  async appendDescriptionGenerationEvent(): Promise<DescriptionGenerationEvent> { return this.fail(); }
+  async listDescriptionGenerationEvents(): Promise<DescriptionGenerationEvent[]> { return this.fail(); }
  async listRelationships(): Promise<CatalogRelationship[]> { return this.fail(); }
  async deleteDatabaseMetadata(): Promise<CatalogMetadataDeleteCounts | undefined> { return this.fail(); }
  async deleteTableMetadata(): Promise<CatalogMetadataDeleteCounts | undefined> { return this.fail(); }