feat: complete catalog-driven preprocessing

backend/src/catalog/description-generation-worker.ts

@@ -59,10 +59,16 @@ const completionResponseSchema = z.object({
 class InvalidModelJsonError extends Error {}
 class InvalidModelSchemaError extends Error {}
 class MissingModelTargetsError extends Error {}
+
+interface DescriptionGenerationFailureTarget {
+  id: string;
+  reference: string;
+}
+
 class DescriptionGenerationBatchError extends Error {
   constructor(
     readonly failure: unknown,
-    readonly failedTargets: readonly { id: string; label: "Catalog Column" | "Catalog Table" }[],
+    readonly failedTargets: readonly DescriptionGenerationFailureTarget[],
   ) {
     super("description generation batch failed");
   }
@@ -138,7 +144,7 @@ type ParsedOutcome = z.infer<typeof outcomeSchema>;
 
 interface DescriptionGenerationPlan {
   columnTargets: SelectedColumnTarget[];
-  tableIds: string[];
+  tableTargets: Array<{ id: string; name: string }>;
 }
 
 interface DescriptionGenerationCounters {
@@ -164,10 +170,17 @@ function persistedCounters(counters: DescriptionGenerationCounters) {
   };
 }
 
-function failureTarget(target: SelectedTarget) {
+function targetReference(target: SelectedTarget): string {
   return target.kind === "column"
-    ? { id: target.column.id, label: "Catalog Column" as const }
-    : { id: target.table.id, label: "Catalog Table" as const };
+    ? `Column ${JSON.stringify(`${target.table.name}.${target.column.name}`)}`
+    : `Table ${JSON.stringify(target.table.name)}`;
+}
+
+function failureTarget(target: SelectedTarget): DescriptionGenerationFailureTarget {
+  return {
+    id: target.kind === "column" ? target.column.id : target.table.id,
+    reference: targetReference(target),
+  };
 }
 
 const NON_GENERATABLE_DESCRIPTION: Record<DescriptionGenerationRun["language"], string> = {
@@ -681,7 +694,7 @@ function safeFailure(error: unknown): string {
 function batchFailureEvent(error: DescriptionGenerationBatchError): string {
   const summary = safeFailure(error);
   return error.failedTargets.length > 0
-    ? `${summary} Affected ${error.failedTargets[0]!.label} target${error.failedTargets.length === 1 ? "" : "s"}: ${error.failedTargets.map((target) => target.id).join(", ")}.`
+    ? `${summary} Affected target${error.failedTargets.length === 1 ? "" : "s"}: ${error.failedTargets.map((target) => target.reference).join(", ")}.`
     : summary;
 }
 
@@ -794,7 +807,7 @@ export class DescriptionGenerationWorker {
           scope === "selected_columns" ? "column" : "table",
         );
       }
-      const total = plan.columnTargets.length + plan.tableIds.length;
+      const total = plan.columnTargets.length + plan.tableTargets.length;
       if (total === 0 && (scope === "all" || scope === "missing")) {
         throw new DescriptionGenerationNoEligibleTargetsError(scope);
       }
@@ -934,12 +947,18 @@ export class DescriptionGenerationWorker {
     };
     await this.processTargets(run, database, plan.columnTargets, model, counters, signal);
     throwIfCancelled(signal);
-    if (plan.tableIds.length > 0) {
-      const tableTargets = await this.resolveTableTargets(run.databaseId, plan.tableIds);
+    if (plan.tableTargets.length > 0) {
+      const tableTargets = await this.resolveTableTargets(
+        run.databaseId,
+        plan.tableTargets.map((target) => target.id),
+      );
       if (!tableTargets) {
         throw new DescriptionGenerationBatchError(
           new Error("selected tables changed during generation"),
-          plan.tableIds.map((id) => ({ id, label: "Catalog Table" })),
+          plan.tableTargets.map((target) => ({
+            id: target.id,
+            reference: `Table ${JSON.stringify(target.name)}`,
+          })),
         );
       }
       await this.processTargets(run, database, tableTargets, model, counters, signal);
@@ -1093,8 +1112,8 @@ export class DescriptionGenerationWorker {
           run.id,
           "info",
           outcome.outcome === "generated"
-            ? `Generated description for ${target.kind === "column" ? "Catalog Column" : "Catalog Table"} ${targetId}.`
-            : `Stored non-generatable result for ${target.kind === "column" ? "Catalog Column" : "Catalog Table"} ${targetId}.`,
+            ? `Generated description for ${targetReference(target)}.`
+            : `Stored non-generatable result for ${targetReference(target)}.`,
         );
       }
     }
@@ -1188,16 +1207,22 @@ export class DescriptionGenerationWorker {
       }
       return {
         columnTargets,
-        tableIds: tables
+        tableTargets: tables
           .filter((table) => scope === "all" || !table.generatedDescription?.trim())
-          .map((table) => table.id),
+          .map((table) => ({ id: table.id, name: table.name })),
       };
     }
     if (scope === "selected_tables") {
       const tableById = new Map(tables.map((table) => [table.id, table]));
       const selected = targetIds.map((tableId) => tableById.get(tableId));
       if (selected.some((table) => table === undefined)) return undefined;
-      return { columnTargets: [], tableIds: [...targetIds] };
+      return {
+        columnTargets: [],
+        tableTargets: (selected as CatalogTable[]).map((table) => ({
+          id: table.id,
+          name: table.name,
+        })),
+      };
     }
 
     const byId = new Map<string, SelectedColumnTarget>();
@@ -1209,7 +1234,7 @@ export class DescriptionGenerationWorker {
     const targets = targetIds.map((columnId) => byId.get(columnId));
     return targets.some((target) => target === undefined)
       ? undefined
-      : { columnTargets: targets as SelectedColumnTarget[], tableIds: [] };
+      : { columnTargets: targets as SelectedColumnTarget[], tableTargets: [] };
   }
 
   private async resolveTableTargets(

backend/src/catalog/memory-repository.ts

@@ -18,6 +18,8 @@ import {
   type CatalogLogicalRelationshipCandidate,
   type CatalogLogicalRelationshipContext,
   type CatalogPhysicalRelationship,
+  type CatalogPreprocessingStartResult,
+  type CatalogPreprocessingClearResult,
   type CatalogSchemaDiff,
   type CatalogSyncCounts,
   type CatalogSyncEvent,
@@ -72,6 +74,77 @@ export class MemoryCatalogRepository implements CatalogRepository {
     const value = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
     return value ? clone(value) : undefined;
   }
+  async beginPreprocessing(
+    workspaceId: string,
+    inputFingerprint: string,
+  ): Promise<CatalogPreprocessingStartResult> {
+    const database = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
+    if (!database) return { kind: "not_found" };
+    if (database.preprocessingStatus === "running") return { kind: "already_running" };
+    if (database.schemaSyncedVersion !== database.version) return { kind: "schema_stale" };
+    const catalogBusy = [...this.syncRuns.values()].some((run) =>
+      run.databaseId === database.id
+      && ["queued", "running", "awaiting_confirmation", "applying"].includes(run.state))
+      || [...this.descriptionGenerationRuns.values()].some((run) =>
+        run.databaseId === database.id && ["queued", "running"].includes(run.status))
+      || [...this.sensitivityAnalysisRuns.values()].some((run) =>
+        run.databaseId === database.id && ["queued", "running"].includes(run.status));
+    if (catalogBusy) return { kind: "catalog_busy" };
+    const now = new Date().toISOString();
+    const updated: WorkspaceDatabase = {
+      ...database,
+      preprocessingStatus: "running",
+      preprocessingInputFingerprint: inputFingerprint,
+      preprocessedMetadataRevision: undefined,
+      preprocessingStartedAt: now,
+      preprocessingFinishedAt: undefined,
+      preprocessingErrorCode: undefined,
+      updatedAt: now,
+    };
+    this.records.set(database.id, updated);
+    return { kind: "started", database: clone(updated) };
+  }
+  async finishPreprocessing(
+    workspaceId: string,
+    metadataContentRevision: number,
+    inputFingerprint: string,
+    outcome: { status: "succeeded" } | { status: "failed"; errorCode: string },
+  ): Promise<WorkspaceDatabase | undefined> {
+    const database = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
+    if (!database
+      || database.preprocessingStatus !== "running"
+      || database.metadataContentRevision !== metadataContentRevision
+      || database.preprocessingInputFingerprint !== inputFingerprint) return undefined;
+    const updated: WorkspaceDatabase = {
+      ...database,
+      preprocessingStatus: outcome.status,
+      preprocessedMetadataRevision: outcome.status === "succeeded"
+        ? metadataContentRevision
+        : undefined,
+      preprocessingFinishedAt: new Date().toISOString(),
+      preprocessingErrorCode: outcome.status === "failed" ? outcome.errorCode : undefined,
+    };
+    this.records.set(database.id, updated);
+    return clone(updated);
+  }
+  async clearPreprocessing(workspaceId: string): Promise<CatalogPreprocessingClearResult> {
+    const database = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
+    if (!database) return { kind: "not_found" };
+    if (database.preprocessingStatus === "running") return { kind: "already_running" };
+    const now = new Date().toISOString();
+    const updated: WorkspaceDatabase = {
+      ...database,
+      preprocessingStatus: "failed",
+      preprocessingInputFingerprint: undefined,
+      preprocessedMetadataRevision: undefined,
+      preprocessingStartedAt: undefined,
+      preprocessingFinishedAt: now,
+      preprocessingErrorCode: "derived_data_cleared",
+      updatedAt: now,
+    };
+    this.records.set(database.id, updated);
+    return { kind: "cleared", database: clone(updated) };
+  }
   async getCatalogMetrics(databaseId?: string): Promise<CatalogMetrics | undefined> {
     if (databaseId !== undefined && !this.records.has(databaseId)) return undefined;
 
@@ -114,6 +187,8 @@ export class MemoryCatalogRepository implements CatalogRepository {
       createdAt: now,
       updatedAt: now,
       connectionStatus: "untested",
+      metadataContentRevision: 0,
+      preprocessingStatus: "failed",
     };
     this.records.set(record.id, record);
     return clone(record);
@@ -286,12 +361,24 @@ export class MemoryCatalogRepository implements CatalogRepository {
       return undefined;
     }
     const now = new Date().toISOString();
-    if (target === "database_columns") {
-      const targets = [...this.columns.values()].filter((column) => (
+    if (target === "database" || target === "database_columns") {
+      const tableTargets = target === "database"
+        ? [...this.tables.values()].filter((table) => table.databaseId === databaseId)
+        : [];
+      const columnTargets = [...this.columns.values()].filter((column) => (
         this.tables.get(column.tableId)?.databaseId === databaseId
       ));
-      const copied = targets.filter((column) => Boolean(column.generatedDescription?.trim()));
-      for (const column of copied) {
+      const copiedTables = tableTargets.filter((table) => Boolean(table.generatedDescription?.trim()));
+      const copiedColumns = columnTargets.filter((column) => Boolean(column.generatedDescription?.trim()));
+      for (const table of copiedTables) {
+        this.tables.set(table.id, {
+          ...table,
+          description: table.generatedDescription,
+          version: table.version + 1,
+          updatedAt: now,
+        });
+      }
+      for (const column of copiedColumns) {
         this.columns.set(column.id, {
           ...column,
           description: column.generatedDescription,
@@ -299,7 +386,8 @@ export class MemoryCatalogRepository implements CatalogRepository {
           updatedAt: now,
         });
       }
-      return { copied: copied.length, skipped: targets.length - copied.length };
+      const copied = copiedTables.length + copiedColumns.length;
+      return { copied, skipped: tableTargets.length + columnTargets.length - copied };
     }
     if (selectedTargetIds.length === 0) return undefined;
     if (target === "tables") {

backend/src/catalog/metadata-snapshot.ts

@@ -0,0 +1,144 @@
+import type {
+  CatalogColumn,
+  CatalogLogicalRelationship,
+  CatalogPhysicalRelationship,
+  CatalogRepository,
+  CatalogTable,
+} from "./types.js";
+
+export type CatalogDescriptionSource = "curated" | "generated" | "source_comment";
+
+export interface CatalogMetadataSnapshotColumn {
+  id: string;
+  name: string;
+  ordinalPosition: number;
+  dataType: string;
+  isNullable: boolean;
+  defaultExpression: string | null;
+  primaryKeyPosition: number | null;
+  sensitive: boolean;
+  description: string | null;
+  descriptionSource: CatalogDescriptionSource | null;
+}
+
+export interface CatalogMetadataSnapshotTable {
+  id: string;
+  name: string;
+  description: string | null;
+  descriptionSource: CatalogDescriptionSource | null;
+  columns: CatalogMetadataSnapshotColumn[];
+}
+
+export interface CatalogMetadataSnapshotRelationship {
+  id: string;
+  origin: "physical" | "generated" | "manual";
+  sourceTable: string;
+  sourceColumns: string[];
+  targetTable: string;
+  targetColumns: string[];
+}
+
+export interface CatalogMetadataSnapshot {
+  schemaVersion: 1;
+  workspaceId: string;
+  databaseId: string;
+  databaseName: string;
+  schemaName: string;
+  metadataContentRevision: number;
+  tables: CatalogMetadataSnapshotTable[];
+  relationships: CatalogMetadataSnapshotRelationship[];
+}
+
+function effectiveDescription(value: {
+  description: string | null;
+  generatedDescription: string | null;
+  sourceComment: string | null;
+}): { description: string | null; descriptionSource: CatalogDescriptionSource | null } {
+  if (value.description?.trim()) {
+    return { description: value.description.trim(), descriptionSource: "curated" };
+  }
+  if (value.generatedDescription?.trim()) {
+    return { description: value.generatedDescription.trim(), descriptionSource: "generated" };
+  }
+  if (value.sourceComment?.trim()) {
+    return { description: value.sourceComment.trim(), descriptionSource: "source_comment" };
+  }
+  return { description: null, descriptionSource: null };
+}
+
+function snapshotColumn(column: CatalogColumn): CatalogMetadataSnapshotColumn {
+  return {
+    id: column.id,
+    name: column.name,
+    ordinalPosition: column.ordinalPosition,
+    dataType: column.dataType,
+    isNullable: column.isNullable,
+    defaultExpression: column.defaultExpression,
+    primaryKeyPosition: column.primaryKeyPosition,
+    sensitive: column.sensitive,
+    ...effectiveDescription(column),
+  };
+}
+
+function snapshotRelationship(
+  relationship: CatalogPhysicalRelationship | CatalogLogicalRelationship,
+): CatalogMetadataSnapshotRelationship {
+  const columns = [...relationship.columns].sort((left, right) => left.position - right.position);
+  return {
+    id: relationship.id,
+    origin: relationship.origin,
+    sourceTable: relationship.sourceTableName,
+    sourceColumns: columns.map((column) => column.sourceColumnName),
+    targetTable: relationship.targetTableName,
+    targetColumns: columns.map((column) => column.targetColumnName),
+  };
+}
+
+export async function buildCatalogMetadataSnapshot(
+  repository: CatalogRepository,
+  workspaceId: string,
+  expectedMetadataContentRevision: number,
+): Promise<CatalogMetadataSnapshot> {
+  const database = await repository.getByWorkspace(workspaceId);
+  if (!database || database.preprocessingStatus !== "running") {
+    throw new Error("catalog preprocessing lease is not active");
+  }
+  if (database.metadataContentRevision !== expectedMetadataContentRevision) {
+    throw new Error("catalog metadata revision changed");
+  }
+
+  const catalogTables = await repository.listTables(database.id);
+  const tables: CatalogMetadataSnapshotTable[] = [];
+  for (const table of [...catalogTables].sort((left, right) => left.name.localeCompare(right.name))) {
+    const columns = await repository.listColumns(database.id, table.id);
+    tables.push({
+      id: table.id,
+      name: table.name,
+      ...effectiveDescription(table),
+      columns: columns
+        .sort((left, right) => left.ordinalPosition - right.ordinalPosition || left.name.localeCompare(right.name))
+        .map(snapshotColumn),
+    });
+  }
+
+  const physical = await repository.listRelationships(database.id);
+  const logical = (await repository.listLogicalRelationships(database.id))
+    .filter((relationship) => relationship.status === "active");
+  const relationships = [...physical, ...logical]
+    .map(snapshotRelationship)
+    .sort((left, right) =>
+      left.sourceTable.localeCompare(right.sourceTable)
+      || left.targetTable.localeCompare(right.targetTable)
+      || left.id.localeCompare(right.id));
+
+  return {
+    schemaVersion: 1,
+    workspaceId,
+    databaseId: database.id,
+    databaseName: database.databaseName,
+    schemaName: database.schema,
+    metadataContentRevision: expectedMetadataContentRevision,
+    tables,
+    relationships,
+  };
+}

backend/src/catalog/migrate.ts

@@ -15,6 +15,7 @@ import * as aiTokenUsageMigration from "./migrations/009_ai_token_usage.js";
 import * as canonicalModelIdsMigration from "./migrations/010_canonical_model_ids.js";
 import * as localSensitivityAnalysisMigration from "./migrations/011_local_sensitivity_analysis.js";
 import * as sensitivityReasonMigration from "./migrations/012_sensitivity_reason.js";
+import * as catalogPreprocessingStateMigration from "./migrations/013_catalog_preprocessing_state.js";
 
 const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL;
 const host = process.env.THT_CATALOG_DB_HOST;
@@ -52,6 +53,7 @@ const provider: MigrationProvider = {
       "010_canonical_model_ids": canonicalModelIdsMigration,
       "011_local_sensitivity_analysis": localSensitivityAnalysisMigration,
       "012_sensitivity_reason": sensitivityReasonMigration,
+      "013_catalog_preprocessing_state": catalogPreprocessingStateMigration,
     };
   },
 };

backend/src/catalog/migrations/013_catalog_preprocessing_state.ts

@@ -0,0 +1,212 @@
+import { type Kysely, sql } from "kysely";
+import type { CatalogDatabase } from "../repository.js";
+
+export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
+  await db.schema.alterTable("workspace_databases")
+    .addColumn("metadata_content_revision", "bigint", (column) => column.notNull().defaultTo(0))
+    .addColumn("preprocessing_status", "text", (column) => column.notNull().defaultTo("failed"))
+    .addColumn("preprocessing_input_fingerprint", "text")
+    .addColumn("preprocessed_metadata_revision", "bigint")
+    .addColumn("preprocessing_started_at", "timestamptz")
+    .addColumn("preprocessing_finished_at", "timestamptz")
+    .addColumn("preprocessing_error_code", "text")
+    .execute();
+  await sql`
+    alter table workspace_databases
+    add constraint workspace_databases_preprocessing_status_check
+    check (preprocessing_status in ('running', 'succeeded', 'failed'))
+  `.execute(db);
+
+  await sql`
+    create function catalog_metadata_write_guard()
+    returns trigger
+    language plpgsql
+    as $$
+    declare
+      resolved_database_id uuid;
+      current_status text;
+      relation_id uuid;
+      table_id uuid;
+    begin
+      if tg_table_name = 'catalog_tables' then
+        resolved_database_id := coalesce(new.database_id, old.database_id);
+      elsif tg_table_name = 'catalog_columns' then
+        table_id := coalesce(new.table_id, old.table_id);
+        select database_id into resolved_database_id from catalog_tables where id = table_id;
+      elsif tg_table_name = 'catalog_relationships' then
+        resolved_database_id := coalesce(new.database_id, old.database_id);
+      elsif tg_table_name = 'catalog_relationship_columns' then
+        relation_id := coalesce(new.relationship_id, old.relationship_id);
+        select database_id into resolved_database_id from catalog_relationships where id = relation_id;
+      elsif tg_table_name = 'catalog_logical_relationships' then
+        resolved_database_id := coalesce(new.database_id, old.database_id);
+      elsif tg_table_name = 'database_bindings' then
+        if tg_op = 'UPDATE' and not (
+          new.transport is distinct from old.transport
+          or new.host is distinct from old.host
+          or new.port is distinct from old.port
+          or new.username is distinct from old.username
+          or new.base_url is distinct from old.base_url
+          or new.rest_path is distinct from old.rest_path
+          or new.rest_auth is distinct from old.rest_auth
+          or new.tls_servername is distinct from old.tls_servername
+          or new.ssh_host is distinct from old.ssh_host
+          or new.ssh_port is distinct from old.ssh_port
+          or new.ssh_username is distinct from old.ssh_username
+          or new.ssh_target_host is distinct from old.ssh_target_host
+          or new.ssh_target_port is distinct from old.ssh_target_port
+        ) then
+          return new;
+        end if;
+        resolved_database_id := coalesce(new.database_id, old.database_id);
+      end if;
+      if resolved_database_id is null then
+        if tg_op = 'DELETE' then return old; else return new; end if;
+      end if;
+
+      select preprocessing_status into current_status
+      from workspace_databases
+      where id = resolved_database_id
+      for update;
+
+      if current_status = 'running' then
+        raise exception 'catalog preprocessing is running'
+          using errcode = '55000';
+      end if;
+
+      update workspace_databases
+      set metadata_content_revision = metadata_content_revision + 1,
+          preprocessing_status = 'failed',
+          preprocessing_finished_at = now(),
+          preprocessing_error_code = 'catalog_changed',
+          updated_at = now()
+      where id = resolved_database_id;
+      if tg_op = 'DELETE' then return old; else return new; end if;
+    end;
+    $$
+  `.execute(db);
+
+  for (const table of [
+    "catalog_tables",
+    "catalog_columns",
+    "catalog_relationships",
+    "catalog_relationship_columns",
+    "catalog_logical_relationships",
+    "database_bindings",
+  ]) {
+    await sql.raw(`
+      create trigger ${table}_metadata_write_guard
+      before insert or update or delete on ${table}
+      for each row execute function catalog_metadata_write_guard()
+    `).execute(db);
+  }
+
+  await sql`
+    create function catalog_database_configuration_guard()
+    returns trigger
+    language plpgsql
+    as $$
+    begin
+      if new.workspace_id is distinct from old.workspace_id
+        or new.engine is distinct from old.engine
+        or new.database_name is distinct from old.database_name
+        or new.schema_name is distinct from old.schema_name then
+        if old.preprocessing_status = 'running' then
+          raise exception 'catalog preprocessing is running'
+            using errcode = '55000';
+        end if;
+        new.metadata_content_revision := old.metadata_content_revision + 1;
+        new.preprocessing_status := 'failed';
+        new.preprocessing_finished_at := now();
+        new.preprocessing_error_code := 'catalog_changed';
+      end if;
+      return new;
+    end;
+    $$
+  `.execute(db);
+  await sql`
+    create trigger workspace_databases_configuration_guard
+    before update of workspace_id, engine, database_name, schema_name on workspace_databases
+    for each row execute function catalog_database_configuration_guard()
+  `.execute(db);
+
+  await sql`
+    create function catalog_operation_start_guard()
+    returns trigger
+    language plpgsql
+    as $$
+    declare
+      current_status text;
+      starting boolean;
+    begin
+      if tg_table_name = 'catalog_sync_runs' then
+        starting := new.state in ('queued', 'running', 'awaiting_confirmation', 'applying');
+      else
+        starting := new.status in ('queued', 'running');
+      end if;
+      if not starting then
+        return new;
+      end if;
+
+      select preprocessing_status into current_status
+      from workspace_databases
+      where id = new.database_id
+      for update;
+      if current_status = 'running' then
+        raise exception 'catalog preprocessing is running'
+          using errcode = '55000';
+      end if;
+      return new;
+    end;
+    $$
+  `.execute(db);
+
+  for (const table of [
+    "catalog_sync_runs",
+    "description_generation_runs",
+    "sensitive_data_suggestion_runs",
+  ]) {
+    await sql.raw(`
+      create trigger ${table}_operation_start_guard
+      before insert or update on ${table}
+      for each row execute function catalog_operation_start_guard()
+    `).execute(db);
+  }
+}
+
+export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
+  for (const table of [
+    "catalog_sync_runs",
+    "description_generation_runs",
+    "sensitive_data_suggestion_runs",
+  ]) {
+    await sql.raw(`drop trigger if exists ${table}_operation_start_guard on ${table}`).execute(db);
+  }
+  await sql`drop function if exists catalog_operation_start_guard()`.execute(db);
+  await sql`drop trigger if exists workspace_databases_configuration_guard on workspace_databases`.execute(db);
+  await sql`drop function if exists catalog_database_configuration_guard()`.execute(db);
+  for (const table of [
+    "catalog_tables",
+    "catalog_columns",
+    "catalog_relationships",
+    "catalog_relationship_columns",
+    "catalog_logical_relationships",
+    "database_bindings",
+  ]) {
+    await sql.raw(`drop trigger if exists ${table}_metadata_write_guard on ${table}`).execute(db);
+  }
+  await sql`drop function if exists catalog_metadata_write_guard()`.execute(db);
+  await db.schema.alterTable("workspace_databases")
+    .dropConstraint("workspace_databases_preprocessing_status_check").execute();
+  for (const column of [
+    "preprocessing_error_code",
+    "preprocessing_finished_at",
+    "preprocessing_started_at",
+    "preprocessed_metadata_revision",
+    "preprocessing_input_fingerprint",
+    "preprocessing_status",
+    "metadata_content_revision",
+  ]) {
+    await sql.raw(`alter table workspace_databases drop column ${column}`).execute(db);
+  }
+}

backend/src/catalog/repository.ts

@@ -27,6 +27,8 @@ import {
   type CatalogLogicalRelationshipCandidate,
   type CatalogLogicalRelationshipContext,
   type CatalogPhysicalRelationship,
+  type CatalogPreprocessingStartResult,
+  type CatalogPreprocessingClearResult,
   type CatalogSchemaDiff,
   type CatalogSyncCounts,
   type CatalogSyncEvent,
@@ -54,6 +56,11 @@ import {
 } from "./types.js";
 
 type Timestamp = ColumnType<Date, Date | string | undefined, Date | string>;
+type NullableTimestamp = ColumnType<
+  Date | null,
+  Date | string | null | undefined,
+  Date | string | null
+>;
 
 interface WorkspaceDatabaseTable {
   id: string;
@@ -66,6 +73,13 @@ interface WorkspaceDatabaseTable {
   updatedAt: Timestamp;
   schemaSyncedVersion: number | null;
   schemaSyncedAt: Timestamp | null;
+  metadataContentRevision: Generated<number>;
+  preprocessingStatus: Generated<WorkspaceDatabase["preprocessingStatus"]>;
+  preprocessingInputFingerprint: Generated<string | null>;
+  preprocessedMetadataRevision: Generated<number | null>;
+  preprocessingStartedAt: NullableTimestamp;
+  preprocessingFinishedAt: NullableTimestamp;
+  preprocessingErrorCode: Generated<string | null>;
 }
 
 interface DatabaseBindingTable {
@@ -325,6 +339,19 @@ function serialize(row: JoinedRow): WorkspaceDatabase {
     lastErrorMessage: present(row.lastErrorMessage),
     schemaSyncedVersion: present(row.schemaSyncedVersion),
     schemaSyncedAt: row.schemaSyncedAt == null ? undefined : new Date(row.schemaSyncedAt).toISOString(),
+    metadataContentRevision: Number(row.metadataContentRevision),
+    preprocessingStatus: row.preprocessingStatus,
+    preprocessingInputFingerprint: present(row.preprocessingInputFingerprint),
+    preprocessedMetadataRevision: row.preprocessedMetadataRevision == null
+      ? undefined
+      : Number(row.preprocessedMetadataRevision),
+    preprocessingStartedAt: row.preprocessingStartedAt == null
+      ? undefined
+      : new Date(row.preprocessingStartedAt).toISOString(),
+    preprocessingFinishedAt: row.preprocessingFinishedAt == null
+      ? undefined
+      : new Date(row.preprocessingFinishedAt).toISOString(),
+    preprocessingErrorCode: present(row.preprocessingErrorCode),
   };
 }
 
@@ -477,6 +504,98 @@ export class KyselyCatalogRepository implements CatalogRepository {
     return id ? await this.get(id.id) : undefined;
   }
 
+  async beginPreprocessing(
+    workspaceId: string,
+    inputFingerprint: string,
+  ): Promise<CatalogPreprocessingStartResult> {
+    return await this.db.transaction().execute(async (trx) => {
+      const database = await trx.selectFrom("workspaceDatabases")
+        .selectAll()
+        .where("workspaceId", "=", workspaceId)
+        .forUpdate()
+        .executeTakeFirst();
+      if (!database) return { kind: "not_found" };
+      if (database.preprocessingStatus === "running") return { kind: "already_running" };
+      if (database.schemaSyncedVersion !== database.version) return { kind: "schema_stale" };
+
+      const activeSync = await trx.selectFrom("catalogSyncRuns")
+        .select("id")
+        .where("databaseId", "=", database.id)
+        .where("state", "in", ["queued", "running", "awaiting_confirmation", "applying"])
+        .executeTakeFirst();
+      const activeDescriptions = await trx.selectFrom("descriptionGenerationRuns")
+        .select("id")
+        .where("databaseId", "=", database.id)
+        .where("status", "in", ["queued", "running"])
+        .executeTakeFirst();
+      const activeSensitivity = await trx.selectFrom("sensitiveDataSuggestionRuns")
+        .select("id")
+        .where("databaseId", "=", database.id)
+        .where("status", "=", "running")
+        .executeTakeFirst();
+      if (activeSync || activeDescriptions || activeSensitivity) return { kind: "catalog_busy" };
+
+      await trx.updateTable("workspaceDatabases")
+        .set({
+          preprocessingStatus: "running",
+          preprocessingInputFingerprint: inputFingerprint,
+          preprocessedMetadataRevision: null,
+          preprocessingStartedAt: sql`now()`,
+          preprocessingFinishedAt: null,
+          preprocessingErrorCode: null,
+          updatedAt: sql`now()`,
+        })
+        .where("id", "=", database.id)
+        .execute();
+      return { kind: "started", database: (await selectOne(trx, database.id))! };
+    });
+  }
+
+  async finishPreprocessing(
+    workspaceId: string,
+    metadataContentRevision: number,
+    inputFingerprint: string,
+    outcome: { status: "succeeded" } | { status: "failed"; errorCode: string },
+  ): Promise<WorkspaceDatabase | undefined> {
+    const result = await this.db.updateTable("workspaceDatabases")
+      .set({
+        preprocessingStatus: outcome.status,
+        preprocessedMetadataRevision: outcome.status === "succeeded" ? metadataContentRevision : null,
+        preprocessingFinishedAt: sql`now()`,
+        preprocessingErrorCode: outcome.status === "failed" ? outcome.errorCode : null,
+        updatedAt: sql`now()`,
+      })
+      .where("workspaceId", "=", workspaceId)
+      .where("preprocessingStatus", "=", "running")
+      .where("metadataContentRevision", "=", metadataContentRevision)
+      .where("preprocessingInputFingerprint", "=", inputFingerprint)
+      .returning("id")
+      .executeTakeFirst();
+    return result ? await this.get(result.id) : undefined;
+  }
+
+  async clearPreprocessing(workspaceId: string): Promise<CatalogPreprocessingClearResult> {
+    return await this.db.transaction().execute(async (trx) => {
+      const database = await trx.selectFrom("workspaceDatabases")
+        .select(["id", "preprocessingStatus"])
+        .where("workspaceId", "=", workspaceId)
+        .forUpdate()
+        .executeTakeFirst();
+      if (!database) return { kind: "not_found" };
+      if (database.preprocessingStatus === "running") return { kind: "already_running" };
+      await trx.updateTable("workspaceDatabases").set({
+        preprocessingStatus: "failed",
+        preprocessingInputFingerprint: null,
+        preprocessedMetadataRevision: null,
+        preprocessingStartedAt: null,
+        preprocessingFinishedAt: sql`now()`,
+        preprocessingErrorCode: "derived_data_cleared",
+        updatedAt: sql`now()`,
+      }).where("id", "=", database.id).execute();
+      return { kind: "cleared", database: (await selectOne(trx, database.id))! };
+    });
+  }
+
   async getCatalogMetrics(databaseId?: string): Promise<CatalogMetrics | undefined> {
     const result = await sql<CatalogMetricsRow>`
       WITH requested_database AS (
@@ -763,7 +882,9 @@ export class KyselyCatalogRepository implements CatalogRepository {
     targetIds: readonly string[],
   ): Promise<CatalogDescriptionConsolidationCounts | undefined> {
     const selectedTargetIds = [...new Set(targetIds)];
-    if (target !== "database_columns" && selectedTargetIds.length === 0) return undefined;
+    if (target !== "database" && target !== "database_columns" && selectedTargetIds.length === 0) {
+      return undefined;
+    }
     return await this.db.transaction().execute(async (trx) => {
       const database = await trx.selectFrom("workspaceDatabases").select("id")
         .where("id", "=", databaseId).forUpdate().executeTakeFirst();
@@ -793,8 +914,23 @@ export class KyselyCatalogRepository implements CatalogRepository {
         };
       }
 
-      const tableRows = await trx.selectFrom("catalogTables").select("id")
-        .where("databaseId", "=", databaseId).execute();
+      const tableRows = await trx.selectFrom("catalogTables").select(["id", "generatedDescription"])
+        .where("databaseId", "=", databaseId)
+        .orderBy("id")
+        .forUpdate()
+        .execute();
+      const copiedTableIds = target === "database"
+        ? tableRows
+          .filter((row) => Boolean(row.generatedDescription?.trim()))
+          .map((row) => row.id)
+        : [];
+      if (copiedTableIds.length > 0) {
+        await trx.updateTable("catalogTables").set({
+          description: sql`generated_description`,
+          version: sql`version + 1`,
+          updatedAt: sql`now()`,
+        }).where("id", "in", copiedTableIds).execute();
+      }
       const rows = tableRows.length === 0 ? [] : await trx.selectFrom("catalogColumns")
         .select(["id", "generatedDescription"])
         .where("tableId", "in", tableRows.map((table) => table.id))
@@ -812,7 +948,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
           version: sql`version + 1`,
           updatedAt: sql`now()`,
         });
-        update = target === "database_columns"
+        update = target === "database" || target === "database_columns"
           ? update
             .where("tableId", "in", tableRows.map((table) => table.id))
             .where(sql<boolean>`nullif(btrim(generated_description), '') is not null`)
@@ -820,8 +956,9 @@ export class KyselyCatalogRepository implements CatalogRepository {
         await update.execute();
       }
       return {
-        copied: copiedIds.length,
-        skipped: rows.length - copiedIds.length,
+        copied: copiedTableIds.length + copiedIds.length,
+        skipped: (target === "database" ? tableRows.length : 0) - copiedTableIds.length
+          + rows.length - copiedIds.length,
       };
     });
   }
@@ -1835,6 +1972,9 @@ export class UnavailableCatalogRepository implements CatalogRepository {
   async list(): Promise<WorkspaceDatabase[]> { return this.fail(); }
   async get(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
   async getByWorkspace(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
+  async beginPreprocessing(): Promise<CatalogPreprocessingStartResult> { return this.fail(); }
+  async finishPreprocessing(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }
+  async clearPreprocessing(): Promise<CatalogPreprocessingClearResult> { return this.fail(); }
   async getCatalogMetrics(): Promise<CatalogMetrics | undefined> { return this.fail(); }
   async create(): Promise<WorkspaceDatabase> { return this.fail(); }
   async update(): Promise<WorkspaceDatabase | undefined> { return this.fail(); }

backend/src/catalog/runtime-binding.ts

@@ -0,0 +1,140 @@
+import { dirname } from "node:path";
+
+import { resolveRuntimeBindings, type RuntimeBindings } from "../workspaces/bindings.js";
+import { buildInstallationContract, type InstallationSuffix } from "../workspaces/contracts.js";
+import {
+  discoverWorkspaceSecretRequirements,
+} from "../workspaces/secret-requirements.js";
+import type {
+  WorkspaceSecretMaterialization,
+  WorkspaceSecretStore,
+} from "../workspaces/secret-store.js";
+import {
+  validateWorkspaceDescriptor,
+  type WorkspaceDescriptor,
+} from "../workspaces/schema.js";
+import { CATALOG_SECRET_IDS } from "./secrets.js";
+import type { WorkspaceDatabase } from "./types.js";
+
+export interface CatalogRuntimeBindingLease {
+  workspace: WorkspaceDescriptor;
+  bindings: RuntimeBindings;
+  release(): void;
+}
+
+const CATALOG_SECRET_BY_SUFFIX: Readonly<Partial<Record<InstallationSuffix, string>>> = {
+  PASSWORD_FILE: CATALOG_SECRET_IDS.password,
+  API_KEY_FILE: CATALOG_SECRET_IDS.apiKey,
+  TLS_CA_FILE: CATALOG_SECRET_IDS.tlsCa,
+  SSH_PRIVATE_KEY_FILE: CATALOG_SECRET_IDS.sshPrivateKey,
+  SSH_KNOWN_HOSTS_FILE: CATALOG_SECRET_IDS.sshKnownHosts,
+};
+
+function runtimeWorkspace(
+  workspace: WorkspaceDescriptor,
+  database: WorkspaceDatabase,
+): WorkspaceDescriptor {
+  if (workspace.workspace.id !== database.workspaceId) {
+    throw new Error("Catalog database binding does not belong to the workspace");
+  }
+  const { dwh: _legacyDwh, diagnostics: _legacyDiagnostics, ...descriptor } = workspace;
+  const binding = database.binding;
+  return validateWorkspaceDescriptor({
+    ...descriptor,
+    dwh: {
+      engine: "postgres",
+      database: database.databaseName,
+      schema: database.schema,
+      ...(binding.port === undefined ? {} : { port: binding.port }),
+      supported_transports: [binding.transport],
+    },
+    ...(binding.transport === "rest_api" ? {
+      diagnostics: {
+        dwh_rest: {
+          method: "GET",
+          path: binding.restPath ?? "/health",
+          auth: binding.restAuth ?? "bearer",
+          response: { database: "database", schema: "schema" },
+        },
+      },
+    } : {}),
+  });
+}
+
+function setIfDefined(
+  environment: NodeJS.ProcessEnv,
+  name: string | undefined,
+  value: string | number | undefined,
+): void {
+  if (name !== undefined && value !== undefined && value !== "") environment[name] = String(value);
+}
+
+/**
+ * Project one PostgreSQL Catalog row into the legacy-shaped configuration consumed by the
+ * Python runtime. The authored workspace remains database-free; this object exists only for
+ * the lifetime of a backend-owned runtime lease.
+ */
+export function resolveCatalogRuntimeBinding(options: {
+  workspace: WorkspaceDescriptor;
+  database: WorkspaceDatabase;
+  environment: NodeJS.ProcessEnv;
+  secretRoots: readonly string[];
+  secretStore: WorkspaceSecretStore;
+}): CatalogRuntimeBindingLease {
+  const workspace = runtimeWorkspace(options.workspace, options.database);
+  const evidenceRequirements = discoverWorkspaceSecretRequirements(
+    options.workspace,
+    options.environment,
+  ).filter(({ connector }) => connector === "evidence");
+  const catalogSecretIds = Object.values(CATALOG_SECRET_IDS);
+  let materialization: WorkspaceSecretMaterialization | undefined;
+  try {
+    materialization = options.secretStore.materialize(
+      options.database.workspaceId,
+      [...catalogSecretIds, ...evidenceRequirements.map(({ id }) => id)],
+    );
+    const environment: NodeJS.ProcessEnv = { ...options.environment };
+    const roots = new Set(options.secretRoots);
+    for (const path of materialization.files.values()) roots.add(dirname(path));
+
+    const variables = buildInstallationContract(workspace).variables;
+    const variable = (role: "DWH" | "EVIDENCE", suffix: InstallationSuffix) => (
+      variables.find((candidate) => candidate.role === role && candidate.suffix === suffix)?.name
+    );
+    const binding = options.database.binding;
+    setIfDefined(environment, variable("DWH", "TRANSPORT"), binding.transport);
+    setIfDefined(environment, variable("DWH", "HOST"), binding.host);
+    setIfDefined(environment, variable("DWH", "PORT"), binding.port);
+    setIfDefined(environment, variable("DWH", "BASE_URL"), binding.baseUrl);
+    setIfDefined(environment, variable("DWH", "USER"), binding.username);
+    setIfDefined(environment, variable("DWH", "SSH_HOST"), binding.sshHost);
+    setIfDefined(environment, variable("DWH", "SSH_PORT"), binding.sshPort);
+    setIfDefined(environment, variable("DWH", "SSH_USER"), binding.sshUsername);
+    setIfDefined(environment, variable("DWH", "SSH_TARGET_HOST"), binding.sshTargetHost);
+    setIfDefined(environment, variable("DWH", "SSH_TARGET_PORT"), binding.sshTargetPort);
+    for (const [suffix, secretId] of Object.entries(CATALOG_SECRET_BY_SUFFIX) as Array<[
+      InstallationSuffix,
+      string,
+    ]>) {
+      setIfDefined(environment, variable("DWH", suffix), materialization.files.get(secretId));
+    }
+    for (const requirement of evidenceRequirements) {
+      setIfDefined(environment, requirement.variable, materialization.files.get(requirement.id));
+    }
+
+    const bindings = resolveRuntimeBindings(workspace, environment, [...roots]);
+    let released = false;
+    return {
+      workspace,
+      bindings,
+      release: () => {
+        if (released) return;
+        released = true;
+        materialization?.release();
+      },
+    };
+  } catch (error) {
+    materialization?.release();
+    throw error;
+  }
+}

backend/src/catalog/service.ts

@@ -1,5 +1,3 @@
-import { buildInstallationContract } from "../workspaces/contracts.js";
-import { resolveBinding } from "../workspaces/bindings.js";
 import type { WorkspaceRegistry } from "../workspaces/registry.js";
 import type { WorkspaceDescriptor } from "../workspaces/schema.js";
 import { discoverWorkspaceSecretRequirements } from "../workspaces/secret-requirements.js";
@@ -45,60 +43,33 @@ export interface CatalogListItem extends Omit<WorkspaceDatabase, "id"> {
   secrets: Record<CatalogSecretName, boolean>;
 }
 
-function bindingValue(workspace: WorkspaceDescriptor, values: Record<string, string>, suffix: string) {
-  const variable = buildInstallationContract(workspace).variables.find((entry) => (
-    entry.role === "DWH" && entry.suffix === suffix
-  ));
-  return variable ? values[variable.name] : undefined;
-}
-
-function numeric(value: string | undefined): number | undefined {
-  if (!value) return undefined;
-  const parsed = Number(value);
-  return Number.isInteger(parsed) && parsed >= 1 && parsed <= 65_535 ? parsed : undefined;
-}
-
-function yamlBinding(workspace: WorkspaceDescriptor, secretRoots: readonly string[]): DatabaseBinding {
-  const effective = resolveBinding(workspace, "DWH", process.env, secretRoots);
-  const value = (suffix: string) => bindingValue(workspace, effective.values, suffix);
-  return {
-    transport: effective.transport,
-    host: value("HOST"),
-    port: numeric(value("PORT")) ?? workspace.dwh.port,
-    username: value("USER"),
-    baseUrl: value("BASE_URL"),
-    restPath: workspace.diagnostics?.dwh_rest?.path ?? "/health",
-    restAuth: workspace.diagnostics?.dwh_rest?.auth ?? "bearer",
-    tlsServername: value("TLS_SERVERNAME"),
-    sshHost: value("SSH_HOST"),
-    sshPort: numeric(value("SSH_PORT")),
-    sshUsername: value("SSH_USER"),
-    sshTargetHost: value("SSH_TARGET_HOST"),
-    sshTargetPort: numeric(value("SSH_TARGET_PORT")),
-  };
-}
-
 function secretState(store: WorkspaceSecretStore, workspaceId: string): Record<CatalogSecretName, boolean> {
   return Object.fromEntries(Object.entries(CATALOG_SECRET_IDS).map(([name, id]) => (
     [name, store.has(workspaceId, id)]
   ))) as Record<CatalogSecretName, boolean>;
 }
 
-function workspaceRuntimeState(
+function databaseRuntimeState(
   store: WorkspaceSecretStore,
-  workspace: WorkspaceDescriptor,
-  secretRoots: readonly string[],
+  database: WorkspaceDatabase,
 ): NonNullable<CatalogListItem["runtimeBinding"]> {
-  const requirements = discoverWorkspaceSecretRequirements(workspace, process.env);
-  const secretVariables = new Set(requirements.map(({ variable }) => variable));
-  const effective = resolveBinding(workspace, "DWH", process.env, secretRoots);
-  const configurationRequired = requirements.some(({ id, required }) => (
-    required && !store.has(workspace.workspace.id, id)
-  )) || effective.missing.some((variable) => !secretVariables.has(variable));
+  const { binding, workspaceId } = database;
+  const configured = (id: string) => store.has(workspaceId, id);
+  const connectionComplete = binding.transport === "postgres_direct"
+    ? Boolean(binding.host && binding.port && binding.username && configured(CATALOG_SECRET_IDS.password))
+    : binding.transport === "rest_api"
+      ? Boolean(binding.baseUrl && (binding.restAuth === "none" || configured(CATALOG_SECRET_IDS.apiKey)))
+      : Boolean(
+        binding.username && binding.sshHost && binding.sshPort && binding.sshUsername
+        && binding.sshTargetHost && binding.sshTargetPort
+        && configured(CATALOG_SECRET_IDS.password)
+        && configured(CATALOG_SECRET_IDS.sshPrivateKey)
+        && configured(CATALOG_SECRET_IDS.sshKnownHosts),
+      );
   return {
-    transport: effective.transport,
-    configurationState: configurationRequired ? "configuration_required" : "ready",
-    sessionTransportSupported: effective.transport !== "ssh_tunnel",
+    transport: binding.transport,
+    configurationState: connectionComplete ? "ready" : "configuration_required",
+    sessionTransportSupported: binding.transport !== "ssh_tunnel",
   };
 }
 
@@ -145,17 +116,18 @@ export class CatalogService {
     const active = await Promise.all(workspaces.map(async (entry) => {
       const database = byWorkspace.get(entry.id);
       const { workspace } = await this.registry.readPinned(entry.id, entry.revision.commit);
-      const runtimeDatabaseBinding = yamlBinding(workspace, this.secretRoots);
       const base = database ?? {
         workspaceId: entry.id,
         engine: "postgres" as const,
-        databaseName: workspace.dwh.database,
-        schema: workspace.dwh.schema,
+        databaseName: "",
+        schema: "",
         version: 0,
         createdAt: "",
         updatedAt: "",
-        binding: runtimeDatabaseBinding,
+        binding: { transport: "postgres_direct" as const },
         connectionStatus: "untested" as const,
+        metadataContentRevision: 0,
+        preprocessingStatus: "failed" as const,
       };
       return {
         ...base,
@@ -167,7 +139,7 @@ export class CatalogService {
           blob: entry.revision.blob,
         },
         workspaceEvidence: workspaceEvidenceState(this.secretStore, workspace),
-        runtimeBinding: workspaceRuntimeState(this.secretStore, workspace, this.secretRoots),
+        runtimeBinding: database ? databaseRuntimeState(this.secretStore, database) : null,
         configured: database !== undefined,
         secrets: secretState(this.secretStore, entry.id),
       };
@@ -195,13 +167,13 @@ export class CatalogService {
   }
 
   async normalizeInput(input: DatabaseConfigurationInput): Promise<DatabaseConfigurationInput> {
-    const workspace = await this.ensureWorkspace(input.workspaceId);
+    await this.ensureWorkspace(input.workspaceId);
     if (input.binding.transport !== "rest_api") return input;
     return {
       ...input,
       binding: {
         ...input.binding,
-        restPath: workspace.diagnostics?.dwh_rest?.path ?? "/health",
+        restPath: input.binding.restPath ?? "/health",
       },
     };
   }

backend/src/catalog/types.ts

@@ -2,6 +2,7 @@ export const DATABASE_TRANSPORTS = ["postgres_direct", "rest_api", "ssh_tunnel"]
 export type DatabaseTransport = (typeof DATABASE_TRANSPORTS)[number];
 
 export type ConnectionStatus = "untested" | "reachable" | "failed";
+export type CatalogPreprocessingStatus = "running" | "succeeded" | "failed";
 
 export interface DatabaseBinding {
   transport: DatabaseTransport;
@@ -36,8 +37,23 @@ export interface WorkspaceDatabase {
   lastErrorMessage?: string;
   schemaSyncedVersion?: number;
   schemaSyncedAt?: string;
+  metadataContentRevision: number;
+  preprocessingStatus: CatalogPreprocessingStatus;
+  preprocessingInputFingerprint?: string;
+  preprocessedMetadataRevision?: number;
+  preprocessingStartedAt?: string;
+  preprocessingFinishedAt?: string;
+  preprocessingErrorCode?: string;
 }
 
+export type CatalogPreprocessingStartResult =
+  | { kind: "started"; database: WorkspaceDatabase }
+  | { kind: "not_found" | "schema_stale" | "catalog_busy" | "already_running" };
+
+export type CatalogPreprocessingClearResult =
+  | { kind: "cleared"; database: WorkspaceDatabase }
+  | { kind: "not_found" | "already_running" };
+
 export interface CatalogMetrics {
   scope: "global" | "database";
   databaseId: string | null;
@@ -196,7 +212,7 @@ export interface CatalogLogicalRelationshipCandidate {
 
 export type CatalogDatabaseMetadataDeleteTarget = "tables" | "relationships";
 export type CatalogTableMetadataDeleteTarget = "columns" | "relationships";
-export type CatalogDescriptionTarget = "tables" | "columns" | "database_columns";
+export type CatalogDescriptionTarget = "tables" | "columns" | "database" | "database_columns";
 
 export interface CatalogMetadataDeleteCounts {
   tables: number;
@@ -433,6 +449,17 @@ export interface CatalogRepository {
   list(): Promise<WorkspaceDatabase[]>;
   get(id: string): Promise<WorkspaceDatabase | undefined>;
   getByWorkspace(workspaceId: string): Promise<WorkspaceDatabase | undefined>;
+  beginPreprocessing(
+    workspaceId: string,
+    inputFingerprint: string,
+  ): Promise<CatalogPreprocessingStartResult>;
+  finishPreprocessing(
+    workspaceId: string,
+    metadataContentRevision: number,
+    inputFingerprint: string,
+    outcome: { status: "succeeded" } | { status: "failed"; errorCode: string },
+  ): Promise<WorkspaceDatabase | undefined>;
+  clearPreprocessing(workspaceId: string): Promise<CatalogPreprocessingClearResult>;
   getCatalogMetrics(databaseId?: string): Promise<CatalogMetrics | undefined>;
   create(input: DatabaseConfigurationInput): Promise<WorkspaceDatabase>;
   update(id: string, expectedVersion: number, input: DatabaseConfigurationInput): Promise<WorkspaceDatabase | undefined>;