feat: complete catalog-driven preprocessing

backend/test/catalog-repository.integration.test.ts

@@ -18,6 +18,7 @@ import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usa
 import { up as upCanonicalModelIds } from "../src/catalog/migrations/010_canonical_model_ids.js";
 import { up as upLocalSensitivityAnalysis } from "../src/catalog/migrations/011_local_sensitivity_analysis.js";
 import { up as upSensitivityReason } from "../src/catalog/migrations/012_sensitivity_reason.js";
+import { up as upCatalogPreprocessingState } from "../src/catalog/migrations/013_catalog_preprocessing_state.js";
 
 const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
 
@@ -58,6 +59,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
     await upCanonicalModelIds(db);
     await upLocalSensitivityAnalysis(db);
     await upSensitivityReason(db);
+    await upCatalogPreprocessingState(db);
     await expect(db.selectFrom("sensitiveDataSuggestionRuns")
       .select(["engine", "modelId", "policyVersion", "unknown"])
       .where("id", "=", historicalSuggestionRunId)
@@ -232,6 +234,77 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
     expect(await repository.listSyncRuns(created.id)).toEqual([
       expect.objectContaining({ id: syncRun.id, tableIds: [patients.id] }),
     ]);
+
+    const lockedDatabase = await repository.create({
+      workspaceId: "preprocessing-lock",
+      engine: "postgres",
+      databaseName: "warehouse",
+      schema: "public",
+      binding: {
+        transport: "postgres_direct", host: "lock.internal", port: 5432, username: "reader",
+      },
+    });
+    await repository.applySchemaSync(
+      lockedDatabase.id,
+      lockedDatabase.version,
+      "all",
+      [],
+      {
+        schemaVersion: 1,
+        capabilities: { tables: "available", columns: "available", relationships: "available" },
+        tables: [], columns: [], relationships: [],
+      },
+    );
+    const beforePreprocessing = (await repository.get(lockedDatabase.id))!;
+    const preprocessing = await repository.beginPreprocessing(
+      "preprocessing-lock",
+      "sha256:" + "1".repeat(64),
+    );
+    expect(preprocessing).toMatchObject({ kind: "started" });
+    await expect(db.insertInto("catalogTables").values({
+      id: randomUUID(),
+      databaseId: lockedDatabase.id,
+      name: "blocked_write",
+      sourceComment: null,
+      description: null,
+      generatedDescription: null,
+    }).execute()).rejects.toThrow("catalog preprocessing is running");
+    await expect(repository.createDescriptionGenerationRun(
+      lockedDatabase.id,
+      "all",
+      "openai/gpt-5-mini",
+      "en",
+      0,
+    )).rejects.toThrow("catalog preprocessing is running");
+    await repository.recordTest(lockedDatabase.id, lockedDatabase.version, {
+      connectionStatus: "reachable",
+      testedVersion: lockedDatabase.version,
+      lastTestedAt: "2026-01-01T00:00:00Z",
+    });
+    expect((await repository.get(lockedDatabase.id))?.metadataContentRevision)
+      .toBe(beforePreprocessing.metadataContentRevision);
+    await expect(repository.finishPreprocessing(
+      "preprocessing-lock",
+      beforePreprocessing.metadataContentRevision,
+      "sha256:" + "1".repeat(64),
+      { status: "succeeded" },
+    )).resolves.toMatchObject({
+      preprocessingStatus: "succeeded",
+      preprocessedMetadataRevision: beforePreprocessing.metadataContentRevision,
+    });
+    await db.insertInto("catalogTables").values({
+      id: randomUUID(),
+      databaseId: lockedDatabase.id,
+      name: "allowed_after_preprocessing",
+      sourceComment: null,
+      description: null,
+      generatedDescription: null,
+    }).execute();
+    await expect(repository.get(lockedDatabase.id)).resolves.toMatchObject({
+      preprocessingStatus: "failed",
+      metadataContentRevision: beforePreprocessing.metadataContentRevision + 1,
+    });
+    expect(await repository.delete(lockedDatabase.id, lockedDatabase.version)).toBe(true);
     expect(await repository.update(created.id, 1, { ...input, schema: "public" })).toMatchObject({ version: 2, schema: "public" });
     expect(await repository.delete(created.id, 2)).toBe(true);
     expect(await repository.list()).toEqual([]);
@@ -355,7 +428,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository performs scoped metadata cl
   }
 }, 60_000);
 
-test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates selected table and column descriptions", async () => {
+test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates selected and database-wide descriptions", async () => {
   const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
   const db = new Kysely<CatalogDatabase>({
     dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
@@ -449,6 +522,22 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates sel
       description: "Still curated visits",
       generatedDescription: "Generated visits",
     });
+
+    expect(await repository.consolidateGeneratedDescriptions(
+      database.id, "database", [],
+    )).toEqual({ copied: 3, skipped: 1 });
+    expect(await repository.getTable(database.id, visits.id)).toMatchObject({
+      description: "Generated visits",
+      generatedDescription: "Generated visits",
+    });
+    expect(await repository.getColumn(database.id, visits.id, id.id)).toMatchObject({
+      description: "Generated id",
+      generatedDescription: "Generated id",
+    });
+    expect(await repository.getColumn(database.id, visits.id, patientId.id)).toMatchObject({
+      description: "Keep patient reference",
+      generatedDescription: null,
+    });
   } finally {
     await db.destroy();
     await container.stop();
@@ -473,6 +562,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
     await upCanonicalModelIds(db);
     await upLocalSensitivityAnalysis(db);
     await upSensitivityReason(db);
+    await upCatalogPreprocessingState(db);
     const repository = new KyselyCatalogRepository(db);
     const firstDatabase = await repository.create({
       workspaceId: "generation-one",