feat: add AI catalog description generation

2026-08-29 16:42:56 +02:00
parent b0afba81ca
commit 376dd5a09d
76 changed files with 14860 additions and 102 deletions
@@ -5,6 +5,7 @@ import { afterEach, expect, test, vi } from "vitest";
 import { buildApp } from "../src/app.js";
 import { loadConfig } from "../src/config.js";
 import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
+import { CatalogOperationCoordinator } from "../src/catalog/operation-coordinator.js";
 import type { CatalogSchemaIntrospector } from "../src/catalog/schema-introspector.js";
 import type { CatalogSyncRun, ObservedSchemaSnapshot } from "../src/catalog/types.js";
 import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
@@ -95,7 +96,7 @@ async function waitFor(repository: MemoryCatalogRepository, runId: string, state
  throw new Error(`Run ${runId} did not reach ${state}`);
 }

-async function setup() {
+async function setup(env: Record<string, string> = {}) {
  const secretRoot = mkdtempSync(join(tmpdir(), "catalog-schema-secret-"));
  const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-schema-runtime-"));
  roots.push(secretRoot, runtimeRoot);
@@ -116,21 +117,23 @@ async function setup() {
    return structuredClone(observed);
  });
  const introspector: CatalogSchemaIntrospector = { scan };
+  const operations = new CatalogOperationCoordinator();
  const registry = {
    list: vi.fn(async () => [revision]),
    listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]),
    read: vi.fn(async () => ({ workspace, revision })),
  } as unknown as WorkspaceRegistry;
-  const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test" }), {
+  const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test", ...env }), {
    thtRunner: {} as never,
    workspaceRegistry: registry,
    workspaceSecretStore: new WorkspaceSecretStore({ root: secretRoot, runtimeRoot, installationId: "test" }),
    catalogRepository: repository,
    catalogSchemaIntrospector: introspector,
+    catalogOperationCoordinator: operations,
    workspaceDiagnoser: vi.fn(),
  });
  return {
-    app, repository, database: (await repository.get(created.id))!, scan,
+    app, repository, database: (await repository.get(created.id))!, scan, operations,
    setObserved(next: ObservedSchemaSnapshot) { observed = next; },
  };
 }
@@ -243,6 +246,207 @@ test("keeps generated descriptions editable and preserves them across synchroniz
  expect(await repository.getColumn(database.id, patients.id, idColumn.id)).toMatchObject({ description: "Reviewed key", generatedDescription: "Generated key draft" });
 });

+test("consolidates non-empty generated table descriptions and reports skipped selections", async () => {
+  const { app, repository, database, scan } = await setup();
+  await seedCatalog(repository, database);
+  const tables = await repository.listTables(database.id);
+  const patients = tables.find((table) => table.name === "patients")!;
+  const visits = tables.find((table) => table.name === "visits")!;
+  await repository.updateTableMetadata(
+    database.id,
+    patients.id,
+    patients.version,
+    "Curated patients",
+    "Generated patients",
+  );
+  await repository.updateTableMetadata(
+    database.id,
+    visits.id,
+    visits.version,
+    "Keep curated visits",
+    null,
+  );
+
+  const response = await app.inject({
+    method: "POST",
+    url: `/catalog/databases/${database.id}/descriptions/consolidate`,
+    payload: { target: "tables", targetIds: [patients.id, visits.id] },
+  });
+
+  expect(response.statusCode).toBe(200);
+  expect(response.json()).toEqual({ copied: 1, skipped: 1 });
+  expect(await repository.getTable(database.id, patients.id)).toMatchObject({
+    description: "Generated patients",
+    generatedDescription: "Generated patients",
+    version: patients.version + 2,
+  });
+  expect(await repository.getTable(database.id, visits.id)).toMatchObject({
+    description: "Keep curated visits",
+    generatedDescription: null,
+    version: visits.version + 1,
+  });
+  expect(scan).not.toHaveBeenCalled();
+});
+
+test("consolidates non-empty generated column descriptions and preserves curated text for empty proposals", async () => {
+  const { app, repository, database, scan } = await setup();
+  await seedCatalog(repository, database);
+  const visits = (await repository.listTables(database.id)).find((table) => table.name === "visits")!;
+  const columns = await repository.listColumns(database.id, visits.id);
+  const id = columns.find((column) => column.name === "id")!;
+  const patientId = columns.find((column) => column.name === "patient_id")!;
+  await repository.updateColumnMetadata(
+    database.id,
+    visits.id,
+    id.id,
+    id.version,
+    "Curated visit identifier",
+    "Generated visit identifier",
+  );
+  await repository.updateColumnMetadata(
+    database.id,
+    visits.id,
+    patientId.id,
+    patientId.version,
+    "Keep curated patient reference",
+    "",
+  );
+
+  const response = await app.inject({
+    method: "POST",
+    url: `/catalog/databases/${database.id}/descriptions/consolidate`,
+    payload: { target: "columns", targetIds: [id.id, patientId.id] },
+  });
+
+  expect(response.statusCode).toBe(200);
+  expect(response.json()).toEqual({ copied: 1, skipped: 1 });
+  expect(await repository.getColumn(database.id, visits.id, id.id)).toMatchObject({
+    description: "Generated visit identifier",
+    generatedDescription: "Generated visit identifier",
+    version: id.version + 2,
+  });
+  expect(await repository.getColumn(database.id, visits.id, patientId.id)).toMatchObject({
+    description: "Keep curated patient reference",
+    generatedDescription: "",
+    version: patientId.version + 1,
+  });
+  expect(scan).not.toHaveBeenCalled();
+});
+
+test("rejects description consolidation while the Workspace Database is reserved", async () => {
+  const { app, repository, database, operations } = await setup();
+  await seedCatalog(repository, database);
+  const table = (await repository.listTables(database.id))[0]!;
+  const edited = await repository.updateTableMetadata(
+    database.id,
+    table.id,
+    table.version,
+    "Existing curated text",
+    "Generated text",
+  );
+  const release = operations.reserve(database.id);
+  try {
+    const response = await app.inject({
+      method: "POST",
+      url: `/catalog/databases/${database.id}/descriptions/consolidate`,
+      payload: { target: "tables", targetIds: [table.id] },
+    });
+
+    expect(response.statusCode).toBe(409);
+    expect(response.json()).toEqual({
+      code: "database_operation_in_progress",
+      message: "A database operation is already in progress.",
+    });
+    expect(await repository.getTable(database.id, table.id)).toMatchObject({
+      description: "Existing curated text",
+      generatedDescription: "Generated text",
+      version: edited!.version,
+    });
+  } finally {
+    release();
+  }
+});
+
+test("requires database.manage for description consolidation", async () => {
+  const { app, repository, database } = await setup({ AUTH_MODE: "upstream" });
+  await seedCatalog(repository, database);
+  const table = (await repository.listTables(database.id))[0]!;
+  const response = await app.inject({
+    method: "POST",
+    url: `/catalog/databases/${database.id}/descriptions/consolidate`,
+    headers: {
+      "x-thoth-principal-issuer": "portal",
+      "x-thoth-principal-subject": "catalog-reader",
+      "x-thoth-is-admin": "0",
+    },
+    payload: { target: "tables", targetIds: [table.id] },
+  });
+
+  expect(response.statusCode).toBe(403);
+  expect(response.json()).toEqual({ code: "auth_forbidden", error: "This operation is not permitted" });
+});
+
+test("strictly validates description consolidation database and target ids", async () => {
+  const { app, repository, database } = await setup();
+  await seedCatalog(repository, database);
+  const table = (await repository.listTables(database.id))[0]!;
+
+  const responses = await Promise.all([
+    app.inject({
+      method: "POST",
+      url: "/catalog/databases/not-a-uuid/descriptions/consolidate",
+      payload: { target: "tables", targetIds: [table.id] },
+    }),
+    app.inject({
+      method: "POST",
+      url: `/catalog/databases/${database.id}/descriptions/consolidate`,
+      payload: { target: "tables", targetIds: ["not-a-uuid"] },
+    }),
+    app.inject({
+      method: "POST",
+      url: `/catalog/databases/${database.id}/descriptions/consolidate`,
+      payload: { target: "tables", targetIds: [table.id], unexpected: true },
+    }),
+  ]);
+
+  expect(responses.map((response) => response.statusCode)).toEqual([400, 400, 400]);
+  for (const response of responses) {
+    expect(response.json()).toEqual({
+      code: "description_consolidation_invalid",
+      message: "Description consolidation request is invalid.",
+    });
+  }
+});
+
+test("rejects a missing consolidation target without copying valid selections", async () => {
+  const { app, repository, database } = await setup();
+  await seedCatalog(repository, database);
+  const table = (await repository.listTables(database.id))[0]!;
+  const edited = await repository.updateTableMetadata(
+    database.id,
+    table.id,
+    table.version,
+    "Existing curated text",
+    "Generated text",
+  );
+
+  const response = await app.inject({
+    method: "POST",
+    url: `/catalog/databases/${database.id}/descriptions/consolidate`,
+    payload: {
+      target: "tables",
+      targetIds: [table.id, "99999999-9999-4999-8999-999999999999"],
+    },
+  });
+
+  expect(response.statusCode).toBe(404);
+  expect(await repository.getTable(database.id, table.id)).toMatchObject({
+    description: "Existing curated text",
+    generatedDescription: "Generated text",
+    version: edited!.version,
+  });
+});
+
 test("waits for confirmation and rescans before applying destructive changes", async () => {
  const { app, repository, database, scan, setObserved } = await setup();
  const first = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });