feat: complete catalog sensitivity enhancements

2026-09-04 15:11:18 +02:00
parent b891246664
commit 7b1d69a65b
72 changed files with 33866 additions and 358 deletions
@@ -7,7 +7,11 @@ import { loadConfig } from "../src/config.js";
 import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
 import { CatalogOperationCoordinator } from "../src/catalog/operation-coordinator.js";
 import type { CatalogSchemaIntrospector } from "../src/catalog/schema-introspector.js";
-import type { CatalogSyncRun, ObservedSchemaSnapshot } from "../src/catalog/types.js";
+import {
+  CatalogConnectorError,
+  type CatalogSyncRun,
+  type ObservedSchemaSnapshot,
+} from "../src/catalog/types.js";
 import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
 import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/registry.js";
 import type { WorkspaceDescriptor } from "../src/workspaces/schema.js";
@@ -164,6 +168,32 @@ test("synchronizes a full physical schema and derives primary and foreign key fl
  expect((await repository.get(database.id))?.schemaSyncedVersion).toBe(database.version);
 });

+test("attempts synchronization after a failed connection test and reports the live access failure", async () => {
+  const { app, repository, database, scan } = await setup();
+  await repository.recordTest(database.id, database.version, {
+    connectionStatus: "failed",
+    testedVersion: database.version,
+    lastTestedAt: new Date().toISOString(),
+    lastErrorCode: "connector_unavailable",
+    lastErrorMessage: "The database connector could not be reached or authenticated.",
+  });
+  scan.mockRejectedValueOnce(new CatalogConnectorError("upstream credentials must not escape"));
+
+  const started = await app.inject({
+    method: "POST",
+    url: `/catalog/databases/${database.id}/sync-runs`,
+    payload: { version: database.version, scope: "all", tableIds: [] },
+  });
+
+  expect(started.statusCode).toBe(202);
+  const failed = await waitFor(repository, started.json().id, "failed");
+  expect(scan).toHaveBeenCalledOnce();
+  expect(failed).toMatchObject({
+    errorCode: "schema_introspection_failed",
+    errorMessage: "The database schema could not be read. Check the connection and credentials, then try again.",
+  });
+});
+
 test("synchronizes columns for every catalog table when no table selection is supplied", async () => {
  const { app, repository, database, setObserved } = await setup();
  const tablesRun = await app.inject({
@@ -247,13 +277,18 @@ test("keeps generated descriptions editable and preserves them across synchroniz
  });
  const sensitiveOnly = await app.inject({
    method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}/columns/${idColumn.id}`,
-    payload: { version: editedColumn.json().version, sensitive: true },
+    payload: {
+      version: editedColumn.json().version,
+      sensitive: true,
+      sensitivityReason: "Local assessment matched content rule pii.email.",
+    },
  });
  expect(sensitiveOnly.statusCode).toBe(200);
  expect(sensitiveOnly.json()).toMatchObject({
    description: "Reviewed key",
    generatedDescription: "Generated key draft",
    sensitive: true,
+    sensitivityReason: "Local assessment matched content rule pii.email.",
  });
  const emptyPatch = await app.inject({
    method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}/columns/${idColumn.id}`,
@@ -268,6 +303,7 @@ test("keeps generated descriptions editable and preserves them across synchroniz
    description: "Reviewed key",
    generatedDescription: "Generated key draft",
    sensitive: true,
+    sensitivityReason: "Local assessment matched content rule pii.email.",
  });
 });

@@ -358,6 +394,67 @@ test("consolidates non-empty generated column descriptions and preserves curated
  expect(scan).not.toHaveBeenCalled();
 });

+test("consolidates generated descriptions for every column in a database", async () => {
+  const { app, repository, database, scan } = await setup();
+  await seedCatalog(repository, database);
+  const tables = await repository.listTables(database.id);
+  const patients = tables.find((table) => table.name === "patients")!;
+  const visits = tables.find((table) => table.name === "visits")!;
+  const patientId = (await repository.listColumns(database.id, patients.id))[0]!;
+  const visitColumns = await repository.listColumns(database.id, visits.id);
+  const visitId = visitColumns.find((column) => column.name === "id")!;
+  const visitPatientId = visitColumns.find((column) => column.name === "patient_id")!;
+  await repository.updateColumnMetadata(
+    database.id,
+    patients.id,
+    patientId.id,
+    patientId.version,
+    "Curated patient identifier",
+    "Generated patient identifier",
+  );
+  await repository.updateColumnMetadata(
+    database.id,
+    visits.id,
+    visitId.id,
+    visitId.version,
+    "Curated visit identifier",
+    "Generated visit identifier",
+  );
+  await repository.updateColumnMetadata(
+    database.id,
+    visits.id,
+    visitPatientId.id,
+    visitPatientId.version,
+    "Keep curated patient reference",
+    "",
+  );
+
+  const response = await app.inject({
+    method: "POST",
+    url: `/catalog/databases/${database.id}/descriptions/consolidate`,
+    payload: { target: "database_columns" },
+  });
+
+  expect(response.statusCode).toBe(200);
+  expect(response.json()).toEqual({ copied: 2, skipped: 1 });
+  expect(await repository.getColumn(database.id, patients.id, patientId.id)).toMatchObject({
+    description: "Generated patient identifier",
+    generatedDescription: "Generated patient identifier",
+    version: patientId.version + 2,
+  });
+  expect(await repository.getColumn(database.id, visits.id, visitId.id)).toMatchObject({
+    description: "Generated visit identifier",
+    generatedDescription: "Generated visit identifier",
+    version: visitId.version + 2,
+  });
+  expect(await repository.getColumn(database.id, visits.id, visitPatientId.id)).toMatchObject({
+    description: "Keep curated patient reference",
+    generatedDescription: "",
+    version: visitPatientId.version + 1,
+  });
+  expect(scan).not.toHaveBeenCalled();
+});
+
 test("rejects description consolidation while the Workspace Database is reserved", async () => {
  const { app, repository, database, operations } = await setup();
  await seedCatalog(repository, database);
@@ -432,9 +529,14 @@ test("strictly validates description consolidation database and target ids", asy
      url: `/catalog/databases/${database.id}/descriptions/consolidate`,
      payload: { target: "tables", targetIds: [table.id], unexpected: true },
    }),
+    app.inject({
+      method: "POST",
+      url: `/catalog/databases/${database.id}/descriptions/consolidate`,
+      payload: { target: "database_columns", targetIds: [table.id] },
+    }),
  ]);

-  expect(responses.map((response) => response.statusCode)).toEqual([400, 400, 400]);
+  expect(responses.map((response) => response.statusCode)).toEqual([400, 400, 400, 400]);
  for (const response of responses) {
    expect(response.json()).toEqual({
      code: "description_consolidation_invalid",
@@ -524,6 +626,18 @@ test("deletes relationships for selected databases without deleting their tables
  const { app, repository, database } = await setup();
  await seedCatalog(repository, database);
  const tables = await repository.listTables(database.id);
+  const context = (await repository.getLogicalRelationshipContext(database.id))!;
+  const source = context.endpoints.find((endpoint) => (
+    endpoint.tableName === "visits" && endpoint.columnName === "id"
+  ))!;
+  const target = context.endpoints.find((endpoint) => (
+    endpoint.tableName === "patients" && endpoint.columnName === "id"
+  ))!;
+  await expect(repository.insertGeneratedLogicalRelationships(database.id, [{
+    sourceColumnId: source.columnId,
+    targetColumnId: target.columnId,
+  }])).resolves.toBe(1);
+  await expect(repository.getCatalogMetrics(database.id)).resolves.toMatchObject({ relationships: 2 });

  const response = await app.inject({
    method: "POST",
@@ -532,10 +646,12 @@ test("deletes relationships for selected databases without deleting their tables
  });

  expect(response.statusCode).toBe(200);
-  expect(response.json()).toEqual({ tables: 0, columns: 0, relationships: 1 });
+  expect(response.json()).toEqual({ tables: 0, columns: 0, relationships: 2 });
  expect(await repository.listTables(database.id)).toHaveLength(2);
  expect(await repository.listColumns(database.id, tables[0]!.id)).not.toEqual([]);
  expect(await repository.listRelationships(database.id)).toEqual([]);
+  expect(await repository.listLogicalRelationships(database.id)).toEqual([]);
+  await expect(repository.getCatalogMetrics(database.id)).resolves.toMatchObject({ relationships: 0 });
  expect((await repository.get(database.id))?.schemaSyncedVersion).toBeUndefined();
 });

@@ -671,6 +787,11 @@ test("rebuilds generated relationships and returns the exact summary", async ()
  });
  expect(first.statusCode).toBe(200);
  expect(first.json()).toEqual({ added: 1, alreadyPresent: 0, excluded: 0, ambiguous: 0 });
+  const metrics = await app.inject({
+    method: "GET", url: `/catalog/metrics?databaseId=${database.id}`,
+  });
+  expect(metrics.statusCode).toBe(200);
+  expect(metrics.json()).toMatchObject({ relationships: 1 });
  const generated = (await repository.listLogicalRelationships(database.id))[0]!;

  await app.inject({