feat: complete catalog sensitivity enhancements

This commit is contained in:
Codex
2026-09-04 15:11:18 +02:00
parent b891246664
commit 7b1d69a65b
72 changed files with 33866 additions and 358 deletions
@@ -1,4 +1,4 @@
import { act, render, screen, waitFor, within } from "@testing-library/react";
import { act, fireEvent, render, screen, waitFor, within } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import { http, HttpResponse } from "msw";
@@ -6,6 +6,7 @@ import { server } from "../test/msw";
import type {
CatalogColumn,
CatalogDatabase,
CatalogRelationship,
CatalogSyncRun,
CatalogTable,
DescriptionGenerationRun,
@@ -376,6 +377,84 @@ test("opens the relationship map directly from a Fleet database and restores foc
})).toHaveFocus());
});
test("refreshes the relationship KPI after building generated relationships", async () => {
const user = userEvent.setup();
const generatedRelationship: CatalogRelationship = {
id: "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee",
databaseId: "11111111-1111-4111-8111-111111111111",
constraintName: null,
sourceTableId: "ffffffff-ffff-4fff-8fff-ffffffffffff",
sourceTableName: "visits",
targetTableId: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa",
targetTableName: "patients",
updateRule: null,
deleteRule: null,
deferrable: false,
initiallyDeferred: false,
columns: [{
position: 1,
sourceColumnId: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb",
sourceColumnName: "patient_id",
targetColumnId: "cccccccc-cccc-4ccc-8ccc-cccccccccccc",
targetColumnName: "id",
}],
origin: "generated",
status: "active",
lastSyncedDatabaseVersion: null,
lastSyncedAt: null,
createdAt: "2026-08-27T10:00:00Z",
updatedAt: "2026-08-27T10:00:00Z",
};
let relationships: CatalogRelationship[] = [];
server.use(
http.get("/api/catalog/metrics", ({ request }) => HttpResponse.json({
scope: new URL(request.url).searchParams.has("databaseId") ? "database" : "global",
databaseId: new URL(request.url).searchParams.get("databaseId"),
tables: 2,
columns: 3,
sensitiveColumns: 0,
relationships: relationships.length,
descriptionTargets: 5,
describedTargets: 0,
descriptionCoverage: 0,
updatedAt: "2026-08-27T10:00:00Z",
})),
http.get(
"/api/catalog/databases/:databaseId/relationships",
() => HttpResponse.json(relationships),
),
http.post(
"/api/catalog/databases/:databaseId/relationships/rebuild-generated",
() => {
relationships = [generatedRelationship];
return HttpResponse.json({ added: 1, alreadyPresent: 0, excluded: 0, ambiguous: 0 });
},
),
);
renderPage({
rows: [makeDatabase({ connectionStatus: "reachable", testedVersion: 3 })],
presentation: "fleet",
});
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
await user.click(within(databaseRow).getByRole("button", {
name: "View relationships for Policlinico San Donato",
}));
const summary = await screen.findByRole("region", { name: "Database summary" });
const relationshipMetric = within(summary).getByText("Relationships").nextElementSibling;
expect(relationshipMetric).toHaveTextContent("0");
await user.selectOptions(
screen.getByRole("combobox", { name: "Relationship action" }),
"rebuild-generated",
);
await user.click(screen.getByRole("button", { name: "Run action" }));
expect(await screen.findByText("Build complete: 1 added, 0 already present, 0 excluded, 0 ambiguous."))
.toBeVisible();
await waitFor(() => expect(relationshipMetric).toHaveTextContent("1"));
});
test("offers catalog configuration directly on an unconfigured Fleet workspace", async () => {
const user = userEvent.setup();
renderPage({ rows: [unconfigured], presentation: "fleet" });
@@ -473,7 +552,10 @@ test("reopens database synchronization history when no run is active", async ()
renderPage({ rows: [makeDatabase()], presentation: "fleet" });
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
await user.click(within(databaseRow).getByText("Policlinico San Donato"));
await user.click(within(databaseRow).getByRole("button", {
name: "View Policlinico San Donato",
}));
await user.click(screen.getByRole("button", { name: "Sync history" }));
const drawer = await screen.findByRole("dialog", { name: "Schema synchronization" });
expect(drawer).toBeVisible();
@@ -756,6 +838,55 @@ test("keeps Fleet Generate missing descriptions behind the source-data disclosur
expect(generationStarts).toBe(0);
});
test("copies generated descriptions to every database column after explicit confirmation", async () => {
const user = userEvent.setup();
let consolidationBody: unknown;
server.use(
http.post("/api/catalog/databases/:databaseId/descriptions/consolidate", async ({ request }) => {
consolidationBody = await request.json();
return HttpResponse.json({ copied: 7, skipped: 2 });
}),
);
const database = makeDatabase();
const { client } = renderPage({ rows: [database], presentation: "fleet" });
const tablesQuery = ["catalog-tables", database.id] as const;
const firstColumnsQuery = ["catalog-columns", database.id, "table-one"] as const;
const secondColumnsQuery = ["catalog-columns", database.id, "table-two"] as const;
const unrelatedColumnsQuery = ["catalog-columns", "other-database", "table-one"] as const;
for (const queryKey of [
tablesQuery,
firstColumnsQuery,
secondColumnsQuery,
unrelatedColumnsQuery,
]) client.setQueryData(queryKey, []);
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
await user.click(within(databaseRow).getByRole("checkbox", { name: /toggle row selection/i }));
await user.selectOptions(
screen.getByRole("combobox", { name: "Batch action" }),
"consolidate-columns",
);
await user.click(screen.getByRole("button", { name: "Run action" }));
expect(screen.getByText(
"Copy generated descriptions to all columns in Policlinico San Donato?",
)).toBeVisible();
expect(screen.getByText(/replace the current Description value/i)).toBeVisible();
expect(consolidationBody).toBeUndefined();
await user.click(screen.getByRole("button", { name: "Copy to all columns" }));
await waitFor(() => expect(consolidationBody).toEqual({ target: "database_columns" }));
expect(await screen.findByText("Copied 7 column descriptions; skipped 2")).toBeVisible();
await waitFor(() => {
expect(client.getQueryState(tablesQuery)?.isInvalidated).toBe(true);
expect(client.getQueryState(firstColumnsQuery)?.isInvalidated).toBe(true);
expect(client.getQueryState(secondColumnsQuery)?.isInvalidated).toBe(true);
});
expect(client.getQueryState(unrelatedColumnsQuery)?.isInvalidated).toBe(false);
expect(screen.getByText("1 selected")).toBeVisible();
});
test("confirms generating all descriptions, supports cancel, and sends the database-wide scope", async () => {
const user = userEvent.setup();
const queuedRun = makeDescriptionGenerationRun({ scope: "all", total: 6 });
@@ -989,9 +1120,9 @@ test("disables database-wide generation while a description generation is active
await user.click(await screen.findByRole("menuitem", { name: "Generate missing descriptions" }));
await user.click(screen.getByRole("button", { name: "Generate missing descriptions" }));
expect(await screen.findByRole("dialog", { name: "Description generation" })).toBeVisible();
await user.click(screen.getByRole("button", { name: "Close description generation" }));
databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
await user.click(within(databaseRow).getByRole("checkbox", { name: /toggle row selection/i }));
await user.click(screen.getByRole("button", { name: "Actions" }));
expect(await screen.findByRole("menuitem", { name: "Generate all descriptions" }))
.toHaveAttribute("aria-disabled", "true");
@@ -1248,7 +1379,6 @@ test("uses an in-page destructive form and returns the YAML workspace to Not con
expect(deleteVersion).toBe("3");
expect(await screen.findByRole("button", { name: "Configure catalog for Policlinico San Donato" })).toBeEnabled();
expect(screen.queryByRole("button", { name: "Delete Policlinico San Donato" })).not.toBeInTheDocument();
expect(screen.getByText("Not configured")).toBeVisible();
});
test("tests only the persisted version and updates the visible connection status", async () => {
@@ -1258,7 +1388,9 @@ test("tests only the persisted version and updates the visible connection status
server.use(http.post("/api/catalog/databases/:id/test", async ({ request }) => {
testBody = await request.json();
row = makeDatabase({ version: 4, connectionStatus: "reachable", testedVersion: 4 });
return HttpResponse.json(row);
const testResponse: Partial<CatalogDatabase> = { ...row };
delete testResponse.workspaceName;
return HttpResponse.json(testResponse);
}));
renderPage({ rows: () => [row] });
@@ -1266,9 +1398,78 @@ test("tests only the persisted version and updates the visible connection status
await user.click(screen.getByRole("button", { name: "Test connection" }));
await waitFor(() => expect(testBody).toEqual({ version: 3 }));
const resultDialog = await screen.findByRole("dialog", { name: "Connection test successful" });
expect(resultDialog).toBeVisible();
expect(within(resultDialog).getByText("Policlinico San Donato is reachable.")).toBeVisible();
await user.click(within(resultDialog).getByRole("button", { name: "Done" }));
expect(within(screen.getByRole("region", { name: "Database details form" })).getByText("Reachable")).toBeVisible();
});
test("reports an unsuccessful selected connection test as a failure", async () => {
const user = userEvent.setup();
const failed = makeDatabase({
connectionStatus: "failed",
testedVersion: 3,
lastErrorMessage: "Password authentication failed",
});
const testResponse: Partial<CatalogDatabase> = { ...failed };
delete testResponse.workspaceName;
server.use(http.post("/api/catalog/databases/:id/test", () => HttpResponse.json(testResponse)));
renderPage({ rows: [makeDatabase()] });
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
await user.click(within(databaseRow).getByRole("checkbox", { name: /toggle row selection/i }));
await user.click(screen.getByRole("button", { name: "Actions" }));
await user.click(await screen.findByRole("menuitem", { name: "Test connections" }));
const resultDialog = await screen.findByRole("dialog", { name: "Connection test failed" });
expect(resultDialog).toBeVisible();
expect(within(resultDialog).getByText(/Policlinico San Donato: Password authentication failed/)).toBeVisible();
expect(screen.queryByText("1 connection tested")).not.toBeInTheDocument();
});
test("keeps Fleet synchronization available after an informative connection failure", async () => {
const user = userEvent.setup();
renderPage({
rows: [makeDatabase({
connectionStatus: "failed",
testedVersion: 3,
lastErrorCode: "connector_unavailable",
lastErrorMessage: "Password authentication failed",
})],
presentation: "fleet",
});
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
await user.click(within(databaseRow).getByRole("checkbox", { name: /toggle row selection/i }));
const actionPicker = screen.getByRole("combobox", { name: "Batch action" });
await user.selectOptions(actionPicker, "sync-all");
expect(within(actionPicker).getByRole("option", { name: "Synchronize all" })).toBeVisible();
expect(screen.getByRole("button", { name: "Run action" })).toBeEnabled();
});
test("reports a successful selected connection test as a success", async () => {
const user = userEvent.setup();
const reachable = makeDatabase({
connectionStatus: "reachable",
testedVersion: 3,
});
const testResponse: Partial<CatalogDatabase> = { ...reachable };
delete testResponse.workspaceName;
server.use(http.post("/api/catalog/databases/:id/test", () => HttpResponse.json(testResponse)));
renderPage({ rows: [makeDatabase()] });
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
await user.click(within(databaseRow).getByRole("checkbox", { name: /toggle row selection/i }));
await user.click(screen.getByRole("button", { name: "Actions" }));
await user.click(await screen.findByRole("menuitem", { name: "Test connections" }));
const resultDialog = await screen.findByRole("dialog", { name: "Connection test successful" });
expect(resultDialog).toBeVisible();
expect(within(resultDialog).getByText("Policlinico San Donato is reachable.")).toBeVisible();
});
test("retains a stale draft and requires an explicit reload", async () => {
const user = userEvent.setup();
server.use(
@@ -1442,6 +1643,147 @@ const patientIdColumn: CatalogColumn = {
updatedAt: "2026-08-27T10:00:00Z",
};
test("keeps column synchronization available when the latest connection test failed", async () => {
const user = userEvent.setup();
server.use(
http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([patientsTable])),
http.get("/api/catalog/databases/:databaseId/tables/:tableId/columns", () => HttpResponse.json([patientIdColumn])),
);
renderPage({
rows: [makeDatabase({
connectionStatus: "failed",
testedVersion: 3,
lastErrorCode: "connector_unavailable",
lastErrorMessage: "The database connector could not be reached or authenticated.",
})],
});
await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" }));
await user.click(screen.getByRole("tab", { name: "Tables" }));
await user.click(await screen.findByRole("button", { name: "View columns for patients" }));
const rows = await screen.findAllByRole("row", { name: /Patient identifier/ });
const selectableRow = rows.find((row) => within(row).queryByRole("checkbox", { name: /toggle row selection/i }));
expect(selectableRow).toBeDefined();
await user.click(within(selectableRow!).getByRole("checkbox", { name: /toggle row selection/i }));
await user.click(screen.getByRole("button", { name: "Actions" }));
expect(await screen.findByRole("menuitem", { name: "Synchronize columns for this table" }))
.not.toHaveAttribute("aria-disabled", "true");
});
test("saves a manual sensitive flag change from the Fleet columns toolbar", async () => {
const user = userEvent.setup();
const nameColumn: CatalogColumn = {
...patientIdColumn,
id: "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee",
name: "name",
ordinalPosition: 2,
primaryKeyPosition: null,
isPrimaryKey: false,
sourceComment: "Patient name",
};
const columns = [patientIdColumn, nameColumn];
const patches: Array<{ columnId: string; body: Record<string, unknown> }> = [];
let requestInFlight = false;
server.use(
http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([patientsTable])),
http.get(
"/api/catalog/databases/:databaseId/tables/:tableId/columns",
() => HttpResponse.json(columns),
),
http.patch(
"/api/catalog/databases/:databaseId/tables/:tableId/columns/:columnId",
async ({ params, request }) => {
if (requestInFlight) {
return HttpResponse.json({
code: "catalog_operation_active",
message: "Another catalog operation is in progress.",
}, { status: 409 });
}
requestInFlight = true;
const body = await request.json() as Record<string, unknown>;
await new Promise((resolve) => window.setTimeout(resolve, 20));
patches.push({ columnId: String(params.columnId), body });
requestInFlight = false;
const current = columns.find((column) => column.id === params.columnId)!;
return HttpResponse.json({ ...current, ...body, version: current.version + 1 });
},
),
);
renderPage({
rows: [makeDatabase({ connectionStatus: "reachable", testedVersion: 3 })],
presentation: "fleet",
});
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
await user.click(within(databaseRow).getByRole("button", {
name: "View tables for Policlinico San Donato",
}));
await user.click(await screen.findByRole("button", { name: "View columns for patients" }));
expect(screen.queryByRole("button", { name: "Save sensitive fields" })).not.toBeInTheDocument();
await user.click(await screen.findByRole("checkbox", { name: "Sensitive data for id" }));
await user.click(screen.getByRole("checkbox", { name: "Sensitive data for name" }));
const saveButton = screen.getByRole("button", { name: "Save sensitive fields" });
expect(saveButton).toBeVisible();
await user.click(saveButton);
await waitFor(() => expect(patches).toHaveLength(2));
expect(patches).toEqual(expect.arrayContaining([
expect.objectContaining({
columnId: patientIdColumn.id,
body: expect.objectContaining({ version: patientIdColumn.version, sensitive: true }),
}),
expect.objectContaining({
columnId: nameColumn.id,
body: expect.objectContaining({ version: nameColumn.version, sensitive: true }),
}),
]));
expect(await screen.findByText("Sensitive fields saved")).toBeVisible();
await waitFor(() => {
expect(screen.queryByRole("button", { name: "Save sensitive fields" })).not.toBeInTheDocument();
});
});
test("shows a read-only sensitivity reason in the Fleet column metadata matrix", async () => {
const user = userEvent.setup();
const sensitiveColumn: CatalogColumn = {
...patientIdColumn,
sensitive: true,
sensitivityReason: "Local assessment matched content rule pii.email.",
};
server.use(
http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([patientsTable])),
http.get(
"/api/catalog/databases/:databaseId/tables/:tableId/columns",
() => HttpResponse.json([sensitiveColumn]),
),
);
renderPage({
rows: [makeDatabase({ connectionStatus: "reachable", testedVersion: 3 })],
presentation: "fleet",
});
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
await user.click(within(databaseRow).getByRole("button", {
name: "View tables for Policlinico San Donato",
}));
await user.click(await screen.findByRole("button", { name: "View columns for patients" }));
await user.click(await screen.findByRole("button", { name: "Edit metadata for id" }));
const drawer = await screen.findByRole("dialog", { name: "Review column description" });
const matrix = within(drawer).getByRole("group", { name: "Column metadata fields" });
expect(matrix).toHaveClass("sm:grid-cols-2");
expect(within(matrix).getByLabelText("Source comment")).toHaveAttribute("readonly");
expect(within(matrix).getByLabelText("Description")).not.toHaveAttribute("readonly");
expect(within(matrix).getByLabelText("Generated description")).not.toHaveAttribute("readonly");
expect(within(matrix).getByLabelText("Sensitive motivation")).toHaveValue(
"Local assessment matched content rule pii.email.",
);
expect(within(matrix).getByLabelText("Sensitive motivation")).toHaveAttribute("readonly");
});
test("filters catalog tables as the operator types", async () => {
const user = userEvent.setup();
const mediciTable: CatalogTable = {
@@ -2050,6 +2392,98 @@ test("shows database sensitivity analysis only for a selection and rejects multi
expect(suggestionCalls).toBe(0);
});
test("opens sensitivity analysis progress while the assessment request is still running", async () => {
const user = userEvent.setup();
let analysisStarted = false;
let releaseAnalysis!: () => void;
const analysisGate = new Promise<void>((resolve) => {
releaseAnalysis = resolve;
});
const runningRun = makeSensitivityAnalysisRun({
scope: "all",
status: "running",
total: 2,
suggestedSensitive: 0,
suggestedNonSensitive: 0,
unknown: 2,
finishedAt: null,
});
const completedRun = makeSensitivityAnalysisRun({ scope: "all", total: 2 });
let activityEvents = [{
runId: runningRun.id,
sequence: 1,
level: "info",
message: "Scanning source columns",
createdAt: "2026-08-28T11:00:00Z",
}];
server.use(
http.get("/api/catalog/sensitive-data-suggestion-runs", () => (
HttpResponse.json(analysisStarted ? [runningRun] : [])
)),
http.get("/api/catalog/sensitive-data-suggestion-runs/:runId", () => (
HttpResponse.json(runningRun)
)),
http.get("/api/catalog/sensitive-data-suggestion-runs/:runId/events-list", () => (
HttpResponse.json(activityEvents)
)),
http.post("/api/catalog/databases/:databaseId/sensitive-data-suggestions", async () => {
analysisStarted = true;
await analysisGate;
return HttpResponse.json({ run: completedRun, suggestions: [] });
}),
);
renderPage({ rows: [makeDatabase()] });
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
await user.click(within(databaseRow).getByRole("checkbox", { name: /toggle row selection/i }));
const launch = user.click(screen.getByRole("button", { name: "Analyze sensitive fields" }));
await waitFor(() => expect(analysisStarted).toBe(true));
const progress = await screen.findByRole("dialog", { name: "Sensitivity analysis history" });
expect(within(progress).getByRole("heading", { name: "Starting sensitivity analysis" })).toBeVisible();
expect(within(progress).getByRole("progressbar", { name: "Sensitivity analysis is starting" }))
.toBeVisible();
expect(within(progress).getByText("Preparing sensitivity analysis…")).toBeVisible();
expect(await within(progress).findByRole("heading", { name: "Running" }, { timeout: 2_000 })).toBeVisible();
expect(await within(progress).findByText("Scanning source columns")).toBeVisible();
const eventLog = within(progress).getByRole("log", { name: "Sensitivity analysis events" });
Object.defineProperties(eventLog, {
clientHeight: { configurable: true, value: 100 },
scrollHeight: { configurable: true, get: () => activityEvents.length * 100 },
scrollTop: { configurable: true, value: 0, writable: true },
});
activityEvents = [...activityEvents, {
runId: runningRun.id,
sequence: 2,
level: "info",
message: "Running local entity detection",
createdAt: "2026-08-28T11:00:01Z",
}];
expect(await within(progress).findByText("Running local entity detection", {}, { timeout: 2_000 })).toBeVisible();
await waitFor(() => expect(eventLog.scrollTop).toBe(eventLog.scrollHeight));
eventLog.scrollTop = 0;
fireEvent.scroll(eventLog);
const jumpToLatest = await within(progress).findByRole("button", { name: "Jump to latest" });
activityEvents = [...activityEvents, {
runId: runningRun.id,
sequence: 3,
level: "info",
message: "Classifying the next table",
createdAt: "2026-08-28T11:00:02Z",
}];
expect(await within(progress).findByText("Classifying the next table", {}, { timeout: 2_000 })).toBeVisible();
expect(eventLog.scrollTop).toBe(0);
await user.click(jumpToLatest);
expect(eventLog.scrollTop).toBe(eventLog.scrollHeight);
expect(within(progress).queryByRole("button", { name: "Jump to latest" })).not.toBeInTheDocument();
releaseAnalysis();
await launch;
expect(await screen.findByRole("dialog", { name: "Sensitive field review" })).toBeVisible();
});
test("requests database-level sensitivity analysis for the only selected database", async () => {
const user = userEvent.setup();
let suggestionBody: unknown;
@@ -2304,13 +2738,14 @@ test("allows a human downgrade and saves only explicit sensitivity changes", asy
expect(patches).toHaveLength(0);
await user.click(within(review).getByRole("checkbox", { name: "Protect patients.id" }));
await user.click(within(review).getByRole("button", { name: "Save 1" }));
await user.click(within(review).getByRole("button", { name: "Save all 1 change" }));
await waitFor(() => expect(patches).toEqual([{
columnId: nameColumn.id,
body: {
version: nameColumn.version,
sensitive: false,
sensitivityReason: null,
},
}]));
expect(await screen.findByText("Saved 1 sensitive flag")).toBeVisible();