fix: scope and batch sensitive suggestions
This commit is contained in:
+3
-1
@@ -373,7 +373,9 @@ per le nuove colonne.
|
|||||||
|
|
||||||
**Sensitive Data Policy** — La regola che applica il Sensitive Data Flag ai Catalog Sample:
|
**Sensitive Data Policy** — La regola che applica il Sensitive Data Flag ai Catalog Sample:
|
||||||
valori sintetici per una colonna protetta, valori reali per una colonna non protetta. L'AI può
|
valori sintetici per una colonna protetta, valori reali per una colonna non protetta. L'AI può
|
||||||
suggerire il flag dai soli metadati tecnici, ma soltanto l'utente lo imposta.
|
suggerire il flag dai soli metadati tecnici di un database, delle tabelle o delle colonne
|
||||||
|
esplicitamente selezionate; le richieste ampie vengono divise in batch bounded, ma soltanto
|
||||||
|
l'utente imposta i flag dopo aver rivisto la proposta completa.
|
||||||
_Avoid_: PII filter, sample filter
|
_Avoid_: PII filter, sample filter
|
||||||
|
|
||||||
**Introspection Capability** — Una categoria di struttura fisica che una Database Binding
|
**Introspection Capability** — Una categoria di struttura fisica che una Database Binding
|
||||||
|
|||||||
+4
-1
@@ -111,7 +111,10 @@ deferred to their dedicated slices.
|
|||||||
|
|
||||||
AI Description Generation uses the catalog's human-owned Sensitive Data Flag. The flag defaults to
|
AI Description Generation uses the catalog's human-owned Sensitive Data Flag. The flag defaults to
|
||||||
`false`, including for newly synchronized columns. An administrator may request an AI proposal based
|
`false`, including for newly synchronized columns. An administrator may request an AI proposal based
|
||||||
only on structural metadata, but it remains an unsaved draft until the human reviews and saves it.
|
only on structural metadata for one selected database, selected tables, or selected columns. The
|
||||||
|
backend divides large scopes into deterministic model requests of at most ten columns, also bounded
|
||||||
|
by helper message size, and combines their results, but the proposal remains an unsaved draft until
|
||||||
|
the human reviews and saves it.
|
||||||
For unprotected columns, up to five source rows and five representative non-null values may be sent
|
For unprotected columns, up to five source rows and five representative non-null values may be sent
|
||||||
transiently to the configured model provider. Protected columns are omitted from source reads and
|
transiently to the configured model provider. Protected columns are omitted from source reads and
|
||||||
replaced in the prompt by deterministic plausible values derived only from their metadata. Existing
|
replaced in the prompt by deterministic plausible values derived only from their metadata. Existing
|
||||||
|
|||||||
@@ -1,28 +1,74 @@
|
|||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
import type { MetadataGenerationModels } from "./metadata-generation-models.js";
|
import type { MetadataGenerationModels } from "./metadata-generation-models.js";
|
||||||
import type { ModelCompleter } from "./model-completer.js";
|
import type { ModelCompleter, ModelCompletionMessage } from "./model-completer.js";
|
||||||
import type { CatalogRepository } from "./types.js";
|
import type { CatalogColumn, CatalogRepository, CatalogTable } from "./types.js";
|
||||||
|
|
||||||
const MAX_COLUMNS = 10_000;
|
// The helper accepts at most 64 KiB per message. Keep the same safety margin used by
|
||||||
|
// Description Generation so UTF-8 structural metadata never reaches that hard limit.
|
||||||
|
const MAX_USER_MESSAGE_BYTES = 60 * 1024;
|
||||||
|
// Preserve ThothAI's proven completion granularity: small batches keep generation time and
|
||||||
|
// structured-output accuracy predictable even when the helper byte limit would allow much more.
|
||||||
|
const MAX_COLUMNS_PER_BATCH = 10;
|
||||||
const responseSchema = z.object({
|
const responseSchema = z.object({
|
||||||
suggestions: z.array(z.object({
|
suggestions: z.array(z.object({
|
||||||
columnId: z.uuid(),
|
columnId: z.uuid(),
|
||||||
sensitive: z.boolean(),
|
sensitive: z.boolean(),
|
||||||
}).strict()).max(MAX_COLUMNS),
|
}).strict()),
|
||||||
}).strict();
|
}).strict();
|
||||||
|
|
||||||
|
export type SensitiveDataSuggestionScope = "all" | "selected_tables" | "selected_columns";
|
||||||
|
|
||||||
|
interface StructuralColumn {
|
||||||
|
columnId: string;
|
||||||
|
tableId: string;
|
||||||
|
table: string;
|
||||||
|
column: string;
|
||||||
|
dataType: string;
|
||||||
|
nullable: boolean;
|
||||||
|
primaryKey: boolean;
|
||||||
|
foreignKey: boolean;
|
||||||
|
version: number;
|
||||||
|
currentSensitive: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
export interface SensitiveDataSuggestion {
|
export interface SensitiveDataSuggestion {
|
||||||
columnId: string;
|
columnId: string;
|
||||||
|
tableId: string;
|
||||||
|
tableName: string;
|
||||||
|
columnName: string;
|
||||||
|
version: number;
|
||||||
|
currentSensitive: boolean;
|
||||||
sensitive: boolean;
|
sensitive: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class SensitiveDataSuggestionTargetNotFoundError extends Error {
|
export class SensitiveDataSuggestionTargetNotFoundError extends Error {
|
||||||
constructor() {
|
constructor(readonly target: "database" | "table" | "column") {
|
||||||
super("database not found");
|
super(`${target} not found`);
|
||||||
this.name = "SensitiveDataSuggestionTargetNotFoundError";
|
this.name = "SensitiveDataSuggestionTargetNotFoundError";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export class SensitiveDataSuggestionDuplicateTargetIdsError extends Error {
|
||||||
|
constructor() {
|
||||||
|
super("sensitive-data suggestion target IDs must be unique");
|
||||||
|
this.name = "SensitiveDataSuggestionDuplicateTargetIdsError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class SensitiveDataSuggestionNoEligibleColumnsError extends Error {
|
||||||
|
constructor(readonly scope: SensitiveDataSuggestionScope) {
|
||||||
|
super("selected scope has no catalog columns");
|
||||||
|
this.name = "SensitiveDataSuggestionNoEligibleColumnsError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class SensitiveDataSuggestionPayloadTooLargeError extends Error {
|
||||||
|
constructor() {
|
||||||
|
super("sensitive-data suggestion structural metadata is too large");
|
||||||
|
this.name = "SensitiveDataSuggestionPayloadTooLargeError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export class SensitiveDataSuggestionInvalidResponseError extends Error {
|
export class SensitiveDataSuggestionInvalidResponseError extends Error {
|
||||||
constructor() {
|
constructor() {
|
||||||
super("sensitive-data suggestion response is invalid");
|
super("sensitive-data suggestion response is invalid");
|
||||||
@@ -30,42 +76,68 @@ export class SensitiveDataSuggestionInvalidResponseError extends Error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export class SensitiveDataSuggester {
|
function userContent(
|
||||||
constructor(
|
database: { databaseName: string; schema: string },
|
||||||
private readonly repository: CatalogRepository,
|
columns: readonly StructuralColumn[],
|
||||||
private readonly models: MetadataGenerationModels,
|
): string {
|
||||||
private readonly completer: ModelCompleter,
|
return JSON.stringify({
|
||||||
) {}
|
database: database.databaseName,
|
||||||
|
schema: database.schema,
|
||||||
|
columns: columns.map((column) => ({
|
||||||
|
columnId: column.columnId,
|
||||||
|
table: column.table,
|
||||||
|
column: column.column,
|
||||||
|
dataType: column.dataType,
|
||||||
|
nullable: column.nullable,
|
||||||
|
primaryKey: column.primaryKey,
|
||||||
|
foreignKey: column.foreignKey,
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
async suggest(
|
function batchesFor(
|
||||||
databaseId: string,
|
database: { databaseName: string; schema: string },
|
||||||
modelId: string,
|
columns: readonly StructuralColumn[],
|
||||||
signal: AbortSignal,
|
): StructuralColumn[][] {
|
||||||
): Promise<readonly SensitiveDataSuggestion[]> {
|
const batches: StructuralColumn[][] = [];
|
||||||
const database = await this.repository.get(databaseId);
|
let current: StructuralColumn[] = [];
|
||||||
if (!database) throw new SensitiveDataSuggestionTargetNotFoundError();
|
for (const column of columns) {
|
||||||
|
if (current.length === MAX_COLUMNS_PER_BATCH) {
|
||||||
|
batches.push(current);
|
||||||
|
current = [];
|
||||||
|
}
|
||||||
|
const candidate = [...current, column];
|
||||||
|
if (Buffer.byteLength(userContent(database, candidate), "utf8") <= MAX_USER_MESSAGE_BYTES) {
|
||||||
|
current = candidate;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (current.length === 0) throw new SensitiveDataSuggestionPayloadTooLargeError();
|
||||||
|
batches.push(current);
|
||||||
|
current = [column];
|
||||||
|
if (Buffer.byteLength(userContent(database, current), "utf8") > MAX_USER_MESSAGE_BYTES) {
|
||||||
|
throw new SensitiveDataSuggestionPayloadTooLargeError();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (current.length > 0) batches.push(current);
|
||||||
|
return batches;
|
||||||
|
}
|
||||||
|
|
||||||
const tables = await this.repository.listTables(databaseId);
|
function structuralColumn(table: CatalogTable, column: CatalogColumn): StructuralColumn {
|
||||||
const columns = (await Promise.all(tables.map(async (table) => ({
|
return {
|
||||||
table,
|
|
||||||
columns: await this.repository.listColumns(databaseId, table.id),
|
|
||||||
})))).flatMap(({ table, columns: tableColumns }) => tableColumns.map((column) => ({
|
|
||||||
columnId: column.id,
|
columnId: column.id,
|
||||||
|
tableId: table.id,
|
||||||
table: table.name,
|
table: table.name,
|
||||||
column: column.name,
|
column: column.name,
|
||||||
dataType: column.dataType,
|
dataType: column.dataType,
|
||||||
nullable: column.isNullable,
|
nullable: column.isNullable,
|
||||||
primaryKey: column.isPrimaryKey,
|
primaryKey: column.isPrimaryKey,
|
||||||
foreignKey: column.isForeignKey,
|
foreignKey: column.isForeignKey,
|
||||||
})));
|
version: column.version,
|
||||||
if (columns.length === 0) return [];
|
currentSensitive: column.sensitive,
|
||||||
if (columns.length > MAX_COLUMNS) throw new SensitiveDataSuggestionInvalidResponseError();
|
};
|
||||||
|
}
|
||||||
|
|
||||||
const content = await this.completer.complete({
|
const systemMessage: ModelCompletionMessage = {
|
||||||
model: this.models.resolve(modelId),
|
|
||||||
signal,
|
|
||||||
messages: [
|
|
||||||
{
|
|
||||||
role: "system",
|
role: "system",
|
||||||
content: [
|
content: [
|
||||||
"Classify whether each database column is likely to contain sensitive source values.",
|
"Classify whether each database column is likely to contain sensitive source values.",
|
||||||
@@ -73,31 +145,95 @@ export class SensitiveDataSuggester {
|
|||||||
'{"suggestions":[{"columnId":"uuid","sensitive":true}]}',
|
'{"suggestions":[{"columnId":"uuid","sensitive":true}]}',
|
||||||
"Return every supplied column exactly once. Do not add explanations or markdown.",
|
"Return every supplied column exactly once. Do not add explanations or markdown.",
|
||||||
].join("\n"),
|
].join("\n"),
|
||||||
},
|
};
|
||||||
{
|
|
||||||
role: "user",
|
|
||||||
content: JSON.stringify({
|
|
||||||
database: database.databaseName,
|
|
||||||
schema: database.schema,
|
|
||||||
columns,
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
|
export class SensitiveDataSuggester {
|
||||||
|
constructor(
|
||||||
|
private readonly repository: CatalogRepository,
|
||||||
|
private readonly models: MetadataGenerationModels,
|
||||||
|
private readonly completer: ModelCompleter,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
private async selectColumns(
|
||||||
|
databaseId: string,
|
||||||
|
scope: SensitiveDataSuggestionScope,
|
||||||
|
targetIds: readonly string[],
|
||||||
|
): Promise<StructuralColumn[]> {
|
||||||
|
if (new Set(targetIds).size !== targetIds.length) {
|
||||||
|
throw new SensitiveDataSuggestionDuplicateTargetIdsError();
|
||||||
|
}
|
||||||
|
const tables = await this.repository.listTables(databaseId);
|
||||||
|
const tableIds = new Set(targetIds);
|
||||||
|
const selectedTables = scope === "selected_tables"
|
||||||
|
? tables.filter((table) => tableIds.has(table.id))
|
||||||
|
: tables;
|
||||||
|
if (scope === "selected_tables" && selectedTables.length !== targetIds.length) {
|
||||||
|
throw new SensitiveDataSuggestionTargetNotFoundError("table");
|
||||||
|
}
|
||||||
|
|
||||||
|
const columns = (await Promise.all(selectedTables.map(async (table) => (
|
||||||
|
(await this.repository.listColumns(databaseId, table.id)).map((column) => (
|
||||||
|
structuralColumn(table, column)
|
||||||
|
))
|
||||||
|
)))).flat();
|
||||||
|
const columnIds = new Set(targetIds);
|
||||||
|
const selectedColumns = scope === "selected_columns"
|
||||||
|
? columns.filter((column) => columnIds.has(column.columnId))
|
||||||
|
: columns;
|
||||||
|
if (scope === "selected_columns" && selectedColumns.length !== targetIds.length) {
|
||||||
|
throw new SensitiveDataSuggestionTargetNotFoundError("column");
|
||||||
|
}
|
||||||
|
if (selectedColumns.length === 0) {
|
||||||
|
throw new SensitiveDataSuggestionNoEligibleColumnsError(scope);
|
||||||
|
}
|
||||||
|
return selectedColumns;
|
||||||
|
}
|
||||||
|
|
||||||
|
async suggest(
|
||||||
|
databaseId: string,
|
||||||
|
modelId: string,
|
||||||
|
scope: SensitiveDataSuggestionScope,
|
||||||
|
targetIds: readonly string[],
|
||||||
|
signal: AbortSignal,
|
||||||
|
): Promise<readonly SensitiveDataSuggestion[]> {
|
||||||
|
const database = await this.repository.get(databaseId);
|
||||||
|
if (!database) throw new SensitiveDataSuggestionTargetNotFoundError("database");
|
||||||
|
const columns = await this.selectColumns(databaseId, scope, targetIds);
|
||||||
|
const model = this.models.resolve(modelId);
|
||||||
|
const suggestions: SensitiveDataSuggestion[] = [];
|
||||||
|
|
||||||
|
for (const batch of batchesFor(database, columns)) {
|
||||||
|
let received: Map<string, { columnId: string; sensitive: boolean }> | undefined;
|
||||||
|
for (let attempt = 0; attempt < 2 && !received; attempt += 1) {
|
||||||
|
const content = await this.completer.complete({
|
||||||
|
model,
|
||||||
|
signal,
|
||||||
|
messages: [systemMessage, { role: "user", content: userContent(database, batch) }],
|
||||||
|
});
|
||||||
try {
|
try {
|
||||||
const parsed = responseSchema.parse(JSON.parse(content));
|
const parsed = responseSchema.parse(JSON.parse(content));
|
||||||
const expected = new Set(columns.map((column) => column.columnId));
|
const expected = new Set(batch.map((column) => column.columnId));
|
||||||
const received = new Set(parsed.suggestions.map((suggestion) => suggestion.columnId));
|
const candidate = new Map(parsed.suggestions.map((suggestion) => [suggestion.columnId, suggestion]));
|
||||||
if (received.size !== parsed.suggestions.length
|
if (candidate.size !== parsed.suggestions.length
|
||||||
|| received.size !== expected.size
|
|| candidate.size !== expected.size
|
||||||
|| [...received].some((columnId) => !expected.has(columnId))) {
|
|| [...candidate.keys()].some((columnId) => !expected.has(columnId))) {
|
||||||
throw new SensitiveDataSuggestionInvalidResponseError();
|
throw new SensitiveDataSuggestionInvalidResponseError();
|
||||||
}
|
}
|
||||||
return parsed.suggestions;
|
received = candidate;
|
||||||
} catch (error) {
|
} catch {
|
||||||
if (error instanceof SensitiveDataSuggestionInvalidResponseError) throw error;
|
if (attempt === 1) throw new SensitiveDataSuggestionInvalidResponseError();
|
||||||
throw new SensitiveDataSuggestionInvalidResponseError();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
suggestions.push(...batch.map((column) => ({
|
||||||
|
columnId: column.columnId,
|
||||||
|
tableId: column.tableId,
|
||||||
|
tableName: column.table,
|
||||||
|
columnName: column.column,
|
||||||
|
version: column.version,
|
||||||
|
currentSensitive: column.currentSensitive,
|
||||||
|
sensitive: received!.get(column.columnId)!.sensitive,
|
||||||
|
})));
|
||||||
|
}
|
||||||
|
return suggestions;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,7 +14,10 @@ import { MetadataGenerationModelUnavailableError } from "../catalog/metadata-gen
|
|||||||
import { ModelCompletionProviderError } from "../catalog/model-completer.js";
|
import { ModelCompletionProviderError } from "../catalog/model-completer.js";
|
||||||
import {
|
import {
|
||||||
SensitiveDataSuggester,
|
SensitiveDataSuggester,
|
||||||
|
SensitiveDataSuggestionDuplicateTargetIdsError,
|
||||||
SensitiveDataSuggestionInvalidResponseError,
|
SensitiveDataSuggestionInvalidResponseError,
|
||||||
|
SensitiveDataSuggestionNoEligibleColumnsError,
|
||||||
|
SensitiveDataSuggestionPayloadTooLargeError,
|
||||||
SensitiveDataSuggestionTargetNotFoundError,
|
SensitiveDataSuggestionTargetNotFoundError,
|
||||||
} from "../catalog/sensitive-data-suggester.js";
|
} from "../catalog/sensitive-data-suggester.js";
|
||||||
import {
|
import {
|
||||||
@@ -28,8 +31,20 @@ import {
|
|||||||
|
|
||||||
const idSchema = z.uuid();
|
const idSchema = z.uuid();
|
||||||
const modelIdSchema = z.string().regex(/^[a-z][a-z0-9._-]{0,63}$/);
|
const modelIdSchema = z.string().regex(/^[a-z][a-z0-9._-]{0,63}$/);
|
||||||
const suggestionSchema = z.object({ modelId: modelIdSchema }).strict();
|
|
||||||
const selectedTargetIdsSchema = z.array(idSchema).min(1);
|
const selectedTargetIdsSchema = z.array(idSchema).min(1);
|
||||||
|
const suggestionSchema = z.discriminatedUnion("scope", [
|
||||||
|
z.object({ modelId: modelIdSchema, scope: z.literal("all") }).strict(),
|
||||||
|
z.object({
|
||||||
|
modelId: modelIdSchema,
|
||||||
|
scope: z.literal("selected_tables"),
|
||||||
|
targetIds: selectedTargetIdsSchema,
|
||||||
|
}).strict(),
|
||||||
|
z.object({
|
||||||
|
modelId: modelIdSchema,
|
||||||
|
scope: z.literal("selected_columns"),
|
||||||
|
targetIds: selectedTargetIdsSchema,
|
||||||
|
}).strict(),
|
||||||
|
]);
|
||||||
const startSchema = z.discriminatedUnion("scope", [
|
const startSchema = z.discriminatedUnion("scope", [
|
||||||
z.object({
|
z.object({
|
||||||
modelId: modelIdSchema,
|
modelId: modelIdSchema,
|
||||||
@@ -123,19 +138,6 @@ function safeError(reply: FastifyReply, error: unknown) {
|
|||||||
message: "The selected metadata-generation model is unavailable.",
|
message: "The selected metadata-generation model is unavailable.",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
if (error instanceof SensitiveDataSuggestionTargetNotFoundError) {
|
|
||||||
return reply.code(404).send({
|
|
||||||
code: "database_not_found",
|
|
||||||
message: "Database configuration was not found.",
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if (error instanceof SensitiveDataSuggestionInvalidResponseError
|
|
||||||
|| error instanceof ModelCompletionProviderError) {
|
|
||||||
return reply.code(502).send({
|
|
||||||
code: "sensitive_data_suggestion_failed",
|
|
||||||
message: "Sensitive-data suggestions could not be prepared.",
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if (error instanceof DescriptionGenerationDuplicateTargetIdsError) {
|
if (error instanceof DescriptionGenerationDuplicateTargetIdsError) {
|
||||||
return reply.code(400).send({
|
return reply.code(400).send({
|
||||||
code: "description_generation_target_ids_duplicate",
|
code: "description_generation_target_ids_duplicate",
|
||||||
@@ -190,6 +192,74 @@ function safeError(reply: FastifyReply, error: unknown) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function safeSuggestionError(reply: FastifyReply, error: unknown) {
|
||||||
|
if (error instanceof CatalogUnavailableError) {
|
||||||
|
return reply.code(503).send({
|
||||||
|
code: "catalog_unavailable",
|
||||||
|
message: "The database catalog is unavailable, so no sensitive-field suggestions were prepared.",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (error instanceof MetadataGenerationModelUnavailableError) {
|
||||||
|
return reply.code(409).send({
|
||||||
|
code: "metadata_generation_model_unavailable",
|
||||||
|
message: "The selected metadata-generation model is unavailable.",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (error instanceof SensitiveDataSuggestionTargetNotFoundError) {
|
||||||
|
const code = error.target === "database"
|
||||||
|
? "database_not_found"
|
||||||
|
: error.target === "table"
|
||||||
|
? "catalog_table_not_found"
|
||||||
|
: "catalog_column_not_found";
|
||||||
|
const message = error.target === "database"
|
||||||
|
? "The database configuration was not found."
|
||||||
|
: error.target === "table"
|
||||||
|
? "One or more selected Catalog Tables were not found in this database."
|
||||||
|
: "One or more selected Catalog Columns were not found in this database.";
|
||||||
|
return reply.code(404).send({ code, message });
|
||||||
|
}
|
||||||
|
if (error instanceof SensitiveDataSuggestionDuplicateTargetIdsError) {
|
||||||
|
return reply.code(400).send({
|
||||||
|
code: "sensitive_data_suggestion_target_ids_duplicate",
|
||||||
|
message: "Each selected table or column must appear only once.",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (error instanceof SensitiveDataSuggestionNoEligibleColumnsError) {
|
||||||
|
return reply.code(409).send({
|
||||||
|
code: "sensitive_data_suggestion_no_columns",
|
||||||
|
message: "The selected scope contains no Catalog Columns to classify.",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (error instanceof SensitiveDataSuggestionPayloadTooLargeError) {
|
||||||
|
return reply.code(413).send({
|
||||||
|
code: "sensitive_data_suggestion_payload_too_large",
|
||||||
|
message: "The selected structural metadata cannot be divided into safe LLM requests.",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (error instanceof SensitiveDataSuggestionInvalidResponseError) {
|
||||||
|
return reply.code(502).send({
|
||||||
|
code: "sensitive_data_suggestion_invalid_response",
|
||||||
|
message: "The LLM returned an incomplete or invalid classification. No suggestions were applied.",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (error instanceof ModelCompletionProviderError) {
|
||||||
|
return reply.code(502).send({
|
||||||
|
code: "sensitive_data_suggestion_provider_unavailable",
|
||||||
|
message: "The selected LLM service could not complete the request. No suggestions were applied.",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (error instanceof z.ZodError) {
|
||||||
|
return reply.code(400).send({
|
||||||
|
code: "sensitive_data_suggestion_request_invalid",
|
||||||
|
message: "Choose a database, one or more tables, or one or more columns to classify.",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return reply.code(500).send({
|
||||||
|
code: "sensitive_data_suggestion_failed",
|
||||||
|
message: "Sensitive-field suggestions failed before review. No changes were applied.",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export function catalogDescriptionGenerationRoutes(
|
export function catalogDescriptionGenerationRoutes(
|
||||||
app: FastifyInstance,
|
app: FastifyInstance,
|
||||||
deps: {
|
deps: {
|
||||||
@@ -206,11 +276,13 @@ export function catalogDescriptionGenerationRoutes(
|
|||||||
const suggestions = await deps.sensitiveDataSuggester.suggest(
|
const suggestions = await deps.sensitiveDataSuggester.suggest(
|
||||||
databaseId,
|
databaseId,
|
||||||
input.modelId,
|
input.modelId,
|
||||||
|
input.scope,
|
||||||
|
"targetIds" in input ? input.targetIds : [],
|
||||||
new AbortController().signal,
|
new AbortController().signal,
|
||||||
);
|
);
|
||||||
return { suggestions };
|
return { suggestions };
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
return safeError(reply, error);
|
return safeSuggestionError(reply, error);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -16,10 +16,12 @@ import {
|
|||||||
const idSchema = z.uuid();
|
const idSchema = z.uuid();
|
||||||
const metadataSchema = z.object({
|
const metadataSchema = z.object({
|
||||||
version: z.number().int().positive(),
|
version: z.number().int().positive(),
|
||||||
description: z.string().max(20_000).nullable(),
|
description: z.string().max(20_000).nullable().optional(),
|
||||||
generatedDescription: z.string().max(20_000).nullable(),
|
generatedDescription: z.string().max(20_000).nullable().optional(),
|
||||||
sensitive: z.boolean().optional(),
|
sensitive: z.boolean().optional(),
|
||||||
}).strict();
|
}).strict().refine((value) => (
|
||||||
|
"description" in value || "generatedDescription" in value || "sensitive" in value
|
||||||
|
));
|
||||||
const createRunSchema = z.object({
|
const createRunSchema = z.object({
|
||||||
version: z.number().int().positive(),
|
version: z.number().int().positive(),
|
||||||
scope: z.enum(["tables", "columns", "relationships", "all"]),
|
scope: z.enum(["tables", "columns", "relationships", "all"]),
|
||||||
@@ -116,8 +118,10 @@ export function catalogSchemaRoutes(
|
|||||||
tableId,
|
tableId,
|
||||||
columnId,
|
columnId,
|
||||||
input.version,
|
input.version,
|
||||||
normalized(input.description),
|
"description" in input ? normalized(input.description ?? null) : current.description,
|
||||||
normalized(input.generatedDescription),
|
"generatedDescription" in input
|
||||||
|
? normalized(input.generatedDescription ?? null)
|
||||||
|
: current.generatedDescription,
|
||||||
input.sensitive,
|
input.sensitive,
|
||||||
);
|
);
|
||||||
if (!updated) return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
|
if (!updated) return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
|
||||||
|
|||||||
@@ -147,7 +147,7 @@ test("suggests sensitive flags from structural metadata without persisting them"
|
|||||||
suggestions: [{ columnId: expect.any(String), sensitive: true }],
|
suggestions: [{ columnId: expect.any(String), sensitive: true }],
|
||||||
})),
|
})),
|
||||||
};
|
};
|
||||||
const { app, repository, database, column } = await setup(modelCompleter);
|
const { app, repository, database, table, column } = await setup(modelCompleter);
|
||||||
modelCompleter.complete.mockResolvedValueOnce(JSON.stringify({
|
modelCompleter.complete.mockResolvedValueOnce(JSON.stringify({
|
||||||
suggestions: [{ columnId: column.id, sensitive: true }],
|
suggestions: [{ columnId: column.id, sensitive: true }],
|
||||||
}));
|
}));
|
||||||
@@ -156,12 +156,20 @@ test("suggests sensitive flags from structural metadata without persisting them"
|
|||||||
const response = await app.inject({
|
const response = await app.inject({
|
||||||
method: "POST",
|
method: "POST",
|
||||||
url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
|
url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
|
||||||
payload: { modelId: configuredModel.id },
|
payload: { modelId: configuredModel.id, scope: "all" },
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(response.statusCode).toBe(200);
|
expect(response.statusCode).toBe(200);
|
||||||
expect(response.json()).toEqual({
|
expect(response.json()).toEqual({
|
||||||
suggestions: [{ columnId: column.id, sensitive: true }],
|
suggestions: [{
|
||||||
|
columnId: column.id,
|
||||||
|
tableId: table.id,
|
||||||
|
tableName: table.name,
|
||||||
|
columnName: column.name,
|
||||||
|
version: column.version,
|
||||||
|
currentSensitive: false,
|
||||||
|
sensitive: true,
|
||||||
|
}],
|
||||||
});
|
});
|
||||||
expect(await repository.getColumn(database.id, column.tableId, column.id))
|
expect(await repository.getColumn(database.id, column.tableId, column.id))
|
||||||
.toMatchObject({ sensitive: false });
|
.toMatchObject({ sensitive: false });
|
||||||
@@ -178,6 +186,262 @@ test("suggests sensitive flags from structural metadata without persisting them"
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("limits sensitive-data suggestions to the selected tables or columns", async () => {
|
||||||
|
const modelCompleter: ModelCompleter = {
|
||||||
|
complete: vi.fn(async (request) => {
|
||||||
|
const payload = JSON.parse(request.messages.find((message) => message.role === "user")!.content) as {
|
||||||
|
columns: Array<{ columnId: string; column: string }>;
|
||||||
|
};
|
||||||
|
return JSON.stringify({
|
||||||
|
suggestions: payload.columns.map((column) => ({
|
||||||
|
columnId: column.columnId,
|
||||||
|
sensitive: column.column.includes("name") || column.column.includes("note"),
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
const { app, repository, database } = await setup(modelCompleter);
|
||||||
|
await repository.applySchemaSync(database.id, database.version, "all", [], {
|
||||||
|
schemaVersion: 1,
|
||||||
|
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||||
|
tables: [
|
||||||
|
{ name: "patients", sourceComment: null },
|
||||||
|
{ name: "visits", sourceComment: null },
|
||||||
|
{ name: "billing", sourceComment: null },
|
||||||
|
],
|
||||||
|
columns: [
|
||||||
|
{ tableName: "patients", name: "patient_name", ordinalPosition: 1, dataType: "text", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
||||||
|
{ tableName: "patients", name: "status", ordinalPosition: 2, dataType: "text", isNullable: true, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
||||||
|
{ tableName: "visits", name: "clinical_note", ordinalPosition: 1, dataType: "text", isNullable: true, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
||||||
|
{ tableName: "billing", name: "invoice_total", ordinalPosition: 1, dataType: "numeric", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
||||||
|
],
|
||||||
|
relationships: [],
|
||||||
|
});
|
||||||
|
const tables = await repository.listTables(database.id);
|
||||||
|
const patients = tables.find((table) => table.name === "patients")!;
|
||||||
|
const visits = tables.find((table) => table.name === "visits")!;
|
||||||
|
const billing = tables.find((table) => table.name === "billing")!;
|
||||||
|
const patientColumns = await repository.listColumns(database.id, patients.id);
|
||||||
|
const visitColumns = await repository.listColumns(database.id, visits.id);
|
||||||
|
const billingColumns = await repository.listColumns(database.id, billing.id);
|
||||||
|
const status = patientColumns.find((column) => column.name === "status")!;
|
||||||
|
const clinicalNote = visitColumns.find((column) => column.name === "clinical_note")!;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const tableResponse = await app.inject({
|
||||||
|
method: "POST",
|
||||||
|
url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
|
||||||
|
payload: {
|
||||||
|
modelId: configuredModel.id,
|
||||||
|
scope: "selected_tables",
|
||||||
|
targetIds: [visits.id, patients.id],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(tableResponse.statusCode).toBe(200);
|
||||||
|
expect(tableResponse.json().suggestions).toHaveLength(3);
|
||||||
|
expect(tableResponse.json().suggestions).toEqual(expect.arrayContaining([
|
||||||
|
expect.objectContaining({ tableId: patients.id, columnName: "patient_name", sensitive: true }),
|
||||||
|
expect.objectContaining({ tableId: patients.id, columnName: "status", sensitive: false }),
|
||||||
|
expect.objectContaining({ tableId: visits.id, columnName: "clinical_note", sensitive: true }),
|
||||||
|
]));
|
||||||
|
expect(tableResponse.json().suggestions).not.toEqual(expect.arrayContaining([
|
||||||
|
expect.objectContaining({ tableId: billing.id }),
|
||||||
|
]));
|
||||||
|
|
||||||
|
const columnResponse = await app.inject({
|
||||||
|
method: "POST",
|
||||||
|
url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
|
||||||
|
payload: {
|
||||||
|
modelId: configuredModel.id,
|
||||||
|
scope: "selected_columns",
|
||||||
|
targetIds: [clinicalNote.id, status.id],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(columnResponse.statusCode).toBe(200);
|
||||||
|
expect(columnResponse.json().suggestions).toHaveLength(2);
|
||||||
|
expect(columnResponse.json().suggestions).toEqual(expect.arrayContaining([
|
||||||
|
expect.objectContaining({ tableId: patients.id, columnId: status.id, sensitive: false }),
|
||||||
|
expect.objectContaining({ tableId: visits.id, columnId: clinicalNote.id, sensitive: true }),
|
||||||
|
]));
|
||||||
|
|
||||||
|
const prompts = vi.mocked(modelCompleter.complete).mock.calls.map(([request]) => (
|
||||||
|
JSON.parse(request.messages.find((message) => message.role === "user")!.content) as {
|
||||||
|
columns: Array<{ columnId: string }>;
|
||||||
|
}
|
||||||
|
));
|
||||||
|
expect(prompts[0]!.columns.map((column) => column.columnId).sort()).toEqual(
|
||||||
|
[...patientColumns, ...visitColumns].map((column) => column.id).sort(),
|
||||||
|
);
|
||||||
|
expect(prompts[0]!.columns.map((column) => column.columnId)).not.toContain(billingColumns[0]!.id);
|
||||||
|
expect(prompts[1]!.columns.map((column) => column.columnId).sort()).toEqual(
|
||||||
|
[status.id, clinicalNote.id].sort(),
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
await app.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("explains invalid sensitive-data suggestion selections without calling the model", async () => {
|
||||||
|
const modelCompleter: ModelCompleter = { complete: vi.fn(async () => "unused") };
|
||||||
|
const { app, database, table } = await setup(modelCompleter);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const empty = await app.inject({
|
||||||
|
method: "POST",
|
||||||
|
url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
|
||||||
|
payload: { modelId: configuredModel.id, scope: "selected_tables", targetIds: [] },
|
||||||
|
});
|
||||||
|
expect(empty.statusCode).toBe(400);
|
||||||
|
expect(empty.json()).toEqual({
|
||||||
|
code: "sensitive_data_suggestion_request_invalid",
|
||||||
|
message: "Choose a database, one or more tables, or one or more columns to classify.",
|
||||||
|
});
|
||||||
|
|
||||||
|
const duplicate = await app.inject({
|
||||||
|
method: "POST",
|
||||||
|
url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
|
||||||
|
payload: {
|
||||||
|
modelId: configuredModel.id,
|
||||||
|
scope: "selected_tables",
|
||||||
|
targetIds: [table.id, table.id],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(duplicate.statusCode).toBe(400);
|
||||||
|
expect(duplicate.json()).toEqual({
|
||||||
|
code: "sensitive_data_suggestion_target_ids_duplicate",
|
||||||
|
message: "Each selected table or column must appear only once.",
|
||||||
|
});
|
||||||
|
|
||||||
|
const missingTable = await app.inject({
|
||||||
|
method: "POST",
|
||||||
|
url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
|
||||||
|
payload: {
|
||||||
|
modelId: configuredModel.id,
|
||||||
|
scope: "selected_tables",
|
||||||
|
targetIds: ["00000000-0000-4000-8000-000000000001"],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(missingTable.statusCode).toBe(404);
|
||||||
|
expect(missingTable.json()).toEqual({
|
||||||
|
code: "catalog_table_not_found",
|
||||||
|
message: "One or more selected Catalog Tables were not found in this database.",
|
||||||
|
});
|
||||||
|
|
||||||
|
const missingColumn = await app.inject({
|
||||||
|
method: "POST",
|
||||||
|
url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
|
||||||
|
payload: {
|
||||||
|
modelId: configuredModel.id,
|
||||||
|
scope: "selected_columns",
|
||||||
|
targetIds: ["00000000-0000-4000-8000-000000000002"],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(missingColumn.statusCode).toBe(404);
|
||||||
|
expect(missingColumn.json()).toEqual({
|
||||||
|
code: "catalog_column_not_found",
|
||||||
|
message: "One or more selected Catalog Columns were not found in this database.",
|
||||||
|
});
|
||||||
|
expect(modelCompleter.complete).not.toHaveBeenCalled();
|
||||||
|
} finally {
|
||||||
|
await app.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("batches sensitive-data suggestions for schemas larger than one helper message", async () => {
|
||||||
|
const maxHelperMessageBytes = 64 * 1024;
|
||||||
|
const seenColumnIds: string[] = [];
|
||||||
|
const modelCompleter: ModelCompleter = {
|
||||||
|
complete: vi.fn(async (request) => {
|
||||||
|
const userMessage = request.messages.find((message) => message.role === "user")!;
|
||||||
|
expect(Buffer.byteLength(userMessage.content, "utf8")).toBeLessThanOrEqual(maxHelperMessageBytes);
|
||||||
|
const payload = JSON.parse(userMessage.content) as {
|
||||||
|
columns: Array<{ columnId: string; column: string }>;
|
||||||
|
};
|
||||||
|
expect(payload.columns.length).toBeLessThanOrEqual(10);
|
||||||
|
seenColumnIds.push(...payload.columns.map((column) => column.columnId));
|
||||||
|
return JSON.stringify({
|
||||||
|
suggestions: payload.columns.map((column) => ({
|
||||||
|
columnId: column.columnId,
|
||||||
|
sensitive: column.column.endsWith("_private"),
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
const { app, repository, database } = await setup(modelCompleter);
|
||||||
|
const columnCount = 900;
|
||||||
|
await repository.applySchemaSync(database.id, database.version, "all", [], {
|
||||||
|
schemaVersion: 1,
|
||||||
|
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||||
|
tables: [{ name: "wide_table", sourceComment: null }],
|
||||||
|
columns: Array.from({ length: columnCount }, (_, index) => ({
|
||||||
|
tableName: "wide_table",
|
||||||
|
name: `field_${index.toString().padStart(4, "0")}${index % 10 === 0 ? "_private" : ""}`,
|
||||||
|
ordinalPosition: index + 1,
|
||||||
|
dataType: "character varying(255)",
|
||||||
|
isNullable: true,
|
||||||
|
defaultExpression: null,
|
||||||
|
primaryKeyPosition: null,
|
||||||
|
sourceComment: null,
|
||||||
|
})),
|
||||||
|
relationships: [],
|
||||||
|
});
|
||||||
|
const wideTable = (await repository.listTables(database.id)).find((table) => table.name === "wide_table")!;
|
||||||
|
const expectedColumnIds = (await repository.listColumns(database.id, wideTable.id)).map((column) => column.id);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await app.inject({
|
||||||
|
method: "POST",
|
||||||
|
url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
|
||||||
|
payload: { modelId: configuredModel.id, scope: "all" },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
const suggestions = response.json().suggestions as Array<{
|
||||||
|
columnName: string;
|
||||||
|
currentSensitive: boolean;
|
||||||
|
sensitive: boolean;
|
||||||
|
}>;
|
||||||
|
expect(suggestions).toHaveLength(columnCount);
|
||||||
|
expect(suggestions).toEqual(expect.arrayContaining([
|
||||||
|
expect.objectContaining({ columnName: "field_0000_private", currentSensitive: false, sensitive: true }),
|
||||||
|
expect.objectContaining({ columnName: "field_0001", currentSensitive: false, sensitive: false }),
|
||||||
|
]));
|
||||||
|
expect(vi.mocked(modelCompleter.complete).mock.calls.length).toBeGreaterThan(1);
|
||||||
|
expect(seenColumnIds.slice().sort()).toEqual(expectedColumnIds.slice().sort());
|
||||||
|
expect(new Set(seenColumnIds).size).toBe(columnCount);
|
||||||
|
} finally {
|
||||||
|
await app.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("retries one invalid sensitive-data classification before returning the review draft", async () => {
|
||||||
|
const modelCompleter: ModelCompleter = {
|
||||||
|
complete: vi.fn(async () => "unused"),
|
||||||
|
};
|
||||||
|
const { app, database, column } = await setup(modelCompleter);
|
||||||
|
vi.mocked(modelCompleter.complete)
|
||||||
|
.mockResolvedValueOnce("not-json")
|
||||||
|
.mockResolvedValueOnce(JSON.stringify({
|
||||||
|
suggestions: [{ columnId: column.id, sensitive: true }],
|
||||||
|
}));
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await app.inject({
|
||||||
|
method: "POST",
|
||||||
|
url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
|
||||||
|
payload: { modelId: configuredModel.id, scope: "all" },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
expect(response.json().suggestions).toEqual([
|
||||||
|
expect.objectContaining({ columnId: column.id, sensitive: true }),
|
||||||
|
]);
|
||||||
|
expect(modelCompleter.complete).toHaveBeenCalledTimes(2);
|
||||||
|
} finally {
|
||||||
|
await app.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test.each(["malformed", "incomplete", "duplicate"] as const)(
|
test.each(["malformed", "incomplete", "duplicate"] as const)(
|
||||||
"fails safely when sensitive-data suggestions are %s",
|
"fails safely when sensitive-data suggestions are %s",
|
||||||
async (kind) => {
|
async (kind) => {
|
||||||
@@ -201,13 +465,13 @@ test.each(["malformed", "incomplete", "duplicate"] as const)(
|
|||||||
const response = await app.inject({
|
const response = await app.inject({
|
||||||
method: "POST",
|
method: "POST",
|
||||||
url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
|
url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
|
||||||
payload: { modelId: configuredModel.id },
|
payload: { modelId: configuredModel.id, scope: "all" },
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(response.statusCode).toBe(502);
|
expect(response.statusCode).toBe(502);
|
||||||
expect(response.json()).toEqual({
|
expect(response.json()).toEqual({
|
||||||
code: "sensitive_data_suggestion_failed",
|
code: "sensitive_data_suggestion_invalid_response",
|
||||||
message: "Sensitive-data suggestions could not be prepared.",
|
message: "The LLM returned an incomplete or invalid classification. No suggestions were applied.",
|
||||||
});
|
});
|
||||||
expect(response.body).not.toContain(rawResponse);
|
expect(response.body).not.toContain(rawResponse);
|
||||||
expect(await repository.getColumn(database.id, column.tableId, column.id))
|
expect(await repository.getColumn(database.id, column.tableId, column.id))
|
||||||
@@ -218,6 +482,34 @@ test.each(["malformed", "incomplete", "duplicate"] as const)(
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
test("explains a sensitive-data suggestion provider failure without exposing provider details", async () => {
|
||||||
|
const modelCompleter: ModelCompleter = {
|
||||||
|
complete: vi.fn(async () => {
|
||||||
|
throw new ModelCompletionProviderError();
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
const { app, repository, database, column } = await setup(modelCompleter);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await app.inject({
|
||||||
|
method: "POST",
|
||||||
|
url: `/catalog/databases/${database.id}/sensitive-data-suggestions`,
|
||||||
|
payload: { modelId: configuredModel.id, scope: "all" },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(response.statusCode).toBe(502);
|
||||||
|
expect(response.json()).toEqual({
|
||||||
|
code: "sensitive_data_suggestion_provider_unavailable",
|
||||||
|
message: "The selected LLM service could not complete the request. No suggestions were applied.",
|
||||||
|
});
|
||||||
|
expect(response.body).not.toContain("model completion failed");
|
||||||
|
expect(await repository.getColumn(database.id, column.tableId, column.id))
|
||||||
|
.toMatchObject({ sensitive: false });
|
||||||
|
} finally {
|
||||||
|
await app.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
interface SseFrame {
|
interface SseFrame {
|
||||||
id?: string;
|
id?: string;
|
||||||
event?: string;
|
event?: string;
|
||||||
|
|||||||
@@ -241,14 +241,29 @@ test("keeps generated descriptions editable and preserves them across synchroniz
|
|||||||
version: idColumn.version,
|
version: idColumn.version,
|
||||||
description: "Reviewed key",
|
description: "Reviewed key",
|
||||||
generatedDescription: "Generated key draft",
|
generatedDescription: "Generated key draft",
|
||||||
sensitive: true,
|
sensitive: false,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
expect(editedColumn.json()).toMatchObject({
|
expect(editedColumn.json()).toMatchObject({
|
||||||
|
description: "Reviewed key",
|
||||||
|
generatedDescription: "Generated key draft",
|
||||||
|
sensitive: false,
|
||||||
|
});
|
||||||
|
const sensitiveOnly = await app.inject({
|
||||||
|
method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}/columns/${idColumn.id}`,
|
||||||
|
payload: { version: editedColumn.json().version, sensitive: true },
|
||||||
|
});
|
||||||
|
expect(sensitiveOnly.statusCode).toBe(200);
|
||||||
|
expect(sensitiveOnly.json()).toMatchObject({
|
||||||
description: "Reviewed key",
|
description: "Reviewed key",
|
||||||
generatedDescription: "Generated key draft",
|
generatedDescription: "Generated key draft",
|
||||||
sensitive: true,
|
sensitive: true,
|
||||||
});
|
});
|
||||||
|
const emptyPatch = await app.inject({
|
||||||
|
method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}/columns/${idColumn.id}`,
|
||||||
|
payload: { version: sensitiveOnly.json().version },
|
||||||
|
});
|
||||||
|
expect(emptyPatch.statusCode).toBe(400);
|
||||||
|
|
||||||
const second = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
const second = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
||||||
await waitFor(repository, second.json().id, "succeeded");
|
await waitFor(repository, second.json().id, "succeeded");
|
||||||
|
|||||||
@@ -9,6 +9,13 @@ draft suggestions from structural metadata only, but the user decides and only t
|
|||||||
persisted; there is no rationale, history, audit ledger, fingerprint, review state, or retroactive
|
persisted; there is no rationale, history, audit ledger, fingerprint, review state, or retroactive
|
||||||
regeneration of existing descriptions.
|
regeneration of existing descriptions.
|
||||||
|
|
||||||
|
The user starts a suggestion from an explicit selection at database, table, or column level. A
|
||||||
|
database request accepts exactly one selected database; a table or column request contains only the
|
||||||
|
selected tables' columns or the selected columns, respectively. The backend divides that structural
|
||||||
|
metadata into deterministic model requests of at most ten columns, also bounded by helper message
|
||||||
|
size, and returns one combined draft for human review. No flag changes until the user saves the
|
||||||
|
reviewed draft.
|
||||||
|
|
||||||
Description generation extends ADR-0010 by sending bounded real values when `sensitive` is false
|
Description generation extends ADR-0010 by sending bounded real values when `sensitive` is false
|
||||||
and deterministic plausible synthetic values when it is true, without identifying the synthetic
|
and deterministic plausible synthetic values when it is true, without identifying the synthetic
|
||||||
values to the model. The false default deliberately favors the expected stable schemas and the
|
values to the model. The false default deliberately favors the expected stable schemas and the
|
||||||
|
|||||||
@@ -99,6 +99,11 @@ export interface CatalogColumn {
|
|||||||
|
|
||||||
export interface SensitiveDataSuggestion {
|
export interface SensitiveDataSuggestion {
|
||||||
columnId: string;
|
columnId: string;
|
||||||
|
tableId: string;
|
||||||
|
tableName: string;
|
||||||
|
columnName: string;
|
||||||
|
version: number;
|
||||||
|
currentSensitive: boolean;
|
||||||
sensitive: boolean;
|
sensitive: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -106,6 +111,11 @@ export interface SensitiveDataSuggestions {
|
|||||||
suggestions: SensitiveDataSuggestion[];
|
suggestions: SensitiveDataSuggestion[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type SensitiveDataSuggestionRequest =
|
||||||
|
| { scope: "all" }
|
||||||
|
| { scope: "selected_tables"; targetIds: string[] }
|
||||||
|
| { scope: "selected_columns"; targetIds: string[] };
|
||||||
|
|
||||||
export interface CatalogRelationshipColumn {
|
export interface CatalogRelationshipColumn {
|
||||||
position: number;
|
position: number;
|
||||||
sourceColumnId: string;
|
sourceColumnId: string;
|
||||||
@@ -357,10 +367,25 @@ export const updateCatalogColumnMetadata = (
|
|||||||
{ method: "PATCH", body: JSON.stringify({ version, description, generatedDescription, sensitive }) },
|
{ method: "PATCH", body: JSON.stringify({ version, description, generatedDescription, sensitive }) },
|
||||||
);
|
);
|
||||||
|
|
||||||
export const suggestSensitiveFields = (databaseId: string, modelId: string) =>
|
export const updateCatalogColumnSensitive = (
|
||||||
|
databaseId: string,
|
||||||
|
tableId: string,
|
||||||
|
columnId: string,
|
||||||
|
version: number,
|
||||||
|
sensitive: boolean,
|
||||||
|
) => apiFetch<CatalogColumn>(
|
||||||
|
`/catalog/databases/${encodeURIComponent(databaseId)}/tables/${encodeURIComponent(tableId)}/columns/${encodeURIComponent(columnId)}`,
|
||||||
|
{ method: "PATCH", body: JSON.stringify({ version, sensitive }) },
|
||||||
|
);
|
||||||
|
|
||||||
|
export const suggestSensitiveFields = (
|
||||||
|
databaseId: string,
|
||||||
|
modelId: string,
|
||||||
|
selection: SensitiveDataSuggestionRequest,
|
||||||
|
) =>
|
||||||
apiFetch<SensitiveDataSuggestions>(
|
apiFetch<SensitiveDataSuggestions>(
|
||||||
`/catalog/databases/${encodeURIComponent(databaseId)}/sensitive-data-suggestions`,
|
`/catalog/databases/${encodeURIComponent(databaseId)}/sensitive-data-suggestions`,
|
||||||
{ method: "POST", body: JSON.stringify({ modelId }) },
|
{ method: "POST", body: JSON.stringify({ modelId, ...selection }) },
|
||||||
);
|
);
|
||||||
|
|
||||||
export const listCatalogRelationships = (databaseId: string) =>
|
export const listCatalogRelationships = (databaseId: string) =>
|
||||||
|
|||||||
@@ -147,7 +147,9 @@ test.each([
|
|||||||
["description_generation_target_ids_duplicate", "Description generation target IDs must be unique."],
|
["description_generation_target_ids_duplicate", "Description generation target IDs must be unique."],
|
||||||
["description_generation_no_eligible_targets", "No eligible catalog tables or columns need description generation."],
|
["description_generation_no_eligible_targets", "No eligible catalog tables or columns need description generation."],
|
||||||
["catalog_table_not_found", "One or more selected catalog tables were not found."],
|
["catalog_table_not_found", "One or more selected catalog tables were not found."],
|
||||||
])("maps the description-generation error code %s to safe local copy", async (code, message) => {
|
["sensitive_data_suggestion_invalid_response", "The model returned an incomplete or invalid classification. No suggestions were applied."],
|
||||||
|
["sensitive_data_suggestion_provider_unavailable", "The selected model could not complete the request. No suggestions were applied."],
|
||||||
|
])("maps the catalog error code %s to safe local copy", async (code, message) => {
|
||||||
const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue(
|
const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue(
|
||||||
new Response(JSON.stringify({ code, message: "provider detail must not be trusted" }), {
|
new Response(JSON.stringify({ code, message: "provider detail must not be trusted" }), {
|
||||||
status: 400,
|
status: 400,
|
||||||
|
|||||||
@@ -23,6 +23,13 @@ const safeErrorCodes = new Set([
|
|||||||
"description_generation_target_ids_duplicate", "metadata_generation_model_unavailable",
|
"description_generation_target_ids_duplicate", "metadata_generation_model_unavailable",
|
||||||
"catalog_column_not_found", "catalog_table_not_found",
|
"catalog_column_not_found", "catalog_table_not_found",
|
||||||
"workspace_configuration_unavailable",
|
"workspace_configuration_unavailable",
|
||||||
|
"sensitive_data_suggestion_request_invalid",
|
||||||
|
"sensitive_data_suggestion_target_ids_duplicate",
|
||||||
|
"sensitive_data_suggestion_no_columns",
|
||||||
|
"sensitive_data_suggestion_payload_too_large",
|
||||||
|
"sensitive_data_suggestion_invalid_response",
|
||||||
|
"sensitive_data_suggestion_provider_unavailable",
|
||||||
|
"sensitive_data_suggestion_failed",
|
||||||
"schema_sync_conflict", "schema_introspection_failed", "schema_request_invalid",
|
"schema_sync_conflict", "schema_introspection_failed", "schema_request_invalid",
|
||||||
"schema_operation_failed", "sync_run_not_found", "table_stale", "column_stale",
|
"schema_operation_failed", "sync_run_not_found", "table_stale", "column_stale",
|
||||||
]);
|
]);
|
||||||
@@ -77,6 +84,13 @@ const localCodeMessages: Record<string, string> = {
|
|||||||
catalog_column_not_found: "The selected catalog column was not found.",
|
catalog_column_not_found: "The selected catalog column was not found.",
|
||||||
catalog_table_not_found: "One or more selected catalog tables were not found.",
|
catalog_table_not_found: "One or more selected catalog tables were not found.",
|
||||||
workspace_configuration_unavailable: "The database workspace configuration is unavailable.",
|
workspace_configuration_unavailable: "The database workspace configuration is unavailable.",
|
||||||
|
sensitive_data_suggestion_request_invalid: "Select a database, one or more tables, or one or more columns before requesting sensitive-field suggestions.",
|
||||||
|
sensitive_data_suggestion_target_ids_duplicate: "Each selected table or column can be included only once.",
|
||||||
|
sensitive_data_suggestion_no_columns: "The selected scope contains no catalog columns to classify.",
|
||||||
|
sensitive_data_suggestion_payload_too_large: "The selected structural metadata cannot be divided into safe model requests.",
|
||||||
|
sensitive_data_suggestion_invalid_response: "The model returned an incomplete or invalid classification. No suggestions were applied.",
|
||||||
|
sensitive_data_suggestion_provider_unavailable: "The selected model could not complete the request. No suggestions were applied.",
|
||||||
|
sensitive_data_suggestion_failed: "Sensitive-field suggestions failed before review. No changes were applied.",
|
||||||
schema_sync_conflict: "A schema synchronization is already active or no longer current.",
|
schema_sync_conflict: "A schema synchronization is already active or no longer current.",
|
||||||
schema_introspection_failed: "The database schema could not be read safely.",
|
schema_introspection_failed: "The database schema could not be read safely.",
|
||||||
schema_request_invalid: "The schema request is invalid.",
|
schema_request_invalid: "The schema request is invalid.",
|
||||||
|
|||||||
@@ -1571,6 +1571,125 @@ test("starts one selected column with the configured default model", async () =>
|
|||||||
expect(await screen.findByText("Description generation started for 1 column")).toBeVisible();
|
expect(await screen.findByText("Description generation started for 1 column")).toBeVisible();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("shows database sensitive suggestions only for a selection and rejects multiple databases clearly", async () => {
|
||||||
|
const user = userEvent.setup();
|
||||||
|
let suggestionCalls = 0;
|
||||||
|
const radiology = makeDatabase({
|
||||||
|
id: "44444444-4444-4444-8444-444444444444",
|
||||||
|
workspaceId: "radiology",
|
||||||
|
workspaceName: "Radiology",
|
||||||
|
});
|
||||||
|
server.use(
|
||||||
|
http.get("/api/catalog/metadata-generation/models", () => HttpResponse.json({
|
||||||
|
models: [{ id: "local-qwen", label: "Local Qwen" }],
|
||||||
|
default: "local-qwen",
|
||||||
|
})),
|
||||||
|
http.post("/api/catalog/databases/:databaseId/sensitive-data-suggestions", () => {
|
||||||
|
suggestionCalls += 1;
|
||||||
|
return HttpResponse.json({ suggestions: [] });
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
renderPage({ rows: [makeDatabase(), radiology] });
|
||||||
|
|
||||||
|
expect(screen.queryByRole("button", { name: "Suggest sensitive fields" })).not.toBeInTheDocument();
|
||||||
|
const psdRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
|
||||||
|
const radiologyRow = await screen.findByRole("row", { name: /Radiology/ });
|
||||||
|
await user.click(within(psdRow).getByRole("checkbox", { name: /toggle row selection/i }));
|
||||||
|
expect(screen.getByRole("button", { name: "Suggest sensitive fields" })).toBeVisible();
|
||||||
|
await user.click(within(radiologyRow).getByRole("checkbox", { name: /toggle row selection/i }));
|
||||||
|
await user.click(screen.getByRole("button", { name: "Suggest sensitive fields" }));
|
||||||
|
|
||||||
|
expect(await screen.findByText("Sensitive-field suggestions can be requested for only one database at a time. Select one database and try again.")).toBeVisible();
|
||||||
|
expect(suggestionCalls).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("requests database-level sensitive suggestions for the only selected database", async () => {
|
||||||
|
const user = userEvent.setup();
|
||||||
|
let suggestionBody: unknown;
|
||||||
|
server.use(
|
||||||
|
http.get("/api/catalog/metadata-generation/models", () => HttpResponse.json({
|
||||||
|
models: [{ id: "local-qwen", label: "Local Qwen" }],
|
||||||
|
default: "local-qwen",
|
||||||
|
})),
|
||||||
|
http.post("/api/catalog/databases/:databaseId/sensitive-data-suggestions", async ({ request }) => {
|
||||||
|
suggestionBody = await request.json();
|
||||||
|
return HttpResponse.json({
|
||||||
|
suggestions: [{
|
||||||
|
columnId: patientIdColumn.id,
|
||||||
|
tableId: patientsTable.id,
|
||||||
|
tableName: patientsTable.name,
|
||||||
|
columnName: patientIdColumn.name,
|
||||||
|
version: patientIdColumn.version,
|
||||||
|
currentSensitive: false,
|
||||||
|
sensitive: true,
|
||||||
|
}],
|
||||||
|
});
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
renderPage({ rows: [makeDatabase()] });
|
||||||
|
|
||||||
|
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
|
||||||
|
await user.click(within(databaseRow).getByRole("checkbox", { name: /toggle row selection/i }));
|
||||||
|
await user.click(screen.getByRole("button", { name: "Suggest sensitive fields" }));
|
||||||
|
|
||||||
|
await waitFor(() => expect(suggestionBody).toEqual({ modelId: "local-qwen", scope: "all" }));
|
||||||
|
expect(await screen.findByRole("complementary", { name: "Sensitive field review" })).toBeVisible();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("requests sensitive suggestions only for selected tables", async () => {
|
||||||
|
const user = userEvent.setup();
|
||||||
|
const visitsTable: CatalogTable = {
|
||||||
|
...patientsTable,
|
||||||
|
id: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb",
|
||||||
|
name: "visits",
|
||||||
|
};
|
||||||
|
const visitColumn = {
|
||||||
|
...patientIdColumn,
|
||||||
|
id: "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee",
|
||||||
|
tableId: visitsTable.id,
|
||||||
|
name: "clinical_note",
|
||||||
|
};
|
||||||
|
let suggestionBody: unknown;
|
||||||
|
server.use(
|
||||||
|
http.get("/api/catalog/metadata-generation/models", () => HttpResponse.json({
|
||||||
|
models: [{ id: "local-qwen", label: "Local Qwen" }],
|
||||||
|
default: "local-qwen",
|
||||||
|
})),
|
||||||
|
http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([
|
||||||
|
patientsTable,
|
||||||
|
visitsTable,
|
||||||
|
])),
|
||||||
|
http.post("/api/catalog/databases/:databaseId/sensitive-data-suggestions", async ({ request }) => {
|
||||||
|
suggestionBody = await request.json();
|
||||||
|
return HttpResponse.json({
|
||||||
|
suggestions: [{
|
||||||
|
columnId: visitColumn.id,
|
||||||
|
tableId: visitsTable.id,
|
||||||
|
tableName: visitsTable.name,
|
||||||
|
columnName: visitColumn.name,
|
||||||
|
version: visitColumn.version,
|
||||||
|
currentSensitive: false,
|
||||||
|
sensitive: true,
|
||||||
|
}],
|
||||||
|
});
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
renderPage({ rows: [makeDatabase({ connectionStatus: "reachable", testedVersion: 3 })] });
|
||||||
|
|
||||||
|
await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" }));
|
||||||
|
await user.click(screen.getByRole("tab", { name: "Tables" }));
|
||||||
|
const visitsRow = await screen.findByRole("row", { name: /visits/ });
|
||||||
|
await user.click(within(visitsRow).getByRole("checkbox", { name: /toggle row selection/i }));
|
||||||
|
await user.click(screen.getByRole("button", { name: "Suggest sensitive fields" }));
|
||||||
|
|
||||||
|
await waitFor(() => expect(suggestionBody).toEqual({
|
||||||
|
modelId: "local-qwen",
|
||||||
|
scope: "selected_tables",
|
||||||
|
targetIds: [visitsTable.id],
|
||||||
|
}));
|
||||||
|
expect(await screen.findByRole("complementary", { name: "Sensitive field review" })).toBeVisible();
|
||||||
|
});
|
||||||
|
|
||||||
test("reviews AI-sensitive-field suggestions as an editable draft and saves only changed columns", async () => {
|
test("reviews AI-sensitive-field suggestions as an editable draft and saves only changed columns", async () => {
|
||||||
const user = userEvent.setup();
|
const user = userEvent.setup();
|
||||||
const idColumn = { ...patientIdColumn, sensitive: false };
|
const idColumn = { ...patientIdColumn, sensitive: false };
|
||||||
@@ -1585,7 +1704,18 @@ test("reviews AI-sensitive-field suggestions as an editable draft and saves only
|
|||||||
generatedDescription: "Name of the patient",
|
generatedDescription: "Name of the patient",
|
||||||
sensitive: false,
|
sensitive: false,
|
||||||
};
|
};
|
||||||
let columns = [idColumn, nameColumn];
|
const unselectedColumn = {
|
||||||
|
...patientIdColumn,
|
||||||
|
id: "ffffffff-ffff-4fff-8fff-ffffffffffff",
|
||||||
|
name: "address",
|
||||||
|
ordinalPosition: 3,
|
||||||
|
primaryKeyPosition: null,
|
||||||
|
isPrimaryKey: false,
|
||||||
|
description: "Patient address",
|
||||||
|
generatedDescription: "Address of the patient",
|
||||||
|
sensitive: false,
|
||||||
|
};
|
||||||
|
let columns = [idColumn, nameColumn, unselectedColumn];
|
||||||
let suggestionBody: unknown;
|
let suggestionBody: unknown;
|
||||||
const patches: Array<{ columnId: string; body: unknown }> = [];
|
const patches: Array<{ columnId: string; body: unknown }> = [];
|
||||||
server.use(
|
server.use(
|
||||||
@@ -1604,9 +1734,24 @@ test("reviews AI-sensitive-field suggestions as an editable draft and saves only
|
|||||||
suggestionBody = await request.json();
|
suggestionBody = await request.json();
|
||||||
return HttpResponse.json({
|
return HttpResponse.json({
|
||||||
suggestions: [
|
suggestions: [
|
||||||
{ columnId: idColumn.id, sensitive: true },
|
{
|
||||||
{ columnId: nameColumn.id, sensitive: true },
|
columnId: idColumn.id,
|
||||||
{ columnId: "ffffffff-ffff-4fff-8fff-ffffffffffff", sensitive: true },
|
tableId: patientsTable.id,
|
||||||
|
tableName: patientsTable.name,
|
||||||
|
columnName: idColumn.name,
|
||||||
|
version: idColumn.version,
|
||||||
|
currentSensitive: false,
|
||||||
|
sensitive: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
columnId: nameColumn.id,
|
||||||
|
tableId: patientsTable.id,
|
||||||
|
tableName: patientsTable.name,
|
||||||
|
columnName: nameColumn.name,
|
||||||
|
version: nameColumn.version,
|
||||||
|
currentSensitive: false,
|
||||||
|
sensitive: true,
|
||||||
|
},
|
||||||
],
|
],
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
@@ -1616,8 +1761,6 @@ test("reviews AI-sensitive-field suggestions as an editable draft and saves only
|
|||||||
async ({ params, request }) => {
|
async ({ params, request }) => {
|
||||||
const body = await request.json() as {
|
const body = await request.json() as {
|
||||||
version: number;
|
version: number;
|
||||||
description: string | null;
|
|
||||||
generatedDescription: string | null;
|
|
||||||
sensitive: boolean;
|
sensitive: boolean;
|
||||||
};
|
};
|
||||||
patches.push({ columnId: String(params.columnId), body });
|
patches.push({ columnId: String(params.columnId), body });
|
||||||
@@ -1638,31 +1781,41 @@ test("reviews AI-sensitive-field suggestions as an editable draft and saves only
|
|||||||
const nameSensitive = await screen.findByRole("checkbox", { name: "Sensitive data for name" });
|
const nameSensitive = await screen.findByRole("checkbox", { name: "Sensitive data for name" });
|
||||||
expect(idSensitive).not.toBeChecked();
|
expect(idSensitive).not.toBeChecked();
|
||||||
expect(nameSensitive).not.toBeChecked();
|
expect(nameSensitive).not.toBeChecked();
|
||||||
|
expect(screen.queryByRole("button", { name: "Suggest sensitive fields" })).not.toBeInTheDocument();
|
||||||
|
|
||||||
|
const selectableRow = async (name: RegExp) => {
|
||||||
|
const rows = await screen.findAllByRole("row", { name });
|
||||||
|
return rows.find((row) => within(row).queryByRole("checkbox", { name: /toggle row selection/i }))!;
|
||||||
|
};
|
||||||
|
await user.click(within(await selectableRow(/Patient identifier/))
|
||||||
|
.getByRole("checkbox", { name: /toggle row selection/i }));
|
||||||
|
await user.click(within(await selectableRow(/Patient name/))
|
||||||
|
.getByRole("checkbox", { name: /toggle row selection/i }));
|
||||||
await user.click(screen.getByRole("button", { name: "Suggest sensitive fields" }));
|
await user.click(screen.getByRole("button", { name: "Suggest sensitive fields" }));
|
||||||
|
|
||||||
await waitFor(() => expect(suggestionBody).toEqual({ modelId: "local-qwen" }));
|
await waitFor(() => expect(suggestionBody).toEqual({
|
||||||
await waitFor(() => {
|
modelId: "local-qwen",
|
||||||
expect(screen.getByRole("checkbox", { name: "Sensitive data for id" })).toBeChecked();
|
scope: "selected_columns",
|
||||||
expect(screen.getByRole("checkbox", { name: "Sensitive data for name" })).toBeChecked();
|
targetIds: [idColumn.id, nameColumn.id],
|
||||||
});
|
}));
|
||||||
|
const review = await screen.findByRole("complementary", { name: "Sensitive field review" });
|
||||||
|
expect(within(review).getByRole("checkbox", { name: "Protect patients.id" })).toBeChecked();
|
||||||
|
expect(within(review).getByRole("checkbox", { name: "Protect patients.name" })).toBeChecked();
|
||||||
expect(patches).toHaveLength(0);
|
expect(patches).toHaveLength(0);
|
||||||
|
|
||||||
await user.click(screen.getByRole("checkbox", { name: "Sensitive data for name" }));
|
await user.click(within(review).getByRole("checkbox", { name: "Protect patients.name" }));
|
||||||
expect(screen.getByRole("checkbox", { name: "Sensitive data for name" })).not.toBeChecked();
|
await user.click(within(review).getByRole("button", { name: "Save 1" }));
|
||||||
await user.click(screen.getByRole("button", { name: "Save sensitive fields" }));
|
|
||||||
|
|
||||||
await waitFor(() => expect(patches).toEqual([{
|
await waitFor(() => expect(patches).toEqual([{
|
||||||
columnId: idColumn.id,
|
columnId: idColumn.id,
|
||||||
body: {
|
body: {
|
||||||
version: idColumn.version,
|
version: idColumn.version,
|
||||||
description: idColumn.description,
|
|
||||||
generatedDescription: idColumn.generatedDescription,
|
|
||||||
sensitive: true,
|
sensitive: true,
|
||||||
},
|
},
|
||||||
}]));
|
}]));
|
||||||
expect(await screen.findByText("Sensitive fields saved")).toBeVisible();
|
expect(await screen.findByText("Saved 1 sensitive flag")).toBeVisible();
|
||||||
expect(screen.getByRole("checkbox", { name: "Sensitive data for id" })).toBeChecked();
|
await waitFor(() => expect(screen.queryByRole("complementary", { name: "Sensitive field review" })).not.toBeInTheDocument());
|
||||||
|
await waitFor(() => expect(screen.getByRole("checkbox", { name: "Sensitive data for id" })).toBeChecked());
|
||||||
expect(screen.getByRole("checkbox", { name: "Sensitive data for name" })).not.toBeChecked();
|
expect(screen.getByRole("checkbox", { name: "Sensitive data for name" })).not.toBeChecked();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -20,9 +20,11 @@ import {
|
|||||||
replaceCatalogDatabaseSecrets,
|
replaceCatalogDatabaseSecrets,
|
||||||
startCatalogSync,
|
startCatalogSync,
|
||||||
startDescriptionGenerationRun,
|
startDescriptionGenerationRun,
|
||||||
|
suggestSensitiveFields,
|
||||||
testCatalogDatabase,
|
testCatalogDatabase,
|
||||||
updateCatalogDatabase,
|
updateCatalogDatabase,
|
||||||
type CatalogDatabase,
|
type CatalogDatabase,
|
||||||
|
type CatalogColumn,
|
||||||
type CatalogDatabaseMetadataDeleteTarget,
|
type CatalogDatabaseMetadataDeleteTarget,
|
||||||
type CatalogSecretName,
|
type CatalogSecretName,
|
||||||
type CatalogSyncScope,
|
type CatalogSyncScope,
|
||||||
@@ -30,6 +32,8 @@ import {
|
|||||||
type DatabaseBinding,
|
type DatabaseBinding,
|
||||||
type DatabaseTransport,
|
type DatabaseTransport,
|
||||||
type DescriptionGenerationRun,
|
type DescriptionGenerationRun,
|
||||||
|
type SensitiveDataSuggestion,
|
||||||
|
type SensitiveDataSuggestionRequest,
|
||||||
} from "../api/catalog-databases";
|
} from "../api/catalog-databases";
|
||||||
import { DatabaseGrid } from "./database-management/DatabaseGrid";
|
import { DatabaseGrid } from "./database-management/DatabaseGrid";
|
||||||
import { DatabaseForm } from "./database-management/DatabaseForm";
|
import { DatabaseForm } from "./database-management/DatabaseForm";
|
||||||
@@ -38,6 +42,7 @@ import { DatabaseRelationships } from "./database-management/DatabaseRelationshi
|
|||||||
import { CatalogSyncDrawer } from "./database-management/CatalogSyncDrawer";
|
import { CatalogSyncDrawer } from "./database-management/CatalogSyncDrawer";
|
||||||
import { MetadataGenerationModelSelector } from "./database-management/MetadataGenerationModelSelector";
|
import { MetadataGenerationModelSelector } from "./database-management/MetadataGenerationModelSelector";
|
||||||
import { DescriptionGenerationDrawer } from "./database-management/DescriptionGenerationDrawer";
|
import { DescriptionGenerationDrawer } from "./database-management/DescriptionGenerationDrawer";
|
||||||
|
import { SensitiveDataReviewDrawer } from "./database-management/SensitiveDataReviewDrawer";
|
||||||
import {
|
import {
|
||||||
configurationFingerprint,
|
configurationFingerprint,
|
||||||
configurationFromDraft,
|
configurationFromDraft,
|
||||||
@@ -141,6 +146,11 @@ export function DatabaseManagementPage({
|
|||||||
const [syncDrawerOpen, setSyncDrawerOpen] = useState(false);
|
const [syncDrawerOpen, setSyncDrawerOpen] = useState(false);
|
||||||
const [activeDescriptionGenerationRun, setActiveDescriptionGenerationRun] = useState<DescriptionGenerationRun | null>(null);
|
const [activeDescriptionGenerationRun, setActiveDescriptionGenerationRun] = useState<DescriptionGenerationRun | null>(null);
|
||||||
const [descriptionGenerationDrawerOpen, setDescriptionGenerationDrawerOpen] = useState(false);
|
const [descriptionGenerationDrawerOpen, setDescriptionGenerationDrawerOpen] = useState(false);
|
||||||
|
const [sensitiveReview, setSensitiveReview] = useState<{
|
||||||
|
databaseId: string;
|
||||||
|
scopeLabel: string;
|
||||||
|
suggestions: SensitiveDataSuggestion[];
|
||||||
|
} | null>(null);
|
||||||
|
|
||||||
const originRef = useRef<HTMLElement | null>(null);
|
const originRef = useRef<HTMLElement | null>(null);
|
||||||
const searchInputRef = useRef<HTMLInputElement>(null);
|
const searchInputRef = useRef<HTMLInputElement>(null);
|
||||||
@@ -683,6 +693,67 @@ export function DatabaseManagementPage({
|
|||||||
}
|
}
|
||||||
}, [rememberDescriptionGenerationRun, selectedMetadataModel]);
|
}, [rememberDescriptionGenerationRun, selectedMetadataModel]);
|
||||||
|
|
||||||
|
const requestSensitiveSuggestions = useCallback(async (
|
||||||
|
database: CatalogDatabase,
|
||||||
|
selection: SensitiveDataSuggestionRequest,
|
||||||
|
scopeLabel: string,
|
||||||
|
) => {
|
||||||
|
if (!database.id) {
|
||||||
|
toast.error("The selected database is not configured, so sensitive-field suggestions were not requested.");
|
||||||
|
throw new Error("database is not configured");
|
||||||
|
}
|
||||||
|
if (!selectedMetadataModel) {
|
||||||
|
toast.error("Select a metadata-generation model before requesting sensitive-field suggestions.");
|
||||||
|
throw new Error("metadata-generation model is not selected");
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const result = await suggestSensitiveFields(database.id, selectedMetadataModel, selection);
|
||||||
|
setSensitiveReview({ databaseId: database.id, scopeLabel, suggestions: result.suggestions });
|
||||||
|
toast.success(`Prepared ${result.suggestions.length} sensitive-field suggestion${result.suggestions.length === 1 ? "" : "s"} for review`);
|
||||||
|
} catch (error) {
|
||||||
|
toast.error(apiErrorMessage(error));
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}, [selectedMetadataModel]);
|
||||||
|
|
||||||
|
const suggestDatabaseSensitiveFields = useCallback(async (selected: CatalogDatabase[]) => {
|
||||||
|
if (selected.length !== 1) {
|
||||||
|
toast.error("Sensitive-field suggestions can be requested for only one database at a time. Select one database and try again.");
|
||||||
|
throw new Error("more than one database selected");
|
||||||
|
}
|
||||||
|
const database = selected[0]!;
|
||||||
|
await requestSensitiveSuggestions(database, { scope: "all" }, `Entire database ${database.workspaceName}`);
|
||||||
|
}, [requestSensitiveSuggestions]);
|
||||||
|
|
||||||
|
const suggestActiveDatabaseSensitiveFields = useCallback(async (
|
||||||
|
selection: SensitiveDataSuggestionRequest,
|
||||||
|
scopeLabel: string,
|
||||||
|
) => {
|
||||||
|
if (!activeRow) {
|
||||||
|
toast.error("The database is no longer available, so sensitive-field suggestions were not requested.");
|
||||||
|
throw new Error("database is no longer available");
|
||||||
|
}
|
||||||
|
await requestSensitiveSuggestions(activeRow, selection, scopeLabel);
|
||||||
|
}, [activeRow, requestSensitiveSuggestions]);
|
||||||
|
|
||||||
|
const sensitiveColumnsSaved = useCallback((columns: CatalogColumn[]) => {
|
||||||
|
const savedById = new Map(columns.map((column) => [column.id, column]));
|
||||||
|
setSensitiveReview((current) => current ? {
|
||||||
|
...current,
|
||||||
|
suggestions: current.suggestions.map((suggestion) => {
|
||||||
|
const saved = savedById.get(suggestion.columnId);
|
||||||
|
return saved ? {
|
||||||
|
...suggestion,
|
||||||
|
version: saved.version,
|
||||||
|
currentSensitive: saved.sensitive,
|
||||||
|
sensitive: saved.sensitive,
|
||||||
|
} : suggestion;
|
||||||
|
}),
|
||||||
|
} : null);
|
||||||
|
const databaseId = sensitiveReview?.databaseId;
|
||||||
|
if (databaseId) void queryClient.invalidateQueries({ queryKey: ["catalog-columns", databaseId] });
|
||||||
|
}, [queryClient, sensitiveReview?.databaseId]);
|
||||||
|
|
||||||
const deleteSelectedMetadata = useCallback(async (
|
const deleteSelectedMetadata = useCallback(async (
|
||||||
selected: CatalogDatabase[],
|
selected: CatalogDatabase[],
|
||||||
target: CatalogDatabaseMetadataDeleteTarget,
|
target: CatalogDatabaseMetadataDeleteTarget,
|
||||||
@@ -832,6 +903,7 @@ export function DatabaseManagementPage({
|
|||||||
selectedMetadataModel={selectedMetadataModelAvailable ? selectedMetadataModel : null}
|
selectedMetadataModel={selectedMetadataModelAvailable ? selectedMetadataModel : null}
|
||||||
descriptionGenerationActive={descriptionGenerationActive}
|
descriptionGenerationActive={descriptionGenerationActive}
|
||||||
onGenerateDescriptions={generateDatabaseDescriptions}
|
onGenerateDescriptions={generateDatabaseDescriptions}
|
||||||
|
onSuggestSensitive={suggestDatabaseSensitiveFields}
|
||||||
onDeleteMetadataSelected={deleteSelectedMetadata}
|
onDeleteMetadataSelected={deleteSelectedMetadata}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
@@ -886,6 +958,7 @@ export function DatabaseManagementPage({
|
|||||||
onRunStarted={rememberSyncRun}
|
onRunStarted={rememberSyncRun}
|
||||||
onOpenSync={() => openSync(activeRow)}
|
onOpenSync={() => openSync(activeRow)}
|
||||||
onDescriptionGenerationRunStarted={rememberDescriptionGenerationRun}
|
onDescriptionGenerationRunStarted={rememberDescriptionGenerationRun}
|
||||||
|
onSuggestSensitive={suggestActiveDatabaseSensitiveFields}
|
||||||
/>
|
/>
|
||||||
) : null}
|
) : null}
|
||||||
|
|
||||||
@@ -920,6 +993,15 @@ export function DatabaseManagementPage({
|
|||||||
onRunUpdate={setActiveDescriptionGenerationRun}
|
onRunUpdate={setActiveDescriptionGenerationRun}
|
||||||
onTerminal={(run) => void descriptionGenerationTerminated(run)}
|
onTerminal={(run) => void descriptionGenerationTerminated(run)}
|
||||||
/>
|
/>
|
||||||
|
<SensitiveDataReviewDrawer
|
||||||
|
open={Boolean(sensitiveReview)}
|
||||||
|
databaseId={sensitiveReview?.databaseId ?? null}
|
||||||
|
scopeLabel={sensitiveReview?.scopeLabel ?? ""}
|
||||||
|
suggestions={sensitiveReview?.suggestions ?? []}
|
||||||
|
canManage={canManage}
|
||||||
|
onClose={() => setSensitiveReview(null)}
|
||||||
|
onSaved={sensitiveColumnsSaved}
|
||||||
|
/>
|
||||||
</main>
|
</main>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,11 +11,11 @@ import {
|
|||||||
consolidateCatalogDescriptions,
|
consolidateCatalogDescriptions,
|
||||||
listCatalogColumns,
|
listCatalogColumns,
|
||||||
startDescriptionGenerationRun,
|
startDescriptionGenerationRun,
|
||||||
suggestSensitiveFields,
|
|
||||||
updateCatalogColumnMetadata,
|
updateCatalogColumnMetadata,
|
||||||
type CatalogColumn,
|
type CatalogColumn,
|
||||||
type CatalogTable,
|
type CatalogTable,
|
||||||
type DescriptionGenerationRun,
|
type DescriptionGenerationRun,
|
||||||
|
type SensitiveDataSuggestionRequest,
|
||||||
} from "../../api/catalog-databases";
|
} from "../../api/catalog-databases";
|
||||||
import type { DatabaseNavigationState } from "./model";
|
import type { DatabaseNavigationState } from "./model";
|
||||||
|
|
||||||
@@ -28,6 +28,7 @@ interface Props {
|
|||||||
onDescriptionGenerationRunStarted: (run: DescriptionGenerationRun) => void;
|
onDescriptionGenerationRunStarted: (run: DescriptionGenerationRun) => void;
|
||||||
onNavigationStateChange: (state: DatabaseNavigationState) => void;
|
onNavigationStateChange: (state: DatabaseNavigationState) => void;
|
||||||
onSync: () => void;
|
onSync: () => void;
|
||||||
|
onSuggestSensitive: (selection: SensitiveDataSuggestionRequest, scopeLabel: string) => Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface GridContext {
|
interface GridContext {
|
||||||
@@ -84,6 +85,7 @@ export function DatabaseColumns({
|
|||||||
onDescriptionGenerationRunStarted,
|
onDescriptionGenerationRunStarted,
|
||||||
onNavigationStateChange,
|
onNavigationStateChange,
|
||||||
onSync,
|
onSync,
|
||||||
|
onSuggestSensitive,
|
||||||
}: Props) {
|
}: Props) {
|
||||||
const queryClient = useQueryClient();
|
const queryClient = useQueryClient();
|
||||||
const queryKey = ["catalog-columns", databaseId, table.id] as const;
|
const queryKey = ["catalog-columns", databaseId, table.id] as const;
|
||||||
@@ -222,23 +224,16 @@ export function DatabaseColumns({
|
|||||||
};
|
};
|
||||||
|
|
||||||
const suggestSensitive = async () => {
|
const suggestSensitive = async () => {
|
||||||
if (!selectedMetadataModel) return;
|
if (selectedIds.length === 0) return;
|
||||||
setBusy(true);
|
setBusy(true);
|
||||||
setSensitiveAction("suggest");
|
setSensitiveAction("suggest");
|
||||||
try {
|
try {
|
||||||
const result = await suggestSensitiveFields(databaseId, selectedMetadataModel);
|
await onSuggestSensitive(
|
||||||
const currentById = new Map(data.map((column) => [column.id, column]));
|
{ scope: "selected_columns", targetIds: selectedIds },
|
||||||
const next: Record<string, boolean> = {};
|
`${selectedIds.length} selected column${selectedIds.length === 1 ? "" : "s"} in ${table.name}`,
|
||||||
for (const suggestion of result.suggestions) {
|
);
|
||||||
const column = currentById.get(suggestion.columnId);
|
} catch {
|
||||||
if (column && suggestion.sensitive !== column.sensitive) {
|
// The page-level request reports the safe, specific reason and preserves the selection.
|
||||||
next[column.id] = suggestion.sensitive;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
setSensitiveDrafts(next);
|
|
||||||
toast.success("Sensitive field suggestions ready for review");
|
|
||||||
} catch (error) {
|
|
||||||
toast.error(apiErrorMessage(error));
|
|
||||||
} finally {
|
} finally {
|
||||||
setSensitiveAction(null);
|
setSensitiveAction(null);
|
||||||
setBusy(false);
|
setBusy(false);
|
||||||
@@ -351,6 +346,20 @@ export function DatabaseColumns({
|
|||||||
</Menu.Positioner>
|
</Menu.Positioner>
|
||||||
</Menu.Portal>
|
</Menu.Portal>
|
||||||
</Menu.Root>
|
</Menu.Root>
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
variant="outline"
|
||||||
|
disabled={!canManage || descriptionGenerationActive || busy}
|
||||||
|
title={descriptionGenerationActive ? "Wait for the active description generation to finish" : undefined}
|
||||||
|
onClick={() => void suggestSensitive()}
|
||||||
|
>
|
||||||
|
<Sparkles />{sensitiveAction === "suggest" ? "Suggesting…" : "Suggest sensitive fields"}
|
||||||
|
</Button>
|
||||||
|
{changedSensitiveColumns.length > 0 ? (
|
||||||
|
<Button type="button" disabled={!canManage || busy} onClick={() => void saveSensitive()}>
|
||||||
|
<Save />{sensitiveAction === "save" ? "Saving…" : "Save sensitive fields"}
|
||||||
|
</Button>
|
||||||
|
) : null}
|
||||||
<Button type="button" variant="ghost" onClick={() => { gridRef.current?.api.deselectAll(); setSelectedIds([]); }}><X />Clear</Button>
|
<Button type="button" variant="ghost" onClick={() => { gridRef.current?.api.deselectAll(); setSelectedIds([]); }}><X />Clear</Button>
|
||||||
</>
|
</>
|
||||||
) : (
|
) : (
|
||||||
@@ -358,14 +367,6 @@ export function DatabaseColumns({
|
|||||||
<span className="thot-label whitespace-nowrap">Catalog columns</span>
|
<span className="thot-label whitespace-nowrap">Catalog columns</span>
|
||||||
<input className="h-8 min-w-40 flex-1 rounded-md border border-input bg-background px-2.5 text-sm outline-none focus:border-primary/60 focus:ring-3 focus:ring-ring/15" aria-label="Search columns" placeholder="Search" value={search} onChange={(event) => setSearch(event.target.value)} />
|
<input className="h-8 min-w-40 flex-1 rounded-md border border-input bg-background px-2.5 text-sm outline-none focus:border-primary/60 focus:ring-3 focus:ring-ring/15" aria-label="Search columns" placeholder="Search" value={search} onChange={(event) => setSearch(event.target.value)} />
|
||||||
<span className="text-xs tabular-nums text-muted-foreground">{data.length}</span>
|
<span className="text-xs tabular-nums text-muted-foreground">{data.length}</span>
|
||||||
<Button
|
|
||||||
type="button"
|
|
||||||
variant="outline"
|
|
||||||
disabled={!canManage || !selectedMetadataModel || descriptionGenerationActive || busy}
|
|
||||||
onClick={() => void suggestSensitive()}
|
|
||||||
>
|
|
||||||
<Sparkles />{sensitiveAction === "suggest" ? "Suggesting…" : "Suggest sensitive fields"}
|
|
||||||
</Button>
|
|
||||||
{changedSensitiveColumns.length > 0 ? (
|
{changedSensitiveColumns.length > 0 ? (
|
||||||
<Button type="button" disabled={!canManage || busy} onClick={() => void saveSensitive()}>
|
<Button type="button" disabled={!canManage || busy} onClick={() => void saveSensitive()}>
|
||||||
<Save />{sensitiveAction === "save" ? "Saving…" : "Save sensitive fields"}
|
<Save />{sensitiveAction === "save" ? "Saving…" : "Save sensitive fields"}
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import {
|
|||||||
} from "ag-grid-community";
|
} from "ag-grid-community";
|
||||||
import "ag-grid-community/styles/ag-grid.css";
|
import "ag-grid-community/styles/ag-grid.css";
|
||||||
import "ag-grid-community/styles/ag-theme-alpine.css";
|
import "ag-grid-community/styles/ag-theme-alpine.css";
|
||||||
import { ChevronDown, Eye, Pencil, RefreshCw, Trash2, X } from "lucide-react";
|
import { ChevronDown, Eye, Pencil, RefreshCw, Sparkles, Trash2, X } from "lucide-react";
|
||||||
import { Button } from "../../components/ui/button";
|
import { Button } from "../../components/ui/button";
|
||||||
import type {
|
import type {
|
||||||
CatalogDatabase,
|
CatalogDatabase,
|
||||||
@@ -42,6 +42,7 @@ interface DatabaseGridProps {
|
|||||||
rows: CatalogDatabase[],
|
rows: CatalogDatabase[],
|
||||||
scope: Extract<DescriptionGenerationScope, "all" | "missing">,
|
scope: Extract<DescriptionGenerationScope, "all" | "missing">,
|
||||||
) => Promise<void>;
|
) => Promise<void>;
|
||||||
|
onSuggestSensitive: (rows: CatalogDatabase[]) => Promise<void>;
|
||||||
onDeleteMetadataSelected: (
|
onDeleteMetadataSelected: (
|
||||||
rows: CatalogDatabase[],
|
rows: CatalogDatabase[],
|
||||||
target: CatalogDatabaseMetadataDeleteTarget,
|
target: CatalogDatabaseMetadataDeleteTarget,
|
||||||
@@ -164,13 +165,14 @@ export function DatabaseGrid({
|
|||||||
selectedMetadataModel,
|
selectedMetadataModel,
|
||||||
descriptionGenerationActive,
|
descriptionGenerationActive,
|
||||||
onGenerateDescriptions,
|
onGenerateDescriptions,
|
||||||
|
onSuggestSensitive,
|
||||||
onDeleteMetadataSelected,
|
onDeleteMetadataSelected,
|
||||||
}: DatabaseGridProps) {
|
}: DatabaseGridProps) {
|
||||||
const compact = useCompactViewport();
|
const compact = useCompactViewport();
|
||||||
const gridRef = useRef<AgGridReact<CatalogDatabase>>(null);
|
const gridRef = useRef<AgGridReact<CatalogDatabase>>(null);
|
||||||
const actionsTriggerRef = useRef<HTMLButtonElement>(null);
|
const actionsTriggerRef = useRef<HTMLButtonElement>(null);
|
||||||
const [selectedRows, setSelectedRows] = useState<CatalogDatabase[]>([]);
|
const [selectedRows, setSelectedRows] = useState<CatalogDatabase[]>([]);
|
||||||
const [action, setAction] = useState<"test" | "sync" | "generate" | "delete" | null>(null);
|
const [action, setAction] = useState<"test" | "sync" | "generate" | "suggest" | "delete" | null>(null);
|
||||||
const [pendingDelete, setPendingDelete] = useState<CatalogDatabaseMetadataDeleteTarget | null>(null);
|
const [pendingDelete, setPendingDelete] = useState<CatalogDatabaseMetadataDeleteTarget | null>(null);
|
||||||
const [pendingGenerateAll, setPendingGenerateAll] = useState(false);
|
const [pendingGenerateAll, setPendingGenerateAll] = useState(false);
|
||||||
const context = useMemo<DatabaseGridContext>(
|
const context = useMemo<DatabaseGridContext>(
|
||||||
@@ -283,12 +285,18 @@ export function DatabaseGrid({
|
|||||||
setPendingGenerateAll(false);
|
setPendingGenerateAll(false);
|
||||||
window.setTimeout(() => actionsTriggerRef.current?.focus(), 0);
|
window.setTimeout(() => actionsTriggerRef.current?.focus(), 0);
|
||||||
};
|
};
|
||||||
const perform = async (kind: "test" | "sync" | "generate" | "delete", operation: () => Promise<void>) => {
|
const perform = async (
|
||||||
|
kind: "test" | "sync" | "generate" | "suggest" | "delete",
|
||||||
|
operation: () => Promise<void>,
|
||||||
|
clearSelection = true,
|
||||||
|
) => {
|
||||||
setAction(kind);
|
setAction(kind);
|
||||||
try {
|
try {
|
||||||
await operation();
|
await operation();
|
||||||
|
if (clearSelection) {
|
||||||
gridRef.current?.api.deselectAll();
|
gridRef.current?.api.deselectAll();
|
||||||
setSelectedRows([]);
|
setSelectedRows([]);
|
||||||
|
}
|
||||||
setPendingDelete(null);
|
setPendingDelete(null);
|
||||||
setPendingGenerateAll(false);
|
setPendingGenerateAll(false);
|
||||||
if (kind === "delete") window.setTimeout(() => searchInputRef.current?.focus(), 0);
|
if (kind === "delete") window.setTimeout(() => searchInputRef.current?.focus(), 0);
|
||||||
@@ -422,6 +430,15 @@ export function DatabaseGrid({
|
|||||||
</Menu.Positioner>
|
</Menu.Positioner>
|
||||||
</Menu.Portal>
|
</Menu.Portal>
|
||||||
</Menu.Root>
|
</Menu.Root>
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
variant="outline"
|
||||||
|
disabled={!canManage || action !== null || descriptionGenerationActive}
|
||||||
|
title={descriptionGenerationActive ? "Wait for the active description generation to finish" : undefined}
|
||||||
|
onClick={() => void perform("suggest", () => onSuggestSensitive(selectedRows), false)}
|
||||||
|
>
|
||||||
|
<Sparkles />{action === "suggest" ? "Suggesting…" : "Suggest sensitive fields"}
|
||||||
|
</Button>
|
||||||
<Button type="button" variant="ghost" disabled={action !== null} onClick={() => { gridRef.current?.api.deselectAll(); setSelectedRows([]); }}><X />Clear</Button>
|
<Button type="button" variant="ghost" disabled={action !== null} onClick={() => { gridRef.current?.api.deselectAll(); setSelectedRows([]); }}><X />Clear</Button>
|
||||||
</>
|
</>
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import { Menu } from "@base-ui/react/menu";
|
|||||||
import { useQuery, useQueryClient } from "@tanstack/react-query";
|
import { useQuery, useQueryClient } from "@tanstack/react-query";
|
||||||
import { AgGridReact } from "ag-grid-react";
|
import { AgGridReact } from "ag-grid-react";
|
||||||
import type { ColDef, ICellRendererParams } from "ag-grid-community";
|
import type { ColDef, ICellRendererParams } from "ag-grid-community";
|
||||||
import { ArrowLeft, ChevronDown, Columns3, Pencil, RefreshCw, Save, Trash2, X } from "lucide-react";
|
import { ArrowLeft, ChevronDown, Columns3, Pencil, RefreshCw, Save, Sparkles, Trash2, X } from "lucide-react";
|
||||||
import { toast } from "sonner";
|
import { toast } from "sonner";
|
||||||
import { Button } from "../../components/ui/button";
|
import { Button } from "../../components/ui/button";
|
||||||
import { ApiError, apiErrorMessage } from "../../api/client";
|
import { ApiError, apiErrorMessage } from "../../api/client";
|
||||||
@@ -19,6 +19,7 @@ import {
|
|||||||
type CatalogTable,
|
type CatalogTable,
|
||||||
type CatalogTableMetadataDeleteTarget,
|
type CatalogTableMetadataDeleteTarget,
|
||||||
type DescriptionGenerationRun,
|
type DescriptionGenerationRun,
|
||||||
|
type SensitiveDataSuggestionRequest,
|
||||||
} from "../../api/catalog-databases";
|
} from "../../api/catalog-databases";
|
||||||
import type { DatabaseNavigationState } from "./model";
|
import type { DatabaseNavigationState } from "./model";
|
||||||
import { DatabaseColumns } from "./DatabaseColumns";
|
import { DatabaseColumns } from "./DatabaseColumns";
|
||||||
@@ -36,6 +37,7 @@ interface Props {
|
|||||||
onRunStarted: (run: CatalogSyncRun) => void;
|
onRunStarted: (run: CatalogSyncRun) => void;
|
||||||
onOpenSync: () => void;
|
onOpenSync: () => void;
|
||||||
onDescriptionGenerationRunStarted: (run: DescriptionGenerationRun) => void;
|
onDescriptionGenerationRunStarted: (run: DescriptionGenerationRun) => void;
|
||||||
|
onSuggestSensitive: (selection: SensitiveDataSuggestionRequest, scopeLabel: string) => Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface TableGridContext {
|
interface TableGridContext {
|
||||||
@@ -70,6 +72,7 @@ export function DatabaseTables({
|
|||||||
onRunStarted,
|
onRunStarted,
|
||||||
onOpenSync,
|
onOpenSync,
|
||||||
onDescriptionGenerationRunStarted,
|
onDescriptionGenerationRunStarted,
|
||||||
|
onSuggestSensitive,
|
||||||
}: Props) {
|
}: Props) {
|
||||||
const databaseId = database.id!;
|
const databaseId = database.id!;
|
||||||
const queryClient = useQueryClient();
|
const queryClient = useQueryClient();
|
||||||
@@ -86,7 +89,7 @@ export function DatabaseTables({
|
|||||||
const [editorVersion, setEditorVersion] = useState<number | null>(null);
|
const [editorVersion, setEditorVersion] = useState<number | null>(null);
|
||||||
const [stale, setStale] = useState(false);
|
const [stale, setStale] = useState(false);
|
||||||
const [staleBannerOpen, setStaleBannerOpen] = useState(true);
|
const [staleBannerOpen, setStaleBannerOpen] = useState(true);
|
||||||
const [busy, setBusy] = useState<"sync" | "save" | "delete" | "consolidate" | "generate" | null>(null);
|
const [busy, setBusy] = useState<"sync" | "save" | "delete" | "consolidate" | "generate" | "suggest" | null>(null);
|
||||||
const [pendingDelete, setPendingDelete] = useState<CatalogTableMetadataDeleteTarget | null>(null);
|
const [pendingDelete, setPendingDelete] = useState<CatalogTableMetadataDeleteTarget | null>(null);
|
||||||
const [columnNavigation, setColumnNavigation] = useState<DatabaseNavigationState>({ dirty: false, busy: false });
|
const [columnNavigation, setColumnNavigation] = useState<DatabaseNavigationState>({ dirty: false, busy: false });
|
||||||
const gridRef = useRef<AgGridReact<CatalogTable>>(null);
|
const gridRef = useRef<AgGridReact<CatalogTable>>(null);
|
||||||
@@ -240,6 +243,20 @@ export function DatabaseTables({
|
|||||||
toast.error(apiErrorMessage(error));
|
toast.error(apiErrorMessage(error));
|
||||||
} finally { setBusy(null); }
|
} finally { setBusy(null); }
|
||||||
};
|
};
|
||||||
|
const suggestSensitive = async () => {
|
||||||
|
if (selectedIds.length === 0) return;
|
||||||
|
setBusy("suggest");
|
||||||
|
try {
|
||||||
|
await onSuggestSensitive(
|
||||||
|
{ scope: "selected_tables", targetIds: selectedIds },
|
||||||
|
`${selectedIds.length} selected table${selectedIds.length === 1 ? "" : "s"}`,
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
// The page-level request reports the safe, specific reason and preserves the selection.
|
||||||
|
} finally {
|
||||||
|
setBusy(null);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const columns = useMemo<ColDef<CatalogTable>[]>(() => [
|
const columns = useMemo<ColDef<CatalogTable>[]>(() => [
|
||||||
{ field: "name", headerName: "Name", minWidth: 250, flex: 1, cellClass: "font-mono text-xs" },
|
{ field: "name", headerName: "Name", minWidth: 250, flex: 1, cellClass: "font-mono text-xs" },
|
||||||
@@ -303,6 +320,7 @@ export function DatabaseTables({
|
|||||||
onDescriptionGenerationRunStarted={onDescriptionGenerationRunStarted}
|
onDescriptionGenerationRunStarted={onDescriptionGenerationRunStarted}
|
||||||
onNavigationStateChange={setColumnNavigation}
|
onNavigationStateChange={setColumnNavigation}
|
||||||
onSync={() => void synchronize("columns", [activeTable.id])}
|
onSync={() => void synchronize("columns", [activeTable.id])}
|
||||||
|
onSuggestSensitive={onSuggestSensitive}
|
||||||
/>
|
/>
|
||||||
) : (
|
) : (
|
||||||
<div className="min-h-0 overflow-y-auto px-4 py-5">
|
<div className="min-h-0 overflow-y-auto px-4 py-5">
|
||||||
@@ -420,6 +438,15 @@ export function DatabaseTables({
|
|||||||
</Menu.Positioner>
|
</Menu.Positioner>
|
||||||
</Menu.Portal>
|
</Menu.Portal>
|
||||||
</Menu.Root>
|
</Menu.Root>
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
variant="outline"
|
||||||
|
disabled={!canManage || descriptionGenerationActive || busy !== null}
|
||||||
|
title={descriptionGenerationActive ? "Wait for the active description generation to finish" : undefined}
|
||||||
|
onClick={() => void suggestSensitive()}
|
||||||
|
>
|
||||||
|
<Sparkles />{busy === "suggest" ? "Suggesting…" : "Suggest sensitive fields"}
|
||||||
|
</Button>
|
||||||
<Button type="button" variant="ghost" onClick={() => { gridRef.current?.api.deselectAll(); setSelectedIds([]); }}><X />Clear</Button>
|
<Button type="button" variant="ghost" onClick={() => { gridRef.current?.api.deselectAll(); setSelectedIds([]); }}><X />Clear</Button>
|
||||||
</>
|
</>
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,188 @@
|
|||||||
|
import { useEffect, useMemo, useState } from "react";
|
||||||
|
import { Save, X } from "lucide-react";
|
||||||
|
import { toast } from "sonner";
|
||||||
|
import { Button } from "../../components/ui/button";
|
||||||
|
import { apiErrorMessage } from "../../api/client";
|
||||||
|
import {
|
||||||
|
updateCatalogColumnSensitive,
|
||||||
|
type CatalogColumn,
|
||||||
|
type SensitiveDataSuggestion,
|
||||||
|
} from "../../api/catalog-databases";
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
open: boolean;
|
||||||
|
databaseId: string | null;
|
||||||
|
scopeLabel: string;
|
||||||
|
suggestions: SensitiveDataSuggestion[];
|
||||||
|
canManage: boolean;
|
||||||
|
onClose: () => void;
|
||||||
|
onSaved: (columns: CatalogColumn[]) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function SensitiveDataReviewDrawer({
|
||||||
|
open,
|
||||||
|
databaseId,
|
||||||
|
scopeLabel,
|
||||||
|
suggestions,
|
||||||
|
canManage,
|
||||||
|
onClose,
|
||||||
|
onSaved,
|
||||||
|
}: Props) {
|
||||||
|
const [drafts, setDrafts] = useState<Record<string, boolean>>({});
|
||||||
|
const [search, setSearch] = useState("");
|
||||||
|
const [showAll, setShowAll] = useState(false);
|
||||||
|
const [saving, setSaving] = useState(false);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!open) return;
|
||||||
|
setDrafts(Object.fromEntries(suggestions.map((suggestion) => [
|
||||||
|
suggestion.columnId,
|
||||||
|
suggestion.sensitive,
|
||||||
|
])));
|
||||||
|
setSearch("");
|
||||||
|
setShowAll(false);
|
||||||
|
}, [open, suggestions]);
|
||||||
|
|
||||||
|
const changed = useMemo(() => suggestions.filter((suggestion) => (
|
||||||
|
drafts[suggestion.columnId] !== undefined
|
||||||
|
&& drafts[suggestion.columnId] !== suggestion.currentSensitive
|
||||||
|
)), [drafts, suggestions]);
|
||||||
|
const visible = useMemo(() => {
|
||||||
|
const term = search.trim().toLocaleLowerCase();
|
||||||
|
return suggestions.filter((suggestion) => (
|
||||||
|
(showAll || drafts[suggestion.columnId] !== suggestion.currentSensitive)
|
||||||
|
&& (!term || `${suggestion.tableName}.${suggestion.columnName}`.toLocaleLowerCase().includes(term))
|
||||||
|
));
|
||||||
|
}, [drafts, search, showAll, suggestions]);
|
||||||
|
|
||||||
|
const close = () => {
|
||||||
|
if (saving) return;
|
||||||
|
if (changed.length > 0 && !window.confirm("Discard the sensitive-field review?")) return;
|
||||||
|
onClose();
|
||||||
|
};
|
||||||
|
|
||||||
|
const save = async () => {
|
||||||
|
if (!databaseId || changed.length === 0) return;
|
||||||
|
setSaving(true);
|
||||||
|
const results = await Promise.allSettled(changed.map((suggestion) => (
|
||||||
|
updateCatalogColumnSensitive(
|
||||||
|
databaseId,
|
||||||
|
suggestion.tableId,
|
||||||
|
suggestion.columnId,
|
||||||
|
suggestion.version,
|
||||||
|
drafts[suggestion.columnId]!,
|
||||||
|
)
|
||||||
|
)));
|
||||||
|
const saved = results.flatMap((result) => result.status === "fulfilled" ? [result.value] : []);
|
||||||
|
const failed = results.flatMap((result) => result.status === "rejected" ? [result.reason] : []);
|
||||||
|
if (saved.length > 0) onSaved(saved);
|
||||||
|
if (failed.length > 0) {
|
||||||
|
toast.error(`${failed.length} sensitive flag${failed.length === 1 ? "" : "s"} could not be saved: ${apiErrorMessage(failed[0])}`);
|
||||||
|
} else {
|
||||||
|
toast.success(`Saved ${saved.length} sensitive flag${saved.length === 1 ? "" : "s"}`);
|
||||||
|
onClose();
|
||||||
|
}
|
||||||
|
setSaving(false);
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!open || !databaseId) return null;
|
||||||
|
return (
|
||||||
|
<aside
|
||||||
|
aria-label="Sensitive field review"
|
||||||
|
className="fixed inset-y-2 right-0 z-50 flex w-full max-w-[480px] flex-col border-l border-border bg-background shadow-2xl sm:inset-y-4"
|
||||||
|
>
|
||||||
|
<div className="flex items-start justify-between gap-4 border-b border-border px-5 py-4">
|
||||||
|
<div>
|
||||||
|
<p className="thot-label">Sensitive data</p>
|
||||||
|
<h2 className="mt-1 font-heading text-xl font-semibold">Review suggested flags</h2>
|
||||||
|
<p className="mt-1 text-sm text-muted-foreground">
|
||||||
|
{scopeLabel}. The model proposed values, but only your save changes the catalog.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<Button type="button" variant="ghost" size="icon-lg" aria-label="Close sensitive field review" disabled={saving} onClick={close}>
|
||||||
|
<X aria-hidden="true" />
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex min-h-0 flex-1 flex-col">
|
||||||
|
<div className="border-b border-border px-5 py-4">
|
||||||
|
<div className="flex items-center gap-3">
|
||||||
|
<input
|
||||||
|
className="h-9 min-w-0 flex-1 rounded-md border border-input bg-background px-3 text-sm outline-none focus:border-primary/60 focus:ring-3 focus:ring-ring/15"
|
||||||
|
aria-label="Search sensitive field suggestions"
|
||||||
|
placeholder="Search table or column"
|
||||||
|
value={search}
|
||||||
|
onChange={(event) => setSearch(event.target.value)}
|
||||||
|
/>
|
||||||
|
<span className="whitespace-nowrap text-xs tabular-nums text-muted-foreground">
|
||||||
|
{changed.length} change{changed.length === 1 ? "" : "s"}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<label className="mt-3 inline-flex items-center gap-2 text-sm text-muted-foreground">
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
className="size-4 accent-primary outline-none focus-visible:ring-3 focus-visible:ring-ring/50"
|
||||||
|
checked={showAll}
|
||||||
|
onChange={(event) => setShowAll(event.target.checked)}
|
||||||
|
/>
|
||||||
|
Show all {suggestions.length} classified columns
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="min-h-0 flex-1 overflow-y-auto px-5 py-4">
|
||||||
|
{visible.length === 0 ? (
|
||||||
|
<div className="rounded-md border border-border bg-muted/25 px-4 py-5 text-sm">
|
||||||
|
<p className="font-semibold">No proposed changes in this view.</p>
|
||||||
|
<p className="mt-1 text-muted-foreground">
|
||||||
|
Show all classified columns to inspect unchanged flags, or close this review.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<ul className="divide-y divide-border" aria-label="Sensitive field suggestions">
|
||||||
|
{visible.map((suggestion) => {
|
||||||
|
const proposed = drafts[suggestion.columnId] ?? suggestion.sensitive;
|
||||||
|
const changedFromCurrent = proposed !== suggestion.currentSensitive;
|
||||||
|
return (
|
||||||
|
<li key={suggestion.columnId} className="flex items-start gap-3 py-3">
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
className="mt-1 size-4 shrink-0 accent-primary outline-none focus-visible:ring-3 focus-visible:ring-ring/50"
|
||||||
|
aria-label={`Protect ${suggestion.tableName}.${suggestion.columnName}`}
|
||||||
|
checked={proposed}
|
||||||
|
disabled={!canManage || saving}
|
||||||
|
onChange={(event) => setDrafts((current) => ({
|
||||||
|
...current,
|
||||||
|
[suggestion.columnId]: event.target.checked,
|
||||||
|
}))}
|
||||||
|
/>
|
||||||
|
<div className="min-w-0 flex-1">
|
||||||
|
<p className="truncate font-mono text-xs font-semibold">
|
||||||
|
{suggestion.tableName}.{suggestion.columnName}
|
||||||
|
</p>
|
||||||
|
<p className="mt-1 text-xs text-muted-foreground">
|
||||||
|
Current: {suggestion.currentSensitive ? "protected" : "allowed"}. Proposed: {proposed ? "protected" : "allowed"}.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<span className={`rounded px-2 py-0.5 text-[11px] font-semibold ${changedFromCurrent ? "bg-amber-500/12 text-amber-800 dark:text-amber-300" : "bg-muted text-muted-foreground"}`}>
|
||||||
|
{changedFromCurrent ? "Change" : "No change"}
|
||||||
|
</span>
|
||||||
|
</li>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</ul>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex items-center justify-between gap-3 border-t border-border px-5 py-4">
|
||||||
|
<p className="text-xs text-muted-foreground">Unsaved suggestions never change the catalog.</p>
|
||||||
|
<div className="flex gap-2">
|
||||||
|
<Button type="button" variant="outline" disabled={saving} onClick={close}>Cancel</Button>
|
||||||
|
<Button type="button" disabled={!canManage || saving || changed.length === 0} onClick={() => void save()}>
|
||||||
|
<Save />{saving ? "Saving…" : `Save ${changed.length}`}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</aside>
|
||||||
|
);
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user