mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: classify sensitive columns locally

Browse Source
This commit is contained in:
Codex
2026-09-03 02:11:13 +02:00
parent 7b87e95427
commit f114d0065a
57 changed files with 4038 additions and 1149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/api/catalog-databases.ts
+32 -22
View File
@@ -126,7 +126,7 @@ export interface CatalogColumn {
updatedAt: string;
}
export interface SensitiveDataSuggestion {
export interface SensitivityReviewItem {
columnId: string;
tableId: string;
tableName: string;
@@ -134,29 +134,40 @@ export interface SensitiveDataSuggestion {
version: number;
currentSensitive: boolean;
sensitive: boolean;
assessment: "sensitive" | "non_sensitive" | "unknown";
evidence: Array<{
kind: "metadata" | "content" | "length" | "ner" | "coverage";
ruleId: string;
label?: string;
confidence?: number;
}>;
observedValues: number;
}
export interface SensitiveDataSuggestions {
suggestions: SensitiveDataSuggestion[];
run: SensitiveDataSuggestionRun;
export interface SensitivityAnalysisResult {
suggestions: SensitivityReviewItem[];
run: SensitivityAnalysisRun;
}
export type SensitiveDataSuggestionRequest =
export type SensitivityAnalysisRequest =
| { scope: "all" }
| { scope: "selected_tables"; targetIds: string[] }
| { scope: "selected_columns"; targetIds: string[] };
export type SensitiveDataSuggestionStatus = "running" | "completed" | "failed" | "interrupted";
export type SensitivityAnalysisStatus = "running" | "completed" | "failed" | "interrupted";
export interface SensitiveDataSuggestionRun {
export interface SensitivityAnalysisRun {
id: string;
databaseId: string;
scope: SensitiveDataSuggestionRequest["scope"];
modelId: string;
status: SensitiveDataSuggestionStatus;
scope: SensitivityAnalysisRequest["scope"];
engine: "llm" | "local";
modelId: string | null;
policyVersion: string | null;
status: SensitivityAnalysisStatus;
total: number;
suggestedSensitive: number;
suggestedNonSensitive: number;
unknown: number;
inputTokens?: number;
cacheReadTokens?: number;
outputTokens?: number;
@@ -167,7 +178,7 @@ export interface SensitiveDataSuggestionRun {
errorSummary: string | null;
}
export interface SensitiveDataSuggestionEvent {
export interface SensitivityAnalysisEvent {
runId: string;
sequence: number;
level: "info" | "warning" | "error";
@@ -458,28 +469,27 @@ export const updateCatalogColumnSensitive = (
{ method: "PATCH", body: JSON.stringify({ version, sensitive }) },
);
export const suggestSensitiveFields = (
export const runSensitivityAnalysis = (
databaseId: string,
modelId: string,
selection: SensitiveDataSuggestionRequest,
selection: SensitivityAnalysisRequest,
) =>
apiFetch<SensitiveDataSuggestions>(
apiFetch<SensitivityAnalysisResult>(
`/catalog/databases/${encodeURIComponent(databaseId)}/sensitive-data-suggestions`,
{ method: "POST", body: JSON.stringify({ modelId, ...selection }) },
{ method: "POST", body: JSON.stringify(selection) },
);
export const getSensitiveDataSuggestionRun = (runId: string) =>
apiFetch<SensitiveDataSuggestionRun>(
export const getSensitivityAnalysisRun = (runId: string) =>
apiFetch<SensitivityAnalysisRun>(
`/catalog/sensitive-data-suggestion-runs/${encodeURIComponent(runId)}`,
);
export const listSensitiveDataSuggestionRuns = (limit = 50) =>
apiFetch<SensitiveDataSuggestionRun[]>(
export const listSensitivityAnalysisRuns = (limit = 50) =>
apiFetch<SensitivityAnalysisRun[]>(
`/catalog/sensitive-data-suggestion-runs?limit=${encodeURIComponent(String(limit))}`,
);
export const listSensitiveDataSuggestionEvents = (runId: string, after = 0) =>
apiFetch<SensitiveDataSuggestionEvent[]>(
export const listSensitivityAnalysisEvents = (runId: string, after = 0) =>
apiFetch<SensitivityAnalysisEvent[]>(
`/catalog/sensitive-data-suggestion-runs/${encodeURIComponent(runId)}/events-list?after=${after}`,
);
frontend/src/api/catalog-description-generation.test.ts
+12 -9
View File
@@ -3,13 +3,13 @@ import { server } from "../test/msw";
import {
cancelDescriptionGenerationRun,
descriptionGenerationEventsUrl,
getSensitiveDataSuggestionRun,
getSensitivityAnalysisRun,
listDescriptionGenerationRuns,
listSensitiveDataSuggestionEvents,
listSensitiveDataSuggestionRuns,
listSensitivityAnalysisEvents,
listSensitivityAnalysisRuns,
unlockDescriptionGenerationRun,
type DescriptionGenerationRun,
type SensitiveDataSuggestionRun,
type SensitivityAnalysisRun,
} from "./catalog-databases";
const historicalRun: DescriptionGenerationRun = {
@@ -31,15 +31,18 @@ const historicalRun: DescriptionGenerationRun = {
errorSummary: null,
};
const sensitiveSuggestionRun: SensitiveDataSuggestionRun = {
const sensitiveSuggestionRun: SensitivityAnalysisRun = {
id: "99999999-9999-4999-8999-999999999999",
databaseId: "11111111-1111-4111-8111-111111111111",
scope: "selected_columns",
modelId: "local-qwen",
engine: "local",
modelId: null,
policyVersion: "sensitivity-v1",
status: "completed",
total: 4,
suggestedSensitive: 2,
suggestedNonSensitive: 2,
unknown: 0,
createdAt: "2026-08-28T09:00:00Z",
startedAt: "2026-08-28T09:00:00Z",
updatedAt: "2026-08-28T09:00:01Z",
@@ -119,9 +122,9 @@ test("lists, reads, and replays persisted sensitive-suggestion history", async (
}),
);
await expect(listSensitiveDataSuggestionRuns(50)).resolves.toEqual([sensitiveSuggestionRun]);
await expect(getSensitiveDataSuggestionRun(sensitiveSuggestionRun.id)).resolves.toEqual(sensitiveSuggestionRun);
await expect(listSensitiveDataSuggestionEvents(sensitiveSuggestionRun.id, 2)).resolves.toEqual([event]);
await expect(listSensitivityAnalysisRuns(50)).resolves.toEqual([sensitiveSuggestionRun]);
await expect(getSensitivityAnalysisRun(sensitiveSuggestionRun.id)).resolves.toEqual(sensitiveSuggestionRun);
await expect(listSensitivityAnalysisEvents(sensitiveSuggestionRun.id, 2)).resolves.toEqual([event]);
expect(requestedLimit).toBe("50");
expect(requestedAfter).toBe("2");
});
frontend/src/api/client.test.ts
+5 -5
View File
@@ -147,11 +147,11 @@ test.each([
["description_generation_target_ids_duplicate", "Description generation target IDs must be unique."],
["description_generation_no_eligible_targets", "No eligible catalog tables or columns need description generation."],
["catalog_table_not_found", "One or more selected catalog tables were not found."],
["sensitive_data_suggestion_invalid_response", "The model returned an incomplete or invalid classification. No suggestions were applied."],
["sensitive_data_suggestion_provider_unavailable", "The selected model could not complete the request. No suggestions were applied."],
["sensitive_data_suggestion_history_request_invalid", "Sensitive suggestion history parameters are invalid."],
["sensitive_data_suggestion_history_failed", "Sensitive suggestion history could not be loaded."],
["sensitive_data_suggestion_run_not_found", "The sensitive suggestion run was not found."],
["sensitivity_source_unavailable", "The database content could not be read for sensitivity analysis. No assessments were applied."],
["sensitivity_analysis_timeout", "Sensitivity analysis reached its time limit. No assessments were applied."],
["sensitive_data_suggestion_history_request_invalid", "Sensitivity analysis history parameters are invalid."],
["sensitive_data_suggestion_history_failed", "Sensitivity analysis history could not be loaded."],
["sensitive_data_suggestion_run_not_found", "The sensitivity analysis run was not found."],
["relationship_not_found", "The relationship no longer exists. Refresh and try again."],
["relationship_duplicate", "This relationship already exists."],
["relationship_target_not_unique", "The target column must be the only primary-key column of its table."],
frontend/src/api/client.ts
+10 -12
View File
@@ -26,9 +26,8 @@ const safeErrorCodes = new Set([
"sensitive_data_suggestion_request_invalid",
"sensitive_data_suggestion_target_ids_duplicate",
"sensitive_data_suggestion_no_columns",
"sensitive_data_suggestion_payload_too_large",
"sensitive_data_suggestion_invalid_response",
"sensitive_data_suggestion_provider_unavailable",
"sensitivity_source_unavailable",
"sensitivity_analysis_timeout",
"sensitive_data_suggestion_failed",
"sensitive_data_suggestion_history_request_invalid",
"sensitive_data_suggestion_history_failed",
@@ -90,16 +89,15 @@ const localCodeMessages: Record<string, string> = {
catalog_column_not_found: "The selected catalog column was not found.",
catalog_table_not_found: "One or more selected catalog tables were not found.",
workspace_configuration_unavailable: "The database workspace configuration is unavailable.",
sensitive_data_suggestion_request_invalid: "Select a database, one or more tables, or one or more columns before requesting sensitive-field suggestions.",
sensitive_data_suggestion_request_invalid: "Select a database, one or more tables, or one or more columns before running sensitivity analysis.",
sensitive_data_suggestion_target_ids_duplicate: "Each selected table or column can be included only once.",
sensitive_data_suggestion_no_columns: "The selected scope contains no catalog columns to classify.",
sensitive_data_suggestion_payload_too_large: "The selected structural metadata cannot be divided into safe model requests.",
sensitive_data_suggestion_invalid_response: "The model returned an incomplete or invalid classification. No suggestions were applied.",
sensitive_data_suggestion_provider_unavailable: "The selected model could not complete the request. No suggestions were applied.",
sensitive_data_suggestion_failed: "Sensitive-field suggestions failed before review. No changes were applied.",
sensitive_data_suggestion_history_request_invalid: "Sensitive suggestion history parameters are invalid.",
sensitive_data_suggestion_history_failed: "Sensitive suggestion history could not be loaded.",
sensitive_data_suggestion_run_not_found: "The sensitive suggestion run was not found.",
sensitive_data_suggestion_no_columns: "The selected scope contains no catalog columns to assess.",
sensitivity_source_unavailable: "The database content could not be read for sensitivity analysis. No assessments were applied.",
sensitivity_analysis_timeout: "Sensitivity analysis reached its time limit. No assessments were applied.",
sensitive_data_suggestion_failed: "Sensitivity analysis failed before review. No changes were applied.",
sensitive_data_suggestion_history_request_invalid: "Sensitivity analysis history parameters are invalid.",
sensitive_data_suggestion_history_failed: "Sensitivity analysis history could not be loaded.",
sensitive_data_suggestion_run_not_found: "The sensitivity analysis run was not found.",
schema_sync_conflict: "A schema synchronization is already active or no longer current.",
schema_introspection_failed: "The database schema could not be read safely.",
schema_request_invalid: "The schema request is invalid.",