mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: classify sensitive columns locally

Browse Source
This commit is contained in:
Codex
2026-09-03 02:11:13 +02:00
parent 7b87e95427
commit f114d0065a
57 changed files with 4038 additions and 1149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/routes/catalog-description-generation.ts
+73 -59
View File
@@ -11,38 +11,35 @@ import {
type DescriptionGenerationWorker,
} from "../catalog/description-generation-worker.js";
import { MetadataGenerationModelUnavailableError } from "../catalog/metadata-generation-models.js";
import { ModelCompletionProviderError } from "../catalog/model-completer.js";
import {
SensitiveDataSuggestionDuplicateTargetIdsError,
SensitiveDataSuggestionInvalidResponseError,
SensitiveDataSuggestionNoEligibleColumnsError,
SensitiveDataSuggestionPayloadTooLargeError,
SensitiveDataSuggestionTargetNotFoundError,
} from "../catalog/sensitive-data-suggester.js";
import type { SensitiveDataSuggestionRunner } from "../catalog/sensitive-data-suggestion-runner.js";
SensitivityAnalysisDuplicateTargetIdsError,
SensitivityAnalysisInterruptedError,
SensitivityAnalysisNoEligibleColumnsError,
SensitivityAnalysisTargetNotFoundError,
} from "../catalog/sensitivity-analysis-service.js";
import type { SensitivityAnalysisRunner } from "../catalog/sensitivity-analysis-runner.js";
import {
CatalogOperationInProgressError,
CatalogConnectorError,
CatalogUnavailableError,
DescriptionGenerationRunActiveError,
type CatalogRepository,
type DescriptionGenerationEvent,
type DescriptionGenerationRun,
type SensitiveDataSuggestionEvent,
type SensitiveDataSuggestionRun,
type SensitivityAnalysisEvent,
type SensitivityAnalysisRun,
} from "../catalog/types.js";
const idSchema = z.uuid();
const modelIdSchema = z.string().regex(/^[a-z][a-z0-9._-]{0,63}\/[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/);
const selectedTargetIdsSchema = z.array(idSchema).min(1);
const suggestionSchema = z.discriminatedUnion("scope", [
z.object({ modelId: modelIdSchema, scope: z.literal("all") }).strict(),
z.object({ scope: z.literal("all") }).strict(),
z.object({
modelId: modelIdSchema,
scope: z.literal("selected_tables"),
targetIds: selectedTargetIdsSchema,
}).strict(),
z.object({
modelId: modelIdSchema,
scope: z.literal("selected_columns"),
targetIds: selectedTargetIdsSchema,
}).strict(),
@@ -112,7 +109,7 @@ function publicRun(run: DescriptionGenerationRun) {
};
}
function publicSensitiveDataSuggestionEvent(event: SensitiveDataSuggestionEvent) {
function publicSensitivityAnalysisEvent(event: SensitivityAnalysisEvent) {
return {
runId: event.runId,
sequence: event.sequence,
@@ -122,16 +119,19 @@ function publicSensitiveDataSuggestionEvent(event: SensitiveDataSuggestionEvent)
};
}
function publicSensitiveDataSuggestionRun(run: SensitiveDataSuggestionRun) {
function publicSensitivityAnalysisRun(run: SensitivityAnalysisRun) {
return {
id: run.id,
databaseId: run.databaseId,
scope: run.scope,
engine: run.engine,
modelId: run.modelId,
policyVersion: run.policyVersion,
status: run.status,
total: run.total,
suggestedSensitive: run.suggestedSensitive,
suggestedNonSensitive: run.suggestedNonSensitive,
unknown: run.unknown,
inputTokens: run.inputTokens,
cacheReadTokens: run.cacheReadTokens,
outputTokens: run.outputTokens,
@@ -232,16 +232,10 @@ function safeSuggestionError(reply: FastifyReply, error: unknown) {
if (error instanceof CatalogUnavailableError) {
return reply.code(503).send({
code: "catalog_unavailable",
message: "The database catalog is unavailable, so no sensitive-field suggestions were prepared.",
message: "The database catalog is unavailable, so no sensitivity assessments were prepared.",
});
}
if (error instanceof MetadataGenerationModelUnavailableError) {
return reply.code(409).send({
code: "metadata_generation_model_unavailable",
message: "The selected metadata-generation model is unavailable.",
});
}
if (error instanceof SensitiveDataSuggestionTargetNotFoundError) {
if (error instanceof SensitivityAnalysisTargetNotFoundError) {
const code = error.target === "database"
? "database_not_found"
: error.target === "table"
@@ -254,45 +248,65 @@ function safeSuggestionError(reply: FastifyReply, error: unknown) {
: "One or more selected Catalog Columns were not found in this database.";
return reply.code(404).send({ code, message });
}
if (error instanceof SensitiveDataSuggestionDuplicateTargetIdsError) {
if (error instanceof SensitivityAnalysisDuplicateTargetIdsError) {
return reply.code(400).send({
code: "sensitive_data_suggestion_target_ids_duplicate",
message: "Each selected table or column must appear only once.",
});
}
if (error instanceof SensitiveDataSuggestionNoEligibleColumnsError) {
if (error instanceof SensitivityAnalysisNoEligibleColumnsError) {
return reply.code(409).send({
code: "sensitive_data_suggestion_no_columns",
message: "The selected scope contains no Catalog Columns to classify.",
message: "The selected scope contains no Catalog Columns to assess.",
});
}
if (error instanceof SensitiveDataSuggestionPayloadTooLargeError) {
return reply.code(413).send({
code: "sensitive_data_suggestion_payload_too_large",
message: "The selected structural metadata cannot be divided into safe LLM requests.",
if (error instanceof SensitivityAnalysisInterruptedError) {
return reply.code(504).send({
code: "sensitivity_analysis_timeout",
message: "Sensitivity analysis reached its time limit. No assessments were applied.",
});
}
if (error instanceof SensitiveDataSuggestionInvalidResponseError) {
if (error instanceof CatalogConnectorError) {
return reply.code(502).send({
code: "sensitive_data_suggestion_invalid_response",
message: "The LLM returned an incomplete or invalid classification. No suggestions were applied.",
});
}
if (error instanceof ModelCompletionProviderError) {
return reply.code(502).send({
code: "sensitive_data_suggestion_provider_unavailable",
message: "The selected LLM service could not complete the request. No suggestions were applied.",
code: "sensitivity_source_unavailable",
message: "The source values could not be inspected safely. No assessments were applied.",
});
}
if (error instanceof z.ZodError) {
return reply.code(400).send({
code: "sensitive_data_suggestion_request_invalid",
message: "Choose a database, one or more tables, or one or more columns to classify.",
message: "Choose a database, one or more tables, or one or more columns to assess.",
});
}
return reply.code(500).send({
code: "sensitive_data_suggestion_failed",
message: "Sensitive-field suggestions failed before review. No changes were applied.",
message: "Local sensitivity analysis failed before review. No changes were applied.",
});
}
function untilAborted<T>(operation: Promise<T>, signal: AbortSignal): Promise<T> {
if (signal.aborted) {
void operation.catch(() => undefined);
return Promise.reject(new SensitivityAnalysisInterruptedError());
}
return new Promise<T>((resolve, reject) => {
const abort = () => reject(new SensitivityAnalysisInterruptedError());
signal.addEventListener("abort", abort, { once: true });
if (signal.aborted) {
void operation.catch(() => undefined);
abort();
return;
}
operation.then(
(value) => {
signal.removeEventListener("abort", abort);
resolve(value);
},
(error: unknown) => {
signal.removeEventListener("abort", abort);
reject(error);
},
);
});
}
@@ -300,18 +314,18 @@ function safeSuggestionHistoryError(reply: FastifyReply, error: unknown) {
if (error instanceof CatalogUnavailableError) {
return reply.code(503).send({
code: "catalog_unavailable",
message: "Sensitive Data Suggestion history is unavailable because the database catalog is unavailable.",
message: "Sensitivity Analysis history is unavailable because the database catalog is unavailable.",
});
}
if (error instanceof z.ZodError) {
return reply.code(400).send({
code: "sensitive_data_suggestion_history_request_invalid",
message: "Sensitive Data Suggestion history parameters are invalid.",
message: "Sensitivity Analysis history parameters are invalid.",
});
}
return reply.code(500).send({
code: "sensitive_data_suggestion_history_failed",
message: "Sensitive Data Suggestion history could not be loaded.",
message: "Sensitivity Analysis history could not be loaded.",
});
}
@@ -320,7 +334,7 @@ export function catalogDescriptionGenerationRoutes(
deps: {
repository: CatalogRepository;
worker: DescriptionGenerationWorker;
sensitiveDataSuggestionRunner: SensitiveDataSuggestionRunner;
sensitivityAnalysisRunner: SensitivityAnalysisRunner;
},
): void {
app.post("/catalog/databases/:databaseId/sensitive-data-suggestions", async (request, reply) => {
@@ -328,16 +342,16 @@ export function catalogDescriptionGenerationRoutes(
try {
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
const input = suggestionSchema.parse(request.body);
const result = await deps.sensitiveDataSuggestionRunner.run(
const signal = AbortSignal.timeout(60_000);
const result = await untilAborted(deps.sensitivityAnalysisRunner.run(
databaseId,
input.modelId,
input.scope,
"targetIds" in input ? input.targetIds : [],
new AbortController().signal,
);
signal,
), signal);
return {
suggestions: result.suggestions,
run: publicSensitiveDataSuggestionRun(result.run),
run: publicSensitivityAnalysisRun(result.run),
};
} catch (error) {
return safeSuggestionError(reply, error);
@@ -348,8 +362,8 @@ export function catalogDescriptionGenerationRoutes(
if (!manage(request, reply)) return reply;
try {
const { limit } = historyQuerySchema.parse(request.query);
return (await deps.repository.listSensitiveDataSuggestionRuns(limit))
.map(publicSensitiveDataSuggestionRun);
return (await deps.repository.listSensitivityAnalysisRuns(limit))
.map(publicSensitivityAnalysisRun);
} catch (error) {
return safeSuggestionHistoryError(reply, error);
}
@@ -359,12 +373,12 @@ export function catalogDescriptionGenerationRoutes(
if (!manage(request, reply)) return reply;
try {
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
const run = await deps.repository.getSensitiveDataSuggestionRun(runId);
const run = await deps.repository.getSensitivityAnalysisRun(runId);
if (!run) return reply.code(404).send({
code: "sensitive_data_suggestion_run_not_found",
message: "Sensitive Data Suggestion Run was not found.",
message: "Sensitivity Analysis Run was not found.",
});
return publicSensitiveDataSuggestionRun(run);
return publicSensitivityAnalysisRun(run);
} catch (error) {
return safeSuggestionHistoryError(reply, error);
}
@@ -375,14 +389,14 @@ export function catalogDescriptionGenerationRoutes(
try {
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
const { after } = eventQuerySchema.parse(request.query);
if (!(await deps.repository.getSensitiveDataSuggestionRun(runId))) {
if (!(await deps.repository.getSensitivityAnalysisRun(runId))) {
return reply.code(404).send({
code: "sensitive_data_suggestion_run_not_found",
message: "Sensitive Data Suggestion Run was not found.",
message: "Sensitivity Analysis Run was not found.",
});
}
return (await deps.repository.listSensitiveDataSuggestionEvents(runId, after))
.map(publicSensitiveDataSuggestionEvent);
return (await deps.repository.listSensitivityAnalysisEvents(runId, after))
.map(publicSensitivityAnalysisEvent);
} catch (error) {
return safeSuggestionHistoryError(reply, error);
}