feat: classify sensitive columns locally

This commit is contained in:
Codex
2026-09-03 02:11:13 +02:00
parent 7b87e95427
commit f114d0065a
57 changed files with 4038 additions and 1149 deletions
@@ -16,6 +16,7 @@ import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_s
import { up as upLogicalRelationships } from "../src/catalog/migrations/008_catalog_logical_relationships.js";
import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js";
import { up as upCanonicalModelIds } from "../src/catalog/migrations/010_canonical_model_ids.js";
import { up as upLocalSensitivityAnalysis } from "../src/catalog/migrations/011_local_sensitivity_analysis.js";
const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
@@ -47,12 +48,23 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
modelId: "openai-mini", language: "en", status: "completed", total: 1,
processed: 1, generated: 1,
}).execute();
const historicalSuggestionRunId = randomUUID();
await db.insertInto("sensitiveDataSuggestionRuns").values({
id: randomUUID(), databaseId: historicalDatabaseId, scope: "all",
id: historicalSuggestionRunId, databaseId: historicalDatabaseId, scope: "all",
modelId: "openai-mini", status: "completed", total: 1,
suggestedSensitive: 1,
}).execute();
await upCanonicalModelIds(db);
await upLocalSensitivityAnalysis(db);
await expect(db.selectFrom("sensitiveDataSuggestionRuns")
.select(["engine", "modelId", "policyVersion", "unknown"])
.where("id", "=", historicalSuggestionRunId)
.executeTakeFirstOrThrow()).resolves.toMatchObject({
engine: "llm",
modelId: "openai-mini",
policyVersion: null,
unknown: 0,
});
await expect(db.insertInto("descriptionGenerationRuns").values({
id: randomUUID(), databaseId: historicalDatabaseId, scope: "all",
modelId: "openai/gpt-5-mini", language: "en", status: "completed", total: 1,
@@ -418,6 +430,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
await upSensitiveSuggestionRuns(db);
await upAiTokenUsage(db);
await upCanonicalModelIds(db);
await upLocalSensitivityAnalysis(db);
const repository = new KyselyCatalogRepository(db);
const firstDatabase = await repository.create({
workspaceId: "generation-one",
@@ -586,10 +599,10 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
]);
expect(await repository.getActiveDescriptionGenerationRun()).toBeUndefined();
const suggestionRun = await repository.createSensitiveDataSuggestionRun(
const suggestionRun = await repository.createSensitivityAnalysisRun(
firstDatabase.id,
"selected_columns",
"openai/gpt-4.1-mini",
{ engine: "local", policyVersion: "sensitivity-v1" },
);
expect(suggestionRun).toMatchObject({
databaseId: firstDatabase.id,
@@ -597,43 +610,49 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
total: 0,
suggestedSensitive: 0,
suggestedNonSensitive: 0,
unknown: 0,
engine: "local",
modelId: null,
policyVersion: "sensitivity-v1",
startedAt: expect.any(String),
});
await repository.appendSensitiveDataSuggestionEvent(
await repository.appendSensitivityAnalysisEvent(
suggestionRun.id,
"info",
"Sensitive-field suggestion generation started.",
);
await repository.appendSensitiveDataSuggestionEvent(
await repository.appendSensitivityAnalysisEvent(
suggestionRun.id,
"info",
"Sensitive-field suggestion generation completed for 2 columns.",
);
expect(await repository.updateSensitiveDataSuggestionRun(suggestionRun.id, {
expect(await repository.updateSensitivityAnalysisRun(suggestionRun.id, {
status: "completed",
total: 2,
suggestedSensitive: 1,
suggestedNonSensitive: 1,
suggestedNonSensitive: 0,
unknown: 1,
finishedAt: new Date().toISOString(),
})).toMatchObject({
status: "completed",
total: 2,
suggestedSensitive: 1,
suggestedNonSensitive: 1,
suggestedNonSensitive: 0,
unknown: 1,
});
expect(await repository.listSensitiveDataSuggestionEvents(suggestionRun.id, 1)).toEqual([
expect(await repository.listSensitivityAnalysisEvents(suggestionRun.id, 1)).toEqual([
expect.objectContaining({ sequence: 2, level: "info" }),
]);
expect((await repository.listSensitiveDataSuggestionRuns(1))[0]).toMatchObject({
expect((await repository.listSensitivityAnalysisRuns(1))[0]).toMatchObject({
id: suggestionRun.id,
});
const interruptedSuggestionRun = await repository.createSensitiveDataSuggestionRun(
const interruptedSuggestionRun = await repository.createSensitivityAnalysisRun(
secondDatabase.id,
"all",
"openai/gpt-4.1-mini",
{ engine: "local", policyVersion: "sensitivity-v1" },
);
expect(await repository.interruptActiveSensitiveDataSuggestionRuns(
expect(await repository.interruptActiveSensitivityAnalysisRuns(
"Sensitive-field suggestion generation was interrupted by backend restart.",
)).toEqual([
expect.objectContaining({