feat: classify sensitive columns locally
This commit is contained in:
@@ -16,6 +16,7 @@ import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_s
|
||||
import { up as upLogicalRelationships } from "../src/catalog/migrations/008_catalog_logical_relationships.js";
|
||||
import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js";
|
||||
import { up as upCanonicalModelIds } from "../src/catalog/migrations/010_canonical_model_ids.js";
|
||||
import { up as upLocalSensitivityAnalysis } from "../src/catalog/migrations/011_local_sensitivity_analysis.js";
|
||||
|
||||
const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
|
||||
|
||||
@@ -47,12 +48,23 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
|
||||
modelId: "openai-mini", language: "en", status: "completed", total: 1,
|
||||
processed: 1, generated: 1,
|
||||
}).execute();
|
||||
const historicalSuggestionRunId = randomUUID();
|
||||
await db.insertInto("sensitiveDataSuggestionRuns").values({
|
||||
id: randomUUID(), databaseId: historicalDatabaseId, scope: "all",
|
||||
id: historicalSuggestionRunId, databaseId: historicalDatabaseId, scope: "all",
|
||||
modelId: "openai-mini", status: "completed", total: 1,
|
||||
suggestedSensitive: 1,
|
||||
}).execute();
|
||||
await upCanonicalModelIds(db);
|
||||
await upLocalSensitivityAnalysis(db);
|
||||
await expect(db.selectFrom("sensitiveDataSuggestionRuns")
|
||||
.select(["engine", "modelId", "policyVersion", "unknown"])
|
||||
.where("id", "=", historicalSuggestionRunId)
|
||||
.executeTakeFirstOrThrow()).resolves.toMatchObject({
|
||||
engine: "llm",
|
||||
modelId: "openai-mini",
|
||||
policyVersion: null,
|
||||
unknown: 0,
|
||||
});
|
||||
await expect(db.insertInto("descriptionGenerationRuns").values({
|
||||
id: randomUUID(), databaseId: historicalDatabaseId, scope: "all",
|
||||
modelId: "openai/gpt-5-mini", language: "en", status: "completed", total: 1,
|
||||
@@ -418,6 +430,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
|
||||
await upSensitiveSuggestionRuns(db);
|
||||
await upAiTokenUsage(db);
|
||||
await upCanonicalModelIds(db);
|
||||
await upLocalSensitivityAnalysis(db);
|
||||
const repository = new KyselyCatalogRepository(db);
|
||||
const firstDatabase = await repository.create({
|
||||
workspaceId: "generation-one",
|
||||
@@ -586,10 +599,10 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
|
||||
]);
|
||||
expect(await repository.getActiveDescriptionGenerationRun()).toBeUndefined();
|
||||
|
||||
const suggestionRun = await repository.createSensitiveDataSuggestionRun(
|
||||
const suggestionRun = await repository.createSensitivityAnalysisRun(
|
||||
firstDatabase.id,
|
||||
"selected_columns",
|
||||
"openai/gpt-4.1-mini",
|
||||
{ engine: "local", policyVersion: "sensitivity-v1" },
|
||||
);
|
||||
expect(suggestionRun).toMatchObject({
|
||||
databaseId: firstDatabase.id,
|
||||
@@ -597,43 +610,49 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
|
||||
total: 0,
|
||||
suggestedSensitive: 0,
|
||||
suggestedNonSensitive: 0,
|
||||
unknown: 0,
|
||||
engine: "local",
|
||||
modelId: null,
|
||||
policyVersion: "sensitivity-v1",
|
||||
startedAt: expect.any(String),
|
||||
});
|
||||
await repository.appendSensitiveDataSuggestionEvent(
|
||||
await repository.appendSensitivityAnalysisEvent(
|
||||
suggestionRun.id,
|
||||
"info",
|
||||
"Sensitive-field suggestion generation started.",
|
||||
);
|
||||
await repository.appendSensitiveDataSuggestionEvent(
|
||||
await repository.appendSensitivityAnalysisEvent(
|
||||
suggestionRun.id,
|
||||
"info",
|
||||
"Sensitive-field suggestion generation completed for 2 columns.",
|
||||
);
|
||||
expect(await repository.updateSensitiveDataSuggestionRun(suggestionRun.id, {
|
||||
expect(await repository.updateSensitivityAnalysisRun(suggestionRun.id, {
|
||||
status: "completed",
|
||||
total: 2,
|
||||
suggestedSensitive: 1,
|
||||
suggestedNonSensitive: 1,
|
||||
suggestedNonSensitive: 0,
|
||||
unknown: 1,
|
||||
finishedAt: new Date().toISOString(),
|
||||
})).toMatchObject({
|
||||
status: "completed",
|
||||
total: 2,
|
||||
suggestedSensitive: 1,
|
||||
suggestedNonSensitive: 1,
|
||||
suggestedNonSensitive: 0,
|
||||
unknown: 1,
|
||||
});
|
||||
expect(await repository.listSensitiveDataSuggestionEvents(suggestionRun.id, 1)).toEqual([
|
||||
expect(await repository.listSensitivityAnalysisEvents(suggestionRun.id, 1)).toEqual([
|
||||
expect.objectContaining({ sequence: 2, level: "info" }),
|
||||
]);
|
||||
expect((await repository.listSensitiveDataSuggestionRuns(1))[0]).toMatchObject({
|
||||
expect((await repository.listSensitivityAnalysisRuns(1))[0]).toMatchObject({
|
||||
id: suggestionRun.id,
|
||||
});
|
||||
|
||||
const interruptedSuggestionRun = await repository.createSensitiveDataSuggestionRun(
|
||||
const interruptedSuggestionRun = await repository.createSensitivityAnalysisRun(
|
||||
secondDatabase.id,
|
||||
"all",
|
||||
"openai/gpt-4.1-mini",
|
||||
{ engine: "local", policyVersion: "sensitivity-v1" },
|
||||
);
|
||||
expect(await repository.interruptActiveSensitiveDataSuggestionRuns(
|
||||
expect(await repository.interruptActiveSensitivityAnalysisRuns(
|
||||
"Sensitive-field suggestion generation was interrupted by backend restart.",
|
||||
)).toEqual([
|
||||
expect.objectContaining({
|
||||
|
||||
Reference in New Issue
Block a user