feat: sample sensitive columns progressively

backend/src/catalog/sensitivity-analysis-service.ts

@@ -12,7 +12,7 @@ import type {
 } from "./types.js";
 
 export type { SensitivityAnalysisScope } from "./types.js";
-export const SENSITIVITY_POLICY_VERSION = "sensitivity-v1";
+export const SENSITIVITY_POLICY_VERSION = "sensitivity-v2";
 
 interface SelectedColumn {
   table: CatalogTable;
@@ -30,6 +30,7 @@ export interface SensitivityReviewItem {
   assessment: SensitivityColumnAssessment["assessment"];
   evidence: readonly SensitivityEvidence[];
   observedValues: number;
+  coverage: SensitivityColumnAssessment["coverage"];
 }
 
 export class SensitivityAnalysisTargetNotFoundError extends Error {
@@ -48,7 +49,7 @@ export class SensitivityAnalysisDuplicateTargetIdsError extends Error {
 
 export class SensitivityAnalysisInterruptedError extends Error {
   constructor() {
-    super("sensitivity analysis deadline exceeded");
+    super("sensitivity analysis interrupted");
     this.name = "SensitivityAnalysisInterruptedError";
   }
 }
@@ -69,7 +70,7 @@ export class SensitivityAnalysisService {
   constructor(
     private readonly repository: CatalogRepository,
     private readonly classifier: SensitivityClassifier,
-    private readonly options: { runBudgetMs?: number; nerBudgetMs?: number; now?: () => number } = {},
+    private readonly options: { nerBudgetMs?: number } = {},
   ) {}
 
   private async selectColumns(
@@ -116,8 +117,6 @@ export class SensitivityAnalysisService {
     onPrepared?: (total: number) => void | Promise<void>,
     onProgress?: (processed: number, suggestions: readonly SensitivityReviewItem[]) => void | Promise<void>,
   ): Promise<readonly SensitivityReviewItem[]> {
-    const now = this.options.now ?? Date.now;
-    const deadline = now() + (this.options.runBudgetMs ?? 60_000);
     const configuredNerBudget = this.options.nerBudgetMs ?? 10_000;
     const nerBudget: SensitivityNerBudget = {
       remainingMs: Number.isFinite(configuredNerBudget) && configuredNerBudget >= 0
@@ -137,20 +136,23 @@ export class SensitivityAnalysisService {
       items.push(item);
       byTable.set(item.table.id, items);
     }
-    const suggestions: SensitivityReviewItem[] = [];
-    for (const items of byTable.values()) {
-      ensureActive(signal);
+    const tableTargets = [...byTable.values()].map((items) => {
       const first = items[0]!;
-      const assessments = await this.classifier.assessTable({
+      return {
         database,
         table: first.table,
         columns: items.map(({ column }) => column),
-      }, signal, deadline, nerBudget);
+      };
+    });
+    const assessments = await this.classifier.assess(tableTargets, signal, nerBudget);
+    ensureActive(signal);
+    const assessmentById = new Map(assessments.map((assessment) => [
+      assessment.columnId,
+      assessment,
+    ]));
+    const suggestions: SensitivityReviewItem[] = [];
+    for (const items of byTable.values()) {
       ensureActive(signal);
-      const assessmentById = new Map(assessments.map((assessment) => [
-        assessment.columnId,
-        assessment,
-      ]));
       const batch = items.map(({ table, column }) => {
         const assessment = assessmentById.get(column.id)!;
         return {
@@ -164,6 +166,7 @@ export class SensitivityAnalysisService {
           assessment: assessment.assessment,
           evidence: assessment.evidence,
           observedValues: assessment.observedValues,
+          coverage: assessment.coverage,
         };
       });
       suggestions.push(...batch);