mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: sample sensitive columns progressively

Browse Source
This commit is contained in:
Codex
2026-09-03 10:25:05 +02:00
parent f114d0065a
commit 8e778b9edb
24 changed files with 1001 additions and 581 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/catalog/sensitivity-analysis-runner.ts
+3 -6
View File
@@ -14,7 +14,7 @@ import type {
} from "./types.js";
const interruptedMessage = "Local sensitivity analysis was interrupted by backend restart.";
const deadlineMessage = "Local sensitivity analysis reached its time limit.";
const interruptedDuringRunMessage = "Local sensitivity analysis was interrupted before completion.";
const failedMessage = "Local sensitivity analysis failed.";
function ensureActive(signal: AbortSignal): void {
@@ -98,14 +98,12 @@ export class SensitivityAnalysisRunner {
const suggestedNonSensitive = batch.filter(
(suggestion) => suggestion.assessment === "non_sensitive",
).length;
const unknown = batch.filter((suggestion) => suggestion.assessment === "unknown").length;
const current = await this.repository.getSensitivityAnalysisRun(started.id);
ensureActive(signal);
if (!current) throw new Error("Sensitivity Analysis Run disappeared");
const progress = await this.repository.updateSensitivityAnalysisRun(started.id, {
suggestedSensitive: current.suggestedSensitive + suggestedSensitive,
suggestedNonSensitive: current.suggestedNonSensitive + suggestedNonSensitive,
unknown: current.unknown + unknown,
});
if (!progress) throw new Error("Sensitivity Analysis Run disappeared");
processedSensitive += suggestedSensitive;
@@ -126,7 +124,6 @@ export class SensitivityAnalysisRunner {
const suggestedNonSensitive = suggestions.filter(
(suggestion) => suggestion.assessment === "non_sensitive",
).length;
const unknown = suggestions.filter((suggestion) => suggestion.assessment === "unknown").length;
await this.repository.appendSensitivityAnalysisEvent(
started.id,
"info",
@@ -140,7 +137,7 @@ export class SensitivityAnalysisRunner {
total: suggestions.length,
suggestedSensitive,
suggestedNonSensitive,
unknown,
unknown: 0,
finishedAt: new Date().toISOString(),
errorSummary: null,
});
@@ -149,7 +146,7 @@ export class SensitivityAnalysisRunner {
return { suggestions, run: completed };
} catch (error) {
const interrupted = signal.aborted || error instanceof SensitivityAnalysisInterruptedError;
const message = interrupted ? deadlineMessage : failedMessage;
const message = interrupted ? interruptedDuringRunMessage : failedMessage;
await this.repository.updateSensitivityAnalysisRun(started.id, {
status: interrupted ? "interrupted" : "failed",
...(interrupted ? {
backend/src/catalog/sensitivity-analysis-service.ts
+17 -14
View File
@@ -12,7 +12,7 @@ import type {
} from "./types.js";
export type { SensitivityAnalysisScope } from "./types.js";
export const SENSITIVITY_POLICY_VERSION = "sensitivity-v1";
export const SENSITIVITY_POLICY_VERSION = "sensitivity-v2";
interface SelectedColumn {
table: CatalogTable;
@@ -30,6 +30,7 @@ export interface SensitivityReviewItem {
assessment: SensitivityColumnAssessment["assessment"];
evidence: readonly SensitivityEvidence[];
observedValues: number;
coverage: SensitivityColumnAssessment["coverage"];
}
export class SensitivityAnalysisTargetNotFoundError extends Error {
@@ -48,7 +49,7 @@ export class SensitivityAnalysisDuplicateTargetIdsError extends Error {
export class SensitivityAnalysisInterruptedError extends Error {
constructor() {
super("sensitivity analysis deadline exceeded");
super("sensitivity analysis interrupted");
this.name = "SensitivityAnalysisInterruptedError";
}
}
@@ -69,7 +70,7 @@ export class SensitivityAnalysisService {
constructor(
private readonly repository: CatalogRepository,
private readonly classifier: SensitivityClassifier,
private readonly options: { runBudgetMs?: number; nerBudgetMs?: number; now?: () => number } = {},
private readonly options: { nerBudgetMs?: number } = {},
) {}
private async selectColumns(
@@ -116,8 +117,6 @@ export class SensitivityAnalysisService {
onPrepared?: (total: number) => void | Promise<void>,
onProgress?: (processed: number, suggestions: readonly SensitivityReviewItem[]) => void | Promise<void>,
): Promise<readonly SensitivityReviewItem[]> {
const now = this.options.now ?? Date.now;
const deadline = now() + (this.options.runBudgetMs ?? 60_000);
const configuredNerBudget = this.options.nerBudgetMs ?? 10_000;
const nerBudget: SensitivityNerBudget = {
remainingMs: Number.isFinite(configuredNerBudget) && configuredNerBudget >= 0
@@ -137,20 +136,23 @@ export class SensitivityAnalysisService {
items.push(item);
byTable.set(item.table.id, items);
}
const suggestions: SensitivityReviewItem[] = [];
for (const items of byTable.values()) {
ensureActive(signal);
const tableTargets = [...byTable.values()].map((items) => {
const first = items[0]!;
const assessments = await this.classifier.assessTable({
return {
database,
table: first.table,
columns: items.map(({ column }) => column),
}, signal, deadline, nerBudget);
};
});
const assessments = await this.classifier.assess(tableTargets, signal, nerBudget);
ensureActive(signal);
const assessmentById = new Map(assessments.map((assessment) => [
assessment.columnId,
assessment,
]));
const suggestions: SensitivityReviewItem[] = [];
for (const items of byTable.values()) {
ensureActive(signal);
const assessmentById = new Map(assessments.map((assessment) => [
assessment.columnId,
assessment,
]));
const batch = items.map(({ table, column }) => {
const assessment = assessmentById.get(column.id)!;
return {
@@ -164,6 +166,7 @@ export class SensitivityAnalysisService {
assessment: assessment.assessment,
evidence: assessment.evidence,
observedValues: assessment.observedValues,
coverage: assessment.coverage,
};
});
suggestions.push(...batch);
backend/src/catalog/sensitivity-classifier.ts
+165 -108
View File
@@ -1,11 +1,11 @@
import { CatalogConnectorError, type CatalogColumn, type CatalogTable, type WorkspaceDatabase } from "./types.js";
import type { CatalogColumn, CatalogTable, WorkspaceDatabase } from "./types.js";
import { findPhoneNumbersInText } from "libphonenumber-js/max";
import validator from "validator";
export type SensitivityAssessment = "sensitive" | "non_sensitive" | "unknown";
export type SensitivityAssessment = "sensitive" | "non_sensitive";
export interface SensitivityEvidence {
kind: "metadata" | "content" | "length" | "ner" | "coverage";
kind: "metadata" | "content" | "length" | "ner" | "coverage" | "type";
ruleId: string;
label?: string;
confidence?: number;
@@ -18,8 +18,8 @@ export interface SensitivityValueObservation {
}
export interface SensitivityScanCoverage {
kind: "complete" | "sampled" | "unavailable";
observedRows: number;
kind: "complete" | "sampled";
observedValues: number;
}
export interface SensitivityTableScan {
@@ -31,8 +31,11 @@ export interface SensitivityScanRequest {
database: WorkspaceDatabase;
table: CatalogTable;
columns: readonly CatalogColumn[];
fullScanBudgetMs: number;
deadline: number;
valuesPerColumn: number;
sampleOffset: number;
sampleSeed: number;
queryTimeoutMs: number;
fullScanThreshold?: number;
}
export interface SensitivityValueSource {
@@ -76,6 +79,7 @@ export interface SensitivityColumnAssessment {
proposedSensitive: boolean;
evidence: readonly SensitivityEvidence[];
observedValues: number;
coverage: "metadata" | "complete" | "sampled" | "no_values";
}
export interface SensitivityTableTarget {
@@ -94,7 +98,15 @@ const CREDENTIAL_NAME = /(?:^|_)(?:api_key|credential|password|passwd|private_ke
const HEALTH_NAME = /(?:^|_)(?:anamnesi|clinical|diagnos(?:i|is)|health|medical|patient|patologia|therapy|terapia)(?:_|$)/u;
const CLINICAL_TERM = /(?:^|[^\p{L}])(?:allergi[ae]|anamnesi|carcinoma|chemioterapia|diabete|diagnos[ei]|epatite|farmac[io]|gravidanza|hiv|metastasi|neoplasia|patologia|radioterapia|referto|terapia|tumore)(?:$|[^\p{L}])/iu;
const UNSUPPORTED_BINARY_TYPE = /(?:^|\s)(?:binary|blob|bytea|image|varbinary)(?:\s|$|\()/iu;
const DEEP_TEXT_TYPE = /(?:^|\s)(?:char|character|citext|clob|json|jsonb|nchar|nvarchar|string|text|varchar|xml)(?:\s|$|\()/iu;
const MAX_NER_CANDIDATES_PER_REQUEST = 128;
const MAX_CONCURRENT_TABLE_SCANS = 2;
export const SENSITIVITY_SAMPLE_PHASES = [
{ targetValuesPerColumn: 300, additionalValuesPerColumn: 300, sampleSeed: 37, deepTextOnly: false },
{ targetValuesPerColumn: 1_000, additionalValuesPerColumn: 700, sampleSeed: 73, deepTextOnly: false },
{ targetValuesPerColumn: 3_000, additionalValuesPerColumn: 2_000, sampleSeed: 109, deepTextOnly: true },
] as const;
function normalizedName(value: string): string {
return value.normalize("NFKD")
@@ -284,14 +296,22 @@ function contentEvidence(value: string): SensitivityEvidence | undefined {
return undefined;
}
interface ColumnState {
column: CatalogColumn;
evidence: SensitivityEvidence[];
observedValues: number;
nerCandidates: string[];
coverage: "metadata" | "complete" | "sampled" | "no_values";
sampledTarget: number;
}
/** Sole decision module for local column-level sensitivity assessments. */
export class SensitivityClassifier {
constructor(
private readonly values: SensitivityValueSource,
private readonly detector?: LocalNerDetector,
private readonly options: {
fullScanBudgetMs?: number;
runBudgetMs?: number;
queryTimeoutMs?: number;
nerConfidenceThreshold?: number;
maxNerValuesPerColumn?: number;
maxNerCandidatesPerTable?: number;
@@ -299,95 +319,131 @@ export class SensitivityClassifier {
} = {},
) {}
async assessTable(
target: SensitivityTableTarget,
async assess(
targets: readonly SensitivityTableTarget[],
signal: AbortSignal,
runDeadline?: number,
nerBudget?: SensitivityNerBudget,
sharedNerBudget?: SensitivityNerBudget,
): Promise<readonly SensitivityColumnAssessment[]> {
const now = this.options.now ?? Date.now;
const deadline = runDeadline ?? now() + (this.options.runBudgetMs ?? 60_000);
const evidence = new Map(target.columns.map((column) => {
const match = metadataEvidence(column);
return [column.id, match ? [match] : [] as SensitivityEvidence[]];
}));
const observed = new Map(target.columns.map((column) => [column.id, 0]));
const nerCandidates = new Map(target.columns.map((column) => [column.id, [] as string[]]));
const maxNerValuesPerColumn = boundedCount(this.options.maxNerValuesPerColumn, 8, 8);
const unsupported = new Set(target.columns
.filter((column) => UNSUPPORTED_BINARY_TYPE.test(column.dataType))
.map((column) => column.id));
const scannableColumns = target.columns.filter((column) => (
!unsupported.has(column.id) && evidence.get(column.id)!.length === 0
));
let coverage: SensitivityScanCoverage = { kind: "unavailable", observedRows: 0 };
if (scannableColumns.length > 0 && now() < deadline) {
try {
coverage = await this.values.scanTable({
...target,
columns: scannableColumns,
fullScanBudgetMs: this.options.fullScanBudgetMs ?? 5_000,
deadline,
}, (batch) => {
for (const item of batch) {
if (!evidence.has(item.columnId) || item.value === null) continue;
observed.set(item.columnId, (observed.get(item.columnId) ?? 0) + 1);
const matches = evidence.get(item.columnId)!;
if (matches.length === 0 && (item.characterLength ?? item.value.length) > 500) {
matches.push({ kind: "length", ruleId: "text.over_500_characters" });
} else if (matches.length === 0) {
const match = contentEvidence(item.value);
if (match) matches.push(match);
else {
const candidates = nerCandidates.get(item.columnId)!;
if (candidates.length < maxNerValuesPerColumn && !candidates.includes(item.value)) {
candidates.push(item.value);
}
}
}
}
}, signal);
} catch (error) {
if (!(error instanceof CatalogConnectorError)) throw error;
const states = new Map<string, ColumnState>();
for (const target of targets) {
for (const column of target.columns) {
const metadataMatch = metadataEvidence(column);
const binary = UNSUPPORTED_BINARY_TYPE.test(column.dataType);
states.set(column.id, {
column,
evidence: metadataMatch
? [metadataMatch]
: binary
? [{ kind: "type", ruleId: "type.binary_uninspectable" }]
: [],
observedValues: 0,
nerCandidates: [],
coverage: metadataMatch || binary ? "metadata" : "no_values",
sampledTarget: 0,
});
}
}
if (this.detector && (this.detector.isReady?.() ?? true) && !signal.aborted
&& now() < deadline && (nerBudget?.remainingMs ?? 1) > 0) {
const candidates: LocalNerCandidate[] = [];
const maxCandidates = boundedCount(this.options.maxNerCandidatesPerTable, 2, 1_024);
candidateSelection: for (let valueIndex = 0; valueIndex < maxNerValuesPerColumn; valueIndex += 1) {
for (const column of target.columns) {
if (evidence.get(column.id)!.length > 0) continue;
const text = nerCandidates.get(column.id)![valueIndex];
if (text === undefined) continue;
candidates.push({ columnId: column.id, text });
if (candidates.length >= maxCandidates) break candidateSelection;
const completeTables = new Set<string>();
for (const [phaseIndex, phase] of SENSITIVITY_SAMPLE_PHASES.entries()) {
for (let offset = 0; offset < targets.length; offset += MAX_CONCURRENT_TABLE_SCANS) {
signal.throwIfAborted();
const peerController = new AbortController();
const scanSignal = AbortSignal.any([signal, peerController.signal]);
try {
await Promise.all(targets.slice(offset, offset + MAX_CONCURRENT_TABLE_SCANS).map(async (target) => {
if (completeTables.has(target.table.id)) return;
const columns = target.columns.filter((column) => {
const state = states.get(column.id)!;
return state.evidence.length === 0
&& (!phase.deepTextOnly || DEEP_TEXT_TYPE.test(column.dataType));
});
if (columns.length === 0) return;
const coverage = await this.values.scanTable({
...target,
columns,
valuesPerColumn: phase.additionalValuesPerColumn,
sampleOffset: phase.targetValuesPerColumn - phase.additionalValuesPerColumn,
sampleSeed: phase.sampleSeed,
queryTimeoutMs: this.options.queryTimeoutMs ?? 5_000,
...(phaseIndex === 0 ? { fullScanThreshold: 1_000 } : {}),
}, (batch) => {
for (const item of batch) {
if (item.value === null) continue;
const state = states.get(item.columnId);
if (!state || state.evidence.length > 0) continue;
state.observedValues += 1;
if ((item.characterLength ?? item.value.length) > 500) {
state.evidence.push({ kind: "length", ruleId: "text.over_500_characters" });
continue;
}
const match = contentEvidence(item.value);
if (match) {
state.evidence.push(match);
continue;
}
if (state.nerCandidates.length < maxNerValuesPerColumn
&& !state.nerCandidates.includes(item.value)) {
state.nerCandidates.push(item.value);
}
}
}, scanSignal);
for (const column of columns) {
const state = states.get(column.id)!;
state.sampledTarget = Math.max(state.sampledTarget, phase.targetValuesPerColumn);
state.coverage = coverage.kind === "complete"
? "complete"
: state.observedValues === 0 ? "no_values" : "sampled";
}
if (coverage.kind === "complete") completeTables.add(target.table.id);
}));
} catch (error) {
peerController.abort(error);
throw error;
}
}
if (candidates.length > 0) {
const threshold = this.options.nerConfidenceThreshold ?? 0.8;
const nerStartedAt = now();
const allowedNerMs = nerBudget
? Math.max(0, nerBudget.remainingMs)
: Math.max(0, deadline - nerStartedAt);
const nerDeadline = Math.min(deadline, nerStartedAt + allowedNerMs);
}
const nerBudget = sharedNerBudget ?? { remainingMs: 10_000 };
if (this.detector && (this.detector.isReady?.() ?? true) && !signal.aborted
&& nerBudget.remainingMs > 0) {
const maxCandidates = boundedCount(this.options.maxNerCandidatesPerTable, 2, 1_024);
const threshold = this.options.nerConfidenceThreshold ?? 0.8;
for (const target of targets) {
signal.throwIfAborted();
if (nerBudget.remainingMs <= 0) break;
const candidates: LocalNerCandidate[] = [];
candidateSelection: for (let valueIndex = 0; valueIndex < maxNerValuesPerColumn; valueIndex += 1) {
for (const column of target.columns) {
const state = states.get(column.id)!;
if (state.evidence.length > 0) continue;
const text = state.nerCandidates[valueIndex];
if (text === undefined) continue;
candidates.push({ columnId: column.id, text });
if (candidates.length >= maxCandidates) break candidateSelection;
}
}
if (candidates.length === 0) continue;
const startedAt = now();
const deadline = startedAt + nerBudget.remainingMs;
try {
for (let offset = 0; offset < candidates.length; offset += MAX_NER_CANDIDATES_PER_REQUEST) {
if (signal.aborted || now() >= nerDeadline) break;
if (signal.aborted || now() >= deadline) break;
try {
const detected = await this.detector.detect(
candidates.slice(offset, offset + MAX_NER_CANDIDATES_PER_REQUEST),
signal,
nerDeadline,
deadline,
);
for (const item of detected) {
const matches = evidence.get(item.columnId);
if (!matches || matches.length > 0 || !Number.isFinite(item.confidence)
const state = states.get(item.columnId);
if (!state || state.evidence.length > 0 || !Number.isFinite(item.confidence)
|| item.confidence < threshold || item.confidence > 1) continue;
const label = normalizedName(item.label).slice(0, 80);
if (!label) continue;
matches.push({
state.evidence.push({
kind: "ner",
ruleId: "ner.entity",
label,
@@ -400,42 +456,43 @@ export class SensitivityClassifier {
}
}
} finally {
if (nerBudget) {
const elapsedMs = Math.max(1, now() - nerStartedAt);
nerBudget.remainingMs = Math.max(0, nerBudget.remainingMs - elapsedMs);
}
nerBudget.remainingMs = Math.max(0, nerBudget.remainingMs - Math.max(1, now() - startedAt));
}
}
}
return target.columns.map((column) => {
const matches = evidence.get(column.id)!;
const count = observed.get(column.id) ?? 0;
const assessment: SensitivityAssessment = matches.length > 0
? "sensitive"
: unsupported.has(column.id) || count === 0 || coverage.kind !== "complete"
? "unknown"
: "non_sensitive";
return targets.flatMap((target) => target.columns.map((column) => {
const state = states.get(column.id)!;
const sensitive = state.evidence.length > 0;
const coverage = state.observedValues === 0 && !sensitive ? "no_values" : state.coverage;
const coverageEvidence: SensitivityEvidence[] = sensitive
? state.evidence
: [{
kind: "coverage",
ruleId: coverage === "complete"
? "coverage.complete"
: coverage === "no_values"
? "coverage.no_values"
: `coverage.sampled_${state.sampledTarget}`,
}];
return {
columnId: column.id,
assessment,
proposedSensitive: assessment === "unknown" ? column.sensitive : assessment === "sensitive",
evidence: matches.length > 0
? matches
: assessment === "unknown"
? [{
kind: "coverage",
ruleId: unsupported.has(column.id)
? "coverage.unsupported_type"
: coverage.kind === "unavailable"
? "coverage.unavailable"
: count === 0
? "coverage.no_values"
: "coverage.incomplete",
}]
: [],
observedValues: count,
assessment: sensitive ? "sensitive" : "non_sensitive",
proposedSensitive: sensitive,
evidence: coverageEvidence,
observedValues: state.observedValues,
coverage,
};
});
}));
}
/** Convenience for focused callers and rule-level tests. Production orchestration uses assess(). */
async assessTable(
target: SensitivityTableTarget,
signal: AbortSignal,
_retiredRunDeadline?: number,
nerBudget?: SensitivityNerBudget,
): Promise<readonly SensitivityColumnAssessment[]> {
return await this.assess([target], signal, nerBudget);
}
}
backend/src/catalog/sensitivity-shadow.ts
+12 -6
View File
@@ -5,7 +5,10 @@ import { WorkspaceSecretStore } from "../workspaces/secret-store.js";
import { PythonLocalNerDetector } from "./local-ner-detector.js";
import { ConcreteCatalogPostgresAccess } from "./postgres-access.js";
import { createCatalogRepository } from "./repository.js";
import { SensitivityAnalysisService } from "./sensitivity-analysis-service.js";
import {
SENSITIVITY_POLICY_VERSION,
SensitivityAnalysisService,
} from "./sensitivity-analysis-service.js";
import { SensitivityClassifier } from "./sensitivity-classifier.js";
import { ConcreteSensitivityValueSource } from "./sensitivity-value-source.js";
@@ -60,23 +63,26 @@ async function main(): Promise<void> {
const suggestions = await new SensitivityAnalysisService(
repository,
new SensitivityClassifier(source, detector),
).analyze(database.id, "all", [], AbortSignal.timeout(65_000));
const assessments = { sensitive: 0, nonSensitive: 0, unknown: 0 };
).analyze(database.id, "all", [], new AbortController().signal);
const assessments = { sensitive: 0, nonSensitive: 0 };
const coverage = { metadata: 0, complete: 0, sampled: 0, noValues: 0 };
const rules = new Map<string, number>();
for (const suggestion of suggestions) {
if (suggestion.assessment === "sensitive") assessments.sensitive += 1;
else if (suggestion.assessment === "non_sensitive") assessments.nonSensitive += 1;
else assessments.unknown += 1;
else assessments.nonSensitive += 1;
if (suggestion.coverage === "no_values") coverage.noValues += 1;
else coverage[suggestion.coverage] += 1;
for (const evidence of suggestion.evidence) {
rules.set(evidence.ruleId, (rules.get(evidence.ruleId) ?? 0) + 1);
}
}
process.stdout.write(`${JSON.stringify({
ok: true,
policyVersion: "sensitivity-v1",
policyVersion: SENSITIVITY_POLICY_VERSION,
nerEnabled: detector !== undefined,
total: suggestions.length,
assessments,
coverage,
rules: Object.fromEntries([...rules].sort(([left], [right]) => left.localeCompare(right))),
elapsedMs: Date.now() - startedAt,
})}\n`);
backend/src/catalog/sensitivity-value-source.ts
+175 -143
View File
@@ -8,82 +8,117 @@ import type {
SensitivityValueObservation,
SensitivityValueSource,
} from "./sensitivity-classifier.js";
import { CatalogConnectorError } from "./types.js";
import { CatalogConnectorError, type CatalogColumn } from "./types.js";
const MAX_VALUE_CHARACTERS = 501;
const DEFAULT_BATCH_ROWS = 200;
const DEFAULT_SAMPLE_ROWS = 200;
const MAX_COLUMNS_PER_QUERY = 25;
const SAMPLE_OVERSCAN_FACTOR = 10;
function quoteIdentifier(identifier: string): string {
return `"${identifier.replaceAll('"', '""')}"`;
}
function projections(request: SensitivityScanRequest): string {
return request.columns.flatMap((column, index) => {
function chunks<T>(items: readonly T[], size: number): T[][] {
const result: T[][] = [];
for (let offset = 0; offset < items.length; offset += size) {
result.push(items.slice(offset, offset + size));
}
return result;
}
function tableReference(request: SensitivityScanRequest): string {
return `${quoteIdentifier(request.database.schema)}.${quoteIdentifier(request.table.name)}`;
}
function samplePercentage(valuesPerColumn: number): number {
if (valuesPerColumn <= 300) return 30;
if (valuesPerColumn <= 700) return 70;
return 100;
}
function flatValueQuery(
request: SensitivityScanRequest,
columns: readonly CatalogColumn[],
options: { complete: boolean; randomized: boolean },
): string {
const projections = columns.map((column) => quoteIdentifier(column.name)).join(", ");
const perColumnLimit = options.complete
? request.fullScanThreshold ?? request.valuesPerColumn
: request.valuesPerColumn;
const rowLimit = Math.max(perColumnLimit, perColumnLimit * SAMPLE_OVERSCAN_FACTOR);
const sample = options.complete
? `SELECT ${projections} FROM ${tableReference(request)}`
: [
`SELECT ${projections} FROM ${tableReference(request)}`,
...(options.randomized
? [`TABLESAMPLE SYSTEM (${samplePercentage(request.valuesPerColumn)}) REPEATABLE (${request.sampleSeed})`]
: []),
`LIMIT ${rowLimit} OFFSET ${request.sampleOffset}`,
].join(" ");
const values = columns.map((column, index) => {
const identifier = quoteIdentifier(column.name);
return [
`LEFT((${identifier})::text, ${MAX_VALUE_CHARACTERS}) AS "__value_${index}"`,
`CASE WHEN ${identifier} IS NULL THEN NULL ELSE char_length((${identifier})::text) END AS "__length_${index}"`,
];
`(${index}, LEFT((sampled.${identifier})::text, ${MAX_VALUE_CHARACTERS}),`,
`CASE WHEN sampled.${identifier} IS NULL THEN NULL`,
`ELSE char_length((sampled.${identifier})::text) END)`,
].join(" ");
}).join(", ");
return [
`WITH sampled AS MATERIALIZED (${sample}),`,
"ranked AS (",
"SELECT value.__column_index, value.__value, value.__length,",
"row_number() OVER (PARTITION BY value.__column_index) AS __rank",
"FROM sampled",
`CROSS JOIN LATERAL (VALUES ${values}) AS value(__column_index, __value, __length)`,
"WHERE value.__value IS NOT NULL",
")",
"SELECT __column_index, __value, __length FROM ranked",
`WHERE __rank <= ${perColumnLimit}`,
].join(" ");
}
function observations(
request: SensitivityScanRequest,
columns: readonly CatalogColumn[],
rows: readonly Record<string, unknown>[],
): SensitivityValueObservation[] {
return rows.flatMap((row) => request.columns.map((column, index) => {
const sourceValue = row[`__value_${index}`];
const sourceLength = row[`__length_${index}`];
const value = sourceValue === null || sourceValue === undefined ? null : String(sourceValue);
const parsedLength = sourceLength === null || sourceLength === undefined
return rows.flatMap((row) => {
const index = Number(row.__column_index);
const column = Number.isSafeInteger(index) && index >= 0 ? columns[index] : undefined;
if (!column || row.__value === null || row.__value === undefined) return [];
const value = String(row.__value);
const parsedLength = row.__length === null || row.__length === undefined
? null
: Number(sourceLength);
return {
: Number(row.__length);
return [{
columnId: column.id,
value,
characterLength: parsedLength !== null && Number.isSafeInteger(parsedLength) && parsedLength >= 0
? parsedLength
: value?.length ?? null,
};
}));
: value.length,
}];
});
}
function cancelled(error: unknown): boolean {
return Boolean(error && typeof error === "object" && "code" in error && error.code === "57014");
}
interface SensitivityValueSourceOptions {
now?: () => number;
batchRows?: number;
sampleRows?: number;
}
/**
* PostgreSQL value adapter. It owns bounded read mechanics and emits normalized values, never a
* sensitivity decision.
* Database-specific sampling adapter. Policy stays in SensitivityClassifier; this module only
* produces bounded, normalized non-null observations without persisting or logging values.
*/
export class ConcreteSensitivityValueSource implements SensitivityValueSource {
private readonly now: () => number;
private readonly batchRows: number;
private readonly sampleRows: number;
constructor(
private readonly access: CatalogPostgresAccess,
private readonly secretStore?: Pick<WorkspaceSecretStore, "materialize">,
options: SensitivityValueSourceOptions = {},
) {
this.now = options.now ?? Date.now;
this.batchRows = options.batchRows ?? DEFAULT_BATCH_ROWS;
this.sampleRows = options.sampleRows ?? DEFAULT_SAMPLE_ROWS;
}
) {}
async scanTable(
request: SensitivityScanRequest,
consume: (batch: readonly SensitivityValueObservation[]) => void | Promise<void>,
signal: AbortSignal,
): Promise<SensitivityScanCoverage> {
if (request.columns.length === 0) return { kind: "unavailable", observedRows: 0 };
if (request.columns.length === 0) return { kind: "complete", observedValues: 0 };
if (request.database.binding.transport === "rest_api") {
return await this.scanRest(request, consume, signal);
}
@@ -97,69 +132,63 @@ export class ConcreteSensitivityValueSource implements SensitivityValueSource {
): Promise<SensitivityScanCoverage> {
const client = await this.access.connect(request.database, signal);
let transactionOpen = false;
const startedAt = this.now();
const fullDeadline = Math.min(request.deadline, startedAt + request.fullScanBudgetMs);
let observedRows = 0;
let cursorOpen = false;
let savepointSequence = 0;
let observedValues = 0;
try {
if (signal.aborted || this.now() >= request.deadline) {
return { kind: "sampled", observedRows: 0 };
}
signal.throwIfAborted();
await client.query("BEGIN TRANSACTION READ ONLY", []);
transactionOpen = true;
await client.query("SELECT set_config('statement_timeout', $1, true)", [
`${Math.max(1, Math.floor(fullDeadline - startedAt))}ms`,
`${Math.max(1, Math.floor(request.queryTimeoutMs))}ms`,
]);
await client.query("SAVEPOINT sensitivity_full_scan", []);
const cursor = [
"DECLARE sensitivity_full_scan_cursor NO SCROLL CURSOR FOR",
`SELECT ${projections(request)}`,
`FROM ${quoteIdentifier(request.database.schema)}.${quoteIdentifier(request.table.name)}`,
].join(" ");
await client.query(cursor, []);
cursorOpen = true;
while (!signal.aborted && this.now() < fullDeadline) {
let rows: Array<Record<string, unknown>>;
const boundedQuery = async (sql: string): Promise<Array<Record<string, unknown>> | undefined> => {
signal.throwIfAborted();
savepointSequence += 1;
const savepoint = `sensitivity_scan_${savepointSequence}`;
await client.query(`SAVEPOINT ${savepoint}`, []);
try {
await client.query("SELECT set_config('statement_timeout', $1, true)", [
`${Math.max(1, Math.floor(fullDeadline - this.now()))}ms`,
]);
rows = (await client.query(
`FETCH FORWARD ${this.batchRows} FROM sensitivity_full_scan_cursor`,
[],
)).rows;
return (await client.query(sql, [])).rows;
} catch (error) {
if (!cancelled(error)) throw error;
await client.query("ROLLBACK TO SAVEPOINT sensitivity_full_scan", []);
cursorOpen = false;
break;
}
if (rows.length > 0) {
observedRows += rows.length;
await consume(observations(request, rows));
}
if (rows.length < this.batchRows) {
return { kind: "complete", observedRows };
await client.query(`ROLLBACK TO SAVEPOINT ${savepoint}`, []);
return undefined;
} finally {
await client.query(`RELEASE SAVEPOINT ${savepoint}`, []).catch(() => undefined);
}
};
let complete = false;
if (request.fullScanThreshold !== undefined) {
const probe = await boundedQuery(
`SELECT 1 AS __present FROM ${tableReference(request)} LIMIT ${request.fullScanThreshold + 1}`,
);
complete = probe !== undefined && probe.length <= request.fullScanThreshold;
}
if (signal.aborted || this.now() >= request.deadline) {
return { kind: "sampled", observedRows };
for (const columnChunk of chunks(request.columns, MAX_COLUMNS_PER_QUERY)) {
signal.throwIfAborted();
let rows = await boundedQuery(flatValueQuery(request, columnChunk, {
complete,
randomized: !complete,
}));
if (rows === undefined && complete) {
complete = false;
rows = await boundedQuery(flatValueQuery(request, columnChunk, {
complete: false,
randomized: true,
}));
}
if (!complete && (rows === undefined || rows.length === 0)) {
rows = await boundedQuery(flatValueQuery(request, columnChunk, {
complete: false,
randomized: false,
}));
}
if (rows === undefined) throw new CatalogConnectorError("Sensitivity sample query timed out");
const batch = observations(columnChunk, rows);
observedValues += batch.length;
if (batch.length > 0) await consume(batch);
}
if (cursorOpen) await client.query("CLOSE sensitivity_full_scan_cursor", []);
await client.query("RELEASE SAVEPOINT sensitivity_full_scan", []);
await client.query("SELECT set_config('statement_timeout', $1, true)", [
`${Math.max(1, Math.floor(request.deadline - this.now()))}ms`,
]);
const sampleSql = [
`SELECT ${projections(request)}`,
`FROM ${quoteIdentifier(request.database.schema)}.${quoteIdentifier(request.table.name)}`,
"TABLESAMPLE SYSTEM (1) REPEATABLE (37)",
"LIMIT $1",
].join(" ");
const sampledRows = (await client.query(sampleSql, [this.sampleRows])).rows;
observedRows += sampledRows.length;
if (sampledRows.length > 0) await consume(observations(request, sampledRows));
return { kind: "sampled", observedRows };
return { kind: complete ? "complete" : "sampled", observedValues };
} catch (error) {
if (error instanceof CatalogConnectorError) throw error;
throw new CatalogConnectorError("Sensitivity source scan failed");
@@ -180,9 +209,7 @@ export class ConcreteSensitivityValueSource implements SensitivityValueSource {
request.database.workspaceId,
auth === "none" ? [] : [CATALOG_SECRET_IDS.apiKey],
);
const startedAt = this.now();
const fullDeadline = Math.min(request.deadline, startedAt + request.fullScanBudgetMs);
let observedRows = 0;
let observedValues = 0;
try {
const headers: Record<string, string> = { "content-type": "application/json" };
if (auth !== "none") {
@@ -194,61 +221,66 @@ export class ConcreteSensitivityValueSource implements SensitivityValueSource {
}
const baseUrl = request.database.binding.baseUrl?.replace(/\/+$/u, "");
if (!baseUrl) throw new CatalogConnectorError("Database binding is incomplete");
const runQuery = async (sql: string, deadline: number): Promise<Array<Record<string, unknown>>> => {
const response = await fetch(`${baseUrl}/rpc/run_query`, {
method: "POST",
headers,
body: JSON.stringify({ query_text: sql }),
signal: AbortSignal.any([
signal,
AbortSignal.timeout(Math.max(1, Math.floor(deadline - this.now()))),
]),
});
if (!response.ok) throw new CatalogConnectorError("REST sensitivity source scan failed");
const body: unknown = await response.json();
if (!Array.isArray(body)
|| body.some((row) => !row || typeof row !== "object" || Array.isArray(row))) {
throw new CatalogConnectorError("REST sensitivity source response is invalid");
const runQuery = async (sql: string): Promise<Array<Record<string, unknown>> | undefined> => {
const timeout = AbortSignal.timeout(Math.max(1, Math.floor(request.queryTimeoutMs)));
try {
const response = await fetch(`${baseUrl}/rpc/run_query`, {
method: "POST",
headers,
body: JSON.stringify({ query_text: sql }),
signal: AbortSignal.any([signal, timeout]),
});
if (!response.ok) throw new CatalogConnectorError("REST sensitivity source scan failed");
const body: unknown = await response.json();
if (!Array.isArray(body)
|| body.some((row) => !row || typeof row !== "object" || Array.isArray(row))) {
throw new CatalogConnectorError("REST sensitivity source response is invalid");
}
return body as Array<Record<string, unknown>>;
} catch (error) {
if (signal.aborted) throw error;
if (timeout.aborted) return undefined;
throw error;
}
return body as Array<Record<string, unknown>>;
};
let offset = 0;
const baseSelect = [
`SELECT ${projections(request)}`,
`FROM ${quoteIdentifier(request.database.schema)}.${quoteIdentifier(request.table.name)}`,
].join(" ");
while (!signal.aborted) {
let rows: Array<Record<string, unknown>>;
try {
rows = await runQuery(
`${baseSelect} LIMIT ${this.batchRows} OFFSET ${offset}`,
fullDeadline,
);
} catch (error) {
if (signal.aborted || this.now() < fullDeadline) throw error;
break;
}
observedRows += rows.length;
if (rows.length > 0) await consume(observations(request, rows));
if (rows.length < this.batchRows) {
return { kind: offset === 0 ? "complete" : "sampled", observedRows };
}
offset += rows.length;
if (this.now() >= fullDeadline) break;
let complete = false;
if (request.fullScanThreshold !== undefined) {
const probe = await runQuery(
`SELECT 1 AS __present FROM ${tableReference(request)} LIMIT ${request.fullScanThreshold + 1}`,
);
complete = probe !== undefined && probe.length <= request.fullScanThreshold;
}
if (signal.aborted || this.now() >= request.deadline) {
return { kind: "sampled", observedRows };
let requestCount = request.fullScanThreshold === undefined ? 0 : 1;
for (const columnChunk of chunks(request.columns, MAX_COLUMNS_PER_QUERY)) {
signal.throwIfAborted();
let rows = await runQuery(flatValueQuery(request, columnChunk, {
complete,
randomized: !complete,
}));
requestCount += 1;
if (rows === undefined && complete) {
complete = false;
rows = await runQuery(flatValueQuery(request, columnChunk, {
complete: false,
randomized: true,
}));
requestCount += 1;
}
if (!complete && (rows === undefined || rows.length === 0)) {
rows = await runQuery(flatValueQuery(request, columnChunk, {
complete: false,
randomized: false,
}));
requestCount += 1;
}
if (rows === undefined) throw new CatalogConnectorError("REST sensitivity sample query timed out");
const batch = observations(columnChunk, rows);
observedValues += batch.length;
if (batch.length > 0) await consume(batch);
}
const sampleSql = [
baseSelect,
"TABLESAMPLE SYSTEM (1) REPEATABLE (37)",
`LIMIT ${this.sampleRows}`,
].join(" ");
const sampledRows = await runQuery(sampleSql, request.deadline);
observedRows += sampledRows.length;
if (sampledRows.length > 0) await consume(observations(request, sampledRows));
return { kind: "sampled", observedRows };
// Multiple HTTP requests cannot share a source snapshot, so only one-request reads are complete.
return { kind: complete && requestCount === 1 ? "complete" : "sampled", observedValues };
} catch (error) {
if (error instanceof CatalogConnectorError) throw error;
throw new CatalogConnectorError("REST sensitivity source scan failed");
backend/src/routes/catalog-description-generation.ts
+19 -36
View File
@@ -261,9 +261,9 @@ function safeSuggestionError(reply: FastifyReply, error: unknown) {
});
}
if (error instanceof SensitivityAnalysisInterruptedError) {
return reply.code(504).send({
code: "sensitivity_analysis_timeout",
message: "Sensitivity analysis reached its time limit. No assessments were applied.",
return reply.code(499).send({
code: "sensitivity_analysis_interrupted",
message: "Sensitivity analysis was interrupted before completion. No assessments were applied.",
});
}
if (error instanceof CatalogConnectorError) {
@@ -284,32 +284,6 @@ function safeSuggestionError(reply: FastifyReply, error: unknown) {
});
}
function untilAborted<T>(operation: Promise<T>, signal: AbortSignal): Promise<T> {
if (signal.aborted) {
void operation.catch(() => undefined);
return Promise.reject(new SensitivityAnalysisInterruptedError());
}
return new Promise<T>((resolve, reject) => {
const abort = () => reject(new SensitivityAnalysisInterruptedError());
signal.addEventListener("abort", abort, { once: true });
if (signal.aborted) {
void operation.catch(() => undefined);
abort();
return;
}
operation.then(
(value) => {
signal.removeEventListener("abort", abort);
resolve(value);
},
(error: unknown) => {
signal.removeEventListener("abort", abort);
reject(error);
},
);
});
}
function safeSuggestionHistoryError(reply: FastifyReply, error: unknown) {
if (error instanceof CatalogUnavailableError) {
return reply.code(503).send({
@@ -342,13 +316,22 @@ export function catalogDescriptionGenerationRoutes(
try {
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
const input = suggestionSchema.parse(request.body);
const signal = AbortSignal.timeout(60_000);
const result = await untilAborted(deps.sensitivityAnalysisRunner.run(
databaseId,
input.scope,
"targetIds" in input ? input.targetIds : [],
signal,
), signal);
const controller = new AbortController();
const abort = () => controller.abort();
request.raw.once("aborted", abort);
reply.raw.once("close", abort);
let result;
try {
result = await deps.sensitivityAnalysisRunner.run(
databaseId,
input.scope,
"targetIds" in input ? input.targetIds : [],
controller.signal,
);
} finally {
request.raw.off("aborted", abort);
reply.raw.off("close", abort);
}
return {
suggestions: result.suggestions,
run: publicSensitivityAnalysisRun(result.run),