mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: protect sensitive catalog samples

Browse Source
This commit is contained in:
Codex
2026-08-30 12:14:23 +02:00
parent 6278ee9d81
commit 0736983bc5
28 changed files with 1162 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/catalog/description-generation-worker.ts
+49 -23
View File
@@ -12,6 +12,7 @@ import type {
DescriptionSourceSampleValue,
DescriptionTargetSourceSample,
} from "./description-source-sampler.js";
import { syntheticSampleValue } from "./synthetic-sample-value.js";
import {
DescriptionGenerationRunActiveError,
type CatalogColumn,
@@ -306,35 +307,55 @@ function sourceSampleFor(
): PromptSourceSample | undefined {
const targetId = target.kind === "column" ? target.column.id : target.table.id;
const sample = samples.find((candidate) => candidate.targetId === targetId);
if (!sample) return undefined;
const relevantColumns = new Set(
target.kind === "column" ? [target.column.name] : target.columns.map((column) => column.name),
);
const rows = sample.rows.slice(0, budget.rows).map((row) => {
const columns = target.kind === "column" ? [target.column] : target.columns;
const sourceRows = sample?.rows ?? [];
const hasSensitiveColumns = columns.some((column) => column.sensitive);
const hasNonSensitiveColumns = columns.some((column) => !column.sensitive);
const realRowCount = hasNonSensitiveColumns
? Math.min(sourceRows.length, budget.rows)
: 0;
const rowCount = hasSensitiveColumns ? MAX_SAMPLE_ROWS_PER_REQUEST : realRowCount;
const rows = Array.from({ length: rowCount }, (_, rowIndex) => {
const row = rowIndex < realRowCount ? sourceRows[rowIndex] : undefined;
const sourceFields = new Map((row?.fields ?? []).map((field) => [field.name, field.value]));
const fields: Array<{ name: string; value: PromptSampleValue }> = [];
const seen = new Set<string>();
for (const field of row.fields) {
if (!relevantColumns.has(field.name) || seen.has(field.name)) continue;
seen.add(field.name);
for (const column of columns) {
const value = column.sensitive
? syntheticSampleValue(column, rowIndex + 1)
: sourceFields.get(column.name);
if (value === undefined) continue;
fields.push({
name: boundedJsonText(field.name, MAX_IDENTIFIER_JSON_BYTES),
value: promptSampleValue(field.value),
name: boundedJsonText(column.name, MAX_IDENTIFIER_JSON_BYTES),
value: promptSampleValue(value),
});
if (fields.length === MAX_SAMPLE_FIELDS_PER_ROW) break;
}
return { fields };
});
budget.rows -= rows.length;
budget.rows -= realRowCount;
const representativeValues: PromptSourceSample["representativeValues"] = [];
const seenColumns = new Set<string>();
let remainingRepresentativeValues = budget.representativeValues;
for (const examples of sample.representativeValues) {
if (remainingRepresentativeValues === 0) break;
if (!relevantColumns.has(examples.column) || seenColumns.has(examples.column)) continue;
seenColumns.add(examples.column);
let remainingRealRepresentativeValues = budget.representativeValues;
let remainingSyntheticRepresentativeValues = MAX_REPRESENTATIVE_VALUES_PER_REQUEST;
for (const column of columns) {
if (seenColumns.has(column.name)) continue;
const remainingRepresentativeValues = column.sensitive
? remainingSyntheticRepresentativeValues
: remainingRealRepresentativeValues;
if (remainingRepresentativeValues === 0) continue;
const sourceExamples = sample?.representativeValues.find(
(examples) => examples.column === column.name,
);
const exampleValues = column.sensitive
? Array.from(
{ length: Math.min(Math.max(rowCount, 1), remainingRepresentativeValues) },
(_, index) => syntheticSampleValue(column, index + 1),
)
: sourceExamples?.values ?? [];
seenColumns.add(column.name);
const values: Array<Exclude<PromptSampleValue, null>> = [];
const seenValues = new Set<string>();
for (const value of examples.values) {
for (const value of exampleValues) {
const normalized = promptSampleValue(value);
if (normalized === null) continue;
const key = JSON.stringify([typeof normalized, normalized]);
@@ -345,11 +366,15 @@ function sourceSampleFor(
}
if (values.length > 0) {
representativeValues.push({
column: boundedJsonText(examples.column, MAX_IDENTIFIER_JSON_BYTES),
column: boundedJsonText(column.name, MAX_IDENTIFIER_JSON_BYTES),
values,
});
remainingRepresentativeValues -= values.length;
budget.representativeValues -= values.length;
if (column.sensitive) {
remainingSyntheticRepresentativeValues -= values.length;
} else {
remainingRealRepresentativeValues -= values.length;
budget.representativeValues -= values.length;
}
}
if (representativeValues.length === MAX_SAMPLE_COLUMNS) break;
}
@@ -933,8 +958,9 @@ export class DescriptionGenerationWorker {
targetId: target.kind === "column" ? target.column.id : target.table.id,
tableName: target.table.name,
columnNames: target.kind === "column"
? [target.column.name]
: target.columns.map((column) => column.name),
? target.column.sensitive ? [] : [target.column.name]
: target.columns.filter((column) => !column.sensitive)
.map((column) => column.name),
})),
signal,
);