mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: complete catalog sensitivity enhancements

Browse Source
This commit is contained in:
Codex
2026-09-04 15:11:18 +02:00
parent b891246664
commit 7b1d69a65b
72 changed files with 33866 additions and 358 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/test/catalog-repository.integration.test.ts
+48 -4
View File
@@ -17,6 +17,7 @@ import { up as upLogicalRelationships } from "../src/catalog/migrations/008_cata
import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js";
import { up as upCanonicalModelIds } from "../src/catalog/migrations/010_canonical_model_ids.js";
import { up as upLocalSensitivityAnalysis } from "../src/catalog/migrations/011_local_sensitivity_analysis.js";
import { up as upSensitivityReason } from "../src/catalog/migrations/012_sensitivity_reason.js";
const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
@@ -56,6 +57,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
}).execute();
await upCanonicalModelIds(db);
await upLocalSensitivityAnalysis(db);
await upSensitivityReason(db);
await expect(db.selectFrom("sensitiveDataSuggestionRuns")
.select(["engine", "modelId", "policyVersion", "unknown"])
.where("id", "=", historicalSuggestionRunId)
@@ -142,7 +144,11 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
patientName.description,
patientName.generatedDescription,
true,
)).toMatchObject({ sensitive: true });
"Local assessment matched content rule pii.person_name.",
)).toMatchObject({
sensitive: true,
sensitivityReason: "Local assessment matched content rule pii.person_name.",
});
expect(await repository.getCatalogMetrics(created.id)).toEqual({
scope: "database",
databaseId: created.id,
@@ -187,6 +193,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
)).toMatchObject({ updated: 1 });
expect(await repository.getColumn(created.id, patients.id, patientName.id)).toMatchObject({
sensitive: true,
sensitivityReason: "Local assessment matched content rule pii.person_name.",
sourceComment: "Sensitive patient name",
});
@@ -281,10 +288,33 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository performs scoped metadata cl
};
await repository.applySchemaSync(database.id, database.version, "all", [], snapshot);
const patients = (await repository.listTables(database.id)).find((table) => table.name === "patients")!;
const context = (await repository.getLogicalRelationshipContext(database.id))!;
const source = context.endpoints.find((endpoint) => (
endpoint.tableName === "visits" && endpoint.columnName === "id"
))!;
const target = context.endpoints.find((endpoint) => (
endpoint.tableName === "patients" && endpoint.columnName === "id"
))!;
const generatedCandidate = { sourceColumnId: source.columnId, targetColumnId: target.columnId };
expect(await repository.deleteTableMetadata(database.id, [patients.id], "relationships"))
.toEqual({ tables: 0, columns: 0, relationships: 1 });
await expect(repository.insertGeneratedLogicalRelationships(database.id, [generatedCandidate]))
.resolves.toBe(1);
await expect(repository.getCatalogMetrics(database.id))
.resolves.toMatchObject({ relationships: 2 });
expect(await repository.deleteDatabaseMetadata([database.id], "relationships"))
.toEqual({ tables: 0, columns: 0, relationships: 2 });
expect(await repository.listRelationships(database.id)).toEqual([]);
expect(await repository.listLogicalRelationships(database.id)).toEqual([]);
await expect(repository.getCatalogMetrics(database.id))
.resolves.toMatchObject({ relationships: 0 });
await repository.applySchemaSync(database.id, database.version, "relationships", [], snapshot);
await expect(repository.insertGeneratedLogicalRelationships(database.id, [generatedCandidate]))
.resolves.toBe(1);
expect(await repository.deleteTableMetadata(database.id, [patients.id], "relationships"))
.toEqual({ tables: 0, columns: 0, relationships: 2 });
expect(await repository.listRelationships(database.id)).toEqual([]);
expect(await repository.listLogicalRelationships(database.id)).toEqual([]);
expect(await repository.listColumns(database.id, patients.id)).toHaveLength(2);
expect((await repository.get(database.id))?.schemaSyncedVersion).toBeUndefined();
@@ -298,13 +328,24 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository performs scoped metadata cl
await repository.applySchemaSync(database.id, database.version, "columns", [patients.id], snapshot);
await repository.applySchemaSync(database.id, database.version, "relationships", [], snapshot);
const refreshedContext = (await repository.getLogicalRelationshipContext(database.id))!;
const refreshedSource = refreshedContext.endpoints.find((endpoint) => (
endpoint.tableName === "visits" && endpoint.columnName === "id"
))!;
const refreshedTarget = refreshedContext.endpoints.find((endpoint) => (
endpoint.tableName === "patients" && endpoint.columnName === "id"
))!;
await expect(repository.insertGeneratedLogicalRelationships(database.id, [{
sourceColumnId: refreshedSource.columnId,
targetColumnId: refreshedTarget.columnId,
}])).resolves.toBe(1);
expect(await repository.deleteDatabaseMetadata([
database.id,
"99999999-9999-4999-8999-999999999999",
], "tables")).toBeUndefined();
expect(await repository.listTables(database.id)).toHaveLength(2);
expect(await repository.deleteDatabaseMetadata([database.id], "tables"))
.toEqual({ tables: 2, columns: 4, relationships: 1 });
.toEqual({ tables: 2, columns: 4, relationships: 2 });
expect(await repository.get(database.id)).toBeDefined();
expect(await repository.listTables(database.id)).toEqual([]);
expect(await repository.listRelationships(database.id)).toEqual([]);
@@ -431,6 +472,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
await upAiTokenUsage(db);
await upCanonicalModelIds(db);
await upLocalSensitivityAnalysis(db);
await upSensitivityReason(db);
const repository = new KyselyCatalogRepository(db);
const firstDatabase = await repository.create({
workspaceId: "generation-one",
@@ -703,6 +745,8 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists logical relationsh
const target = context.endpoints.find((item) => item.tableName === "users" && item.columnName === "id")!;
const created = await repository.insertLogicalRelationship(database.id, source.columnId, target.columnId, false);
expect(created).toMatchObject({ origin: "manual", status: "active" });
expect(await repository.getCatalogMetrics(database.id))
.toMatchObject({ relationships: 1 });
await expect(repository.insertLogicalRelationship(database.id, source.columnId, target.columnId, true))
.resolves.toBeUndefined();