mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: complete catalog sensitivity enhancements

Browse Source
This commit is contained in:
Codex
2026-09-04 15:11:18 +02:00
parent b891246664
commit 7b1d69a65b
72 changed files with 33866 additions and 358 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/catalog/repository.ts
+74 -16
View File
@@ -118,6 +118,7 @@ interface CatalogColumnTable {
description: string | null;
generatedDescription: string | null;
sensitive: Generated<boolean>;
sensitivityReason: Generated<string | null>;
lastSyncedDatabaseVersion: number | null;
lastSyncedAt: Timestamp | null;
version: Generated<number>;
@@ -360,6 +361,7 @@ function serializeColumn(row: Selectable<CatalogColumnTable>, foreignKeyCount =
description: row.description,
generatedDescription: row.generatedDescription,
sensitive: row.sensitive,
sensitivityReason: row.sensitivityReason,
lastSyncedDatabaseVersion: row.lastSyncedDatabaseVersion,
lastSyncedAt: row.lastSyncedAt === null ? null : new Date(row.lastSyncedAt).toISOString(),
version: row.version,
@@ -516,10 +518,18 @@ export class KyselyCatalogRepository implements CatalogRepository {
relationship_metrics AS (
SELECT
count(*)::int AS relationships,
max(catalog_relationships.updated_at) AS updated_at
FROM catalog_relationships
INNER JOIN selected_databases
ON selected_databases.id = catalog_relationships.database_id
max(relationship.updated_at) AS updated_at
FROM (
SELECT catalog_relationships.updated_at
FROM catalog_relationships
INNER JOIN selected_databases
ON selected_databases.id = catalog_relationships.database_id
UNION ALL
SELECT catalog_logical_relationships.updated_at
FROM catalog_logical_relationships
INNER JOIN selected_databases
ON selected_databases.id = catalog_logical_relationships.database_id
) AS relationship
)
SELECT
(SELECT count(*)::int FROM selected_databases) AS "databaseCount",
@@ -728,6 +738,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
description: string | null,
generatedDescription: string | null,
sensitive?: boolean,
sensitivityReason?: string | null,
): Promise<CatalogColumn | undefined> {
const belongs = await this.db.selectFrom("catalogTables").select("id")
.where("id", "=", tableId).where("databaseId", "=", databaseId).executeTakeFirst();
@@ -736,6 +747,9 @@ export class KyselyCatalogRepository implements CatalogRepository {
description,
generatedDescription,
...(sensitive === undefined ? {} : { sensitive }),
...(sensitive === false
? { sensitivityReason: null }
: sensitivityReason === undefined ? {} : { sensitivityReason }),
version: sql`version + 1`,
updatedAt: sql`now()`,
}).where("id", "=", columnId).where("tableId", "=", tableId)
@@ -749,7 +763,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
targetIds: readonly string[],
): Promise<CatalogDescriptionConsolidationCounts | undefined> {
const selectedTargetIds = [...new Set(targetIds)];
if (selectedTargetIds.length === 0) return undefined;
if (target !== "database_columns" && selectedTargetIds.length === 0) return undefined;
return await this.db.transaction().execute(async (trx) => {
const database = await trx.selectFrom("workspaceDatabases").select("id")
.where("id", "=", databaseId).forUpdate().executeTakeFirst();
@@ -784,24 +798,30 @@ export class KyselyCatalogRepository implements CatalogRepository {
const rows = tableRows.length === 0 ? [] : await trx.selectFrom("catalogColumns")
.select(["id", "generatedDescription"])
.where("tableId", "in", tableRows.map((table) => table.id))
.where("id", "in", selectedTargetIds)
.$if(target === "columns", (query) => query.where("id", "in", selectedTargetIds))
.orderBy("id")
.forUpdate()
.execute();
if (rows.length !== selectedTargetIds.length) return undefined;
if (target === "columns" && rows.length !== selectedTargetIds.length) return undefined;
const copiedIds = rows
.filter((row) => Boolean(row.generatedDescription?.trim()))
.map((row) => row.id);
if (copiedIds.length > 0) {
await trx.updateTable("catalogColumns").set({
let update = trx.updateTable("catalogColumns").set({
description: sql`generated_description`,
version: sql`version + 1`,
updatedAt: sql`now()`,
}).where("id", "in", copiedIds).execute();
});
update = target === "database_columns"
? update
.where("tableId", "in", tableRows.map((table) => table.id))
.where(sql<boolean>`nullif(btrim(generated_description), '') is not null`)
: update.where("id", "in", copiedIds);
await update.execute();
}
return {
copied: copiedIds.length,
skipped: selectedTargetIds.length - copiedIds.length,
skipped: rows.length - copiedIds.length,
};
});
}
@@ -1261,16 +1281,25 @@ export class KyselyCatalogRepository implements CatalogRepository {
if (databases.length !== selectedDatabaseIds.length) return undefined;
if (target === "relationships") {
const count = await trx.selectFrom("catalogRelationships")
const physicalCount = await trx.selectFrom("catalogRelationships")
.select(sql<number>`count(*)::int`.as("count"))
.where("databaseId", "in", selectedDatabaseIds).executeTakeFirst();
const logicalCount = await trx.selectFrom("catalogLogicalRelationships")
.select(sql<number>`count(*)::int`.as("count"))
.where("databaseId", "in", selectedDatabaseIds).executeTakeFirst();
await trx.deleteFrom("catalogLogicalRelationships")
.where("databaseId", "in", selectedDatabaseIds).execute();
await trx.deleteFrom("catalogRelationships")
.where("databaseId", "in", selectedDatabaseIds).execute();
await trx.updateTable("workspaceDatabases").set({
schemaSyncedVersion: null,
schemaSyncedAt: null,
}).where("id", "in", selectedDatabaseIds).execute();
return { tables: 0, columns: 0, relationships: Number(count?.count ?? 0) };
return {
tables: 0,
columns: 0,
relationships: Number(physicalCount?.count ?? 0) + Number(logicalCount?.count ?? 0),
};
}
const tableCount = await trx.selectFrom("catalogTables")
@@ -1280,7 +1309,10 @@ export class KyselyCatalogRepository implements CatalogRepository {
.innerJoin("catalogTables", "catalogTables.id", "catalogColumns.tableId")
.select(sql<number>`count(*)::int`.as("count"))
.where("catalogTables.databaseId", "in", selectedDatabaseIds).executeTakeFirst();
const relationshipCount = await trx.selectFrom("catalogRelationships")
const physicalRelationshipCount = await trx.selectFrom("catalogRelationships")
.select(sql<number>`count(*)::int`.as("count"))
.where("databaseId", "in", selectedDatabaseIds).executeTakeFirst();
const logicalRelationshipCount = await trx.selectFrom("catalogLogicalRelationships")
.select(sql<number>`count(*)::int`.as("count"))
.where("databaseId", "in", selectedDatabaseIds).executeTakeFirst();
await trx.deleteFrom("catalogTables")
@@ -1292,7 +1324,8 @@ export class KyselyCatalogRepository implements CatalogRepository {
return {
tables: Number(tableCount?.count ?? 0),
columns: Number(columnCount?.count ?? 0),
relationships: Number(relationshipCount?.count ?? 0),
relationships: Number(physicalRelationshipCount?.count ?? 0)
+ Number(logicalRelationshipCount?.count ?? 0),
};
});
}
@@ -1326,13 +1359,34 @@ export class KyselyCatalogRepository implements CatalogRepository {
return { tables: 0, columns: Number(count?.count ?? 0), relationships: 0 };
}
const count = await trx.selectFrom("catalogRelationships")
const physicalCount = await trx.selectFrom("catalogRelationships")
.select(sql<number>`count(*)::int`.as("count"))
.where("databaseId", "=", databaseId)
.where((eb) => eb.or([
eb("sourceTableId", "in", selectedTableIds),
eb("targetTableId", "in", selectedTableIds),
])).executeTakeFirst();
const selectedColumnIds = (await trx.selectFrom("catalogColumns")
.select("id")
.where("tableId", "in", selectedTableIds)
.execute()).map((column) => column.id);
const logicalCount = selectedColumnIds.length === 0
? undefined
: await trx.selectFrom("catalogLogicalRelationships")
.select(sql<number>`count(*)::int`.as("count"))
.where("databaseId", "=", databaseId)
.where((eb) => eb.or([
eb("sourceColumnId", "in", selectedColumnIds),
eb("targetColumnId", "in", selectedColumnIds),
])).executeTakeFirst();
if (selectedColumnIds.length > 0) {
await trx.deleteFrom("catalogLogicalRelationships")
.where("databaseId", "=", databaseId)
.where((eb) => eb.or([
eb("sourceColumnId", "in", selectedColumnIds),
eb("targetColumnId", "in", selectedColumnIds),
])).execute();
}
await trx.deleteFrom("catalogRelationships")
.where("databaseId", "=", databaseId)
.where((eb) => eb.or([
@@ -1343,7 +1397,11 @@ export class KyselyCatalogRepository implements CatalogRepository {
schemaSyncedVersion: null,
schemaSyncedAt: null,
}).where("id", "=", databaseId).execute();
return { tables: 0, columns: 0, relationships: Number(count?.count ?? 0) };
return {
tables: 0,
columns: 0,
relationships: Number(physicalCount?.count ?? 0) + Number(logicalCount?.count ?? 0),
};
});
}