mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: add metadata catalog cleanup commands

Browse Source
This commit is contained in:
Codex
2026-08-28 00:37:43 +02:00
parent 79c4c925b5
commit 58ee9cffe4
19 changed files with 942 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/catalog/memory-repository.ts
+80
View File
@@ -3,6 +3,8 @@ import {
CatalogConflictError,
CatalogConnectorError,
type CatalogColumn,
type CatalogDatabaseMetadataDeleteTarget,
type CatalogMetadataDeleteCounts,
type CatalogRelationship,
type CatalogSchemaDiff,
type CatalogSyncCounts,
@@ -11,6 +13,7 @@ import {
type CatalogSyncRunUpdate,
type CatalogSyncScope,
type CatalogTable,
type CatalogTableMetadataDeleteTarget,
type CatalogRepository,
type DatabaseConfigurationInput,
type DatabaseTestResult,
@@ -199,6 +202,71 @@ export class MemoryCatalogRepository implements CatalogRepository {
.map((relationship) => structuredClone(relationship));
}
async deleteDatabaseMetadata(
databaseIds: readonly string[],
target: CatalogDatabaseMetadataDeleteTarget,
): Promise<CatalogMetadataDeleteCounts | undefined> {
const selectedDatabaseIds = [...new Set(databaseIds)];
if (selectedDatabaseIds.length === 0 || selectedDatabaseIds.some((id) => !this.records.has(id))) {
return undefined;
}
const selected = new Set(selectedDatabaseIds);
const tables = [...this.tables.values()].filter((table) => selected.has(table.databaseId));
const tableIds = new Set(tables.map((table) => table.id));
const columns = [...this.columns.values()].filter((column) => tableIds.has(column.tableId));
const relationships = [...this.relationships.values()]
.filter((relationship) => selected.has(relationship.databaseId));
if (target === "tables") {
for (const table of tables) this.deleteTable(table.id);
this.markCatalogIncomplete(selectedDatabaseIds);
return { tables: tables.length, columns: columns.length, relationships: relationships.length };
}
for (const relationship of relationships) this.relationships.delete(relationship.id);
for (const databaseId of selectedDatabaseIds) this.refreshForeignKeyFlags(databaseId);
this.markCatalogIncomplete(selectedDatabaseIds);
return { tables: 0, columns: 0, relationships: relationships.length };
}
async deleteTableMetadata(
databaseId: string,
tableIds: readonly string[],
target: CatalogTableMetadataDeleteTarget,
): Promise<CatalogMetadataDeleteCounts | undefined> {
const selectedTableIds = [...new Set(tableIds)];
if (!this.records.has(databaseId) || selectedTableIds.length === 0) return undefined;
const tables = selectedTableIds.map((id) => this.tables.get(id));
if (tables.some((table) => !table || table.databaseId !== databaseId)) return undefined;
const selected = new Set(selectedTableIds);
if (target === "columns") {
const columns = [...this.columns.values()].filter((column) => selected.has(column.tableId));
const deletedColumnIds = new Set(columns.map((column) => column.id));
for (const column of columns) this.columns.delete(column.id);
for (const [relationshipId, relationship] of this.relationships) {
if (relationship.databaseId !== databaseId) continue;
this.relationships.set(relationshipId, {
...relationship,
columns: relationship.columns.filter((pair) => (
!deletedColumnIds.has(pair.sourceColumnId) && !deletedColumnIds.has(pair.targetColumnId)
)),
});
}
this.refreshForeignKeyFlags(databaseId);
this.markCatalogIncomplete([databaseId]);
return { tables: 0, columns: columns.length, relationships: 0 };
}
const relationships = [...this.relationships.values()].filter((relationship) => (
relationship.databaseId === databaseId
&& (selected.has(relationship.sourceTableId) || selected.has(relationship.targetTableId))
));
for (const relationship of relationships) this.relationships.delete(relationship.id);
this.refreshForeignKeyFlags(databaseId);
this.markCatalogIncomplete([databaseId]);
return { tables: 0, columns: 0, relationships: relationships.length };
}
async planSchemaSync(
databaseId: string,
scope: CatalogSyncScope,
@@ -675,6 +743,18 @@ export class MemoryCatalogRepository implements CatalogRepository {
}
}
private markCatalogIncomplete(databaseIds: readonly string[]): void {
for (const databaseId of databaseIds) {
const database = this.records.get(databaseId);
if (!database) continue;
this.records.set(databaseId, {
...database,
schemaSyncedVersion: undefined,
schemaSyncedAt: undefined,
});
}
}
private refreshForeignKeyFlags(databaseId: string): void {
const counts = new Map<string, number>();
for (const relationship of this.relationships.values()) {
backend/src/catalog/operation-coordinator.ts
+13 -1
View File
@@ -1,6 +1,6 @@
import { CatalogOperationInProgressError } from "./types.js";
/** Serializes connection tests and schema synchronization for each catalog database. */
/** Serializes connection tests, synchronization, and metadata cleanup for each catalog database. */
export class CatalogOperationCoordinator {
private readonly active = new Set<string>();
@@ -25,4 +25,16 @@ export class CatalogOperationCoordinator {
release();
}
}
async runMany<T>(databaseIds: readonly string[], operation: () => Promise<T>): Promise<T> {
const releases: Array<() => void> = [];
try {
for (const databaseId of [...new Set(databaseIds)].sort()) {
releases.push(this.reserve(databaseId));
}
return await operation();
} finally {
for (const release of releases.reverse()) release();
}
}
}
backend/src/catalog/repository.ts
+103
View File
@@ -16,6 +16,8 @@ import {
CatalogConnectorError,
CatalogUnavailableError,
type CatalogColumn,
type CatalogDatabaseMetadataDeleteTarget,
type CatalogMetadataDeleteCounts,
type CatalogRelationship,
type CatalogSchemaDiff,
type CatalogSyncCounts,
@@ -24,6 +26,7 @@ import {
type CatalogSyncRunUpdate,
type CatalogSyncScope,
type CatalogTable,
type CatalogTableMetadataDeleteTarget,
type CatalogRepository,
type DatabaseBinding,
type DatabaseConfigurationInput,
@@ -559,6 +562,104 @@ export class KyselyCatalogRepository implements CatalogRepository {
}));
}
async deleteDatabaseMetadata(
databaseIds: readonly string[],
target: CatalogDatabaseMetadataDeleteTarget,
): Promise<CatalogMetadataDeleteCounts | undefined> {
const selectedDatabaseIds = [...new Set(databaseIds)];
if (selectedDatabaseIds.length === 0) return undefined;
return await this.db.transaction().execute(async (trx) => {
const databases = await trx.selectFrom("workspaceDatabases").select("id")
.where("id", "in", selectedDatabaseIds).orderBy("id").forUpdate().execute();
if (databases.length !== selectedDatabaseIds.length) return undefined;
if (target === "relationships") {
const count = await trx.selectFrom("catalogRelationships")
.select(sql<number>`count(*)::int`.as("count"))
.where("databaseId", "in", selectedDatabaseIds).executeTakeFirst();
await trx.deleteFrom("catalogRelationships")
.where("databaseId", "in", selectedDatabaseIds).execute();
await trx.updateTable("workspaceDatabases").set({
schemaSyncedVersion: null,
schemaSyncedAt: null,
}).where("id", "in", selectedDatabaseIds).execute();
return { tables: 0, columns: 0, relationships: Number(count?.count ?? 0) };
}
const tableCount = await trx.selectFrom("catalogTables")
.select(sql<number>`count(*)::int`.as("count"))
.where("databaseId", "in", selectedDatabaseIds).executeTakeFirst();
const columnCount = await trx.selectFrom("catalogColumns")
.innerJoin("catalogTables", "catalogTables.id", "catalogColumns.tableId")
.select(sql<number>`count(*)::int`.as("count"))
.where("catalogTables.databaseId", "in", selectedDatabaseIds).executeTakeFirst();
const relationshipCount = await trx.selectFrom("catalogRelationships")
.select(sql<number>`count(*)::int`.as("count"))
.where("databaseId", "in", selectedDatabaseIds).executeTakeFirst();
await trx.deleteFrom("catalogTables")
.where("databaseId", "in", selectedDatabaseIds).execute();
await trx.updateTable("workspaceDatabases").set({
schemaSyncedVersion: null,
schemaSyncedAt: null,
}).where("id", "in", selectedDatabaseIds).execute();
return {
tables: Number(tableCount?.count ?? 0),
columns: Number(columnCount?.count ?? 0),
relationships: Number(relationshipCount?.count ?? 0),
};
});
}
async deleteTableMetadata(
databaseId: string,
tableIds: readonly string[],
target: CatalogTableMetadataDeleteTarget,
): Promise<CatalogMetadataDeleteCounts | undefined> {
const selectedTableIds = [...new Set(tableIds)];
if (selectedTableIds.length === 0) return undefined;
return await this.db.transaction().execute(async (trx) => {
const database = await trx.selectFrom("workspaceDatabases").select("id")
.where("id", "=", databaseId).forUpdate().executeTakeFirst();
if (!database) return undefined;
const tables = await trx.selectFrom("catalogTables").select("id")
.where("databaseId", "=", databaseId)
.where("id", "in", selectedTableIds).execute();
if (tables.length !== selectedTableIds.length) return undefined;
if (target === "columns") {
const count = await trx.selectFrom("catalogColumns")
.select(sql<number>`count(*)::int`.as("count"))
.where("tableId", "in", selectedTableIds).executeTakeFirst();
await trx.deleteFrom("catalogColumns")
.where("tableId", "in", selectedTableIds).execute();
await trx.updateTable("workspaceDatabases").set({
schemaSyncedVersion: null,
schemaSyncedAt: null,
}).where("id", "=", databaseId).execute();
return { tables: 0, columns: Number(count?.count ?? 0), relationships: 0 };
}
const count = await trx.selectFrom("catalogRelationships")
.select(sql<number>`count(*)::int`.as("count"))
.where("databaseId", "=", databaseId)
.where((eb) => eb.or([
eb("sourceTableId", "in", selectedTableIds),
eb("targetTableId", "in", selectedTableIds),
])).executeTakeFirst();
await trx.deleteFrom("catalogRelationships")
.where("databaseId", "=", databaseId)
.where((eb) => eb.or([
eb("sourceTableId", "in", selectedTableIds),
eb("targetTableId", "in", selectedTableIds),
])).execute();
await trx.updateTable("workspaceDatabases").set({
schemaSyncedVersion: null,
schemaSyncedAt: null,
}).where("id", "=", databaseId).execute();
return { tables: 0, columns: 0, relationships: Number(count?.count ?? 0) };
});
}
async planSchemaSync(
databaseId: string,
scope: CatalogSyncScope,
@@ -1002,6 +1103,8 @@ export class UnavailableCatalogRepository implements CatalogRepository {
async getColumn(): Promise<CatalogColumn | undefined> { return this.fail(); }
async updateColumnMetadata(): Promise<CatalogColumn | undefined> { return this.fail(); }
async listRelationships(): Promise<CatalogRelationship[]> { return this.fail(); }
async deleteDatabaseMetadata(): Promise<CatalogMetadataDeleteCounts | undefined> { return this.fail(); }
async deleteTableMetadata(): Promise<CatalogMetadataDeleteCounts | undefined> { return this.fail(); }
async planSchemaSync(): Promise<CatalogSchemaDiff> { return this.fail(); }
async applySchemaSync(): Promise<CatalogSyncCounts | undefined> { return this.fail(); }
async createSyncRun(): Promise<CatalogSyncRun> { return this.fail(); }
backend/src/catalog/types.ts
+18
View File
@@ -133,6 +133,15 @@ export interface CatalogRelationship {
updatedAt: string;
}
export type CatalogDatabaseMetadataDeleteTarget = "tables" | "relationships";
export type CatalogTableMetadataDeleteTarget = "columns" | "relationships";
export interface CatalogMetadataDeleteCounts {
tables: number;
columns: number;
relationships: number;
}
export interface ObservedRelationshipColumn {
position: number;
sourceColumnName: string;
@@ -284,6 +293,15 @@ export interface CatalogRepository {
generatedDescription: string | null,
): Promise<CatalogColumn | undefined>;
listRelationships(databaseId: string): Promise<CatalogRelationship[]>;
deleteDatabaseMetadata(
databaseIds: readonly string[],
target: CatalogDatabaseMetadataDeleteTarget,
): Promise<CatalogMetadataDeleteCounts | undefined>;
deleteTableMetadata(
databaseId: string,
tableIds: readonly string[],
target: CatalogTableMetadataDeleteTarget,
): Promise<CatalogMetadataDeleteCounts | undefined>;
planSchemaSync(
databaseId: string,
scope: CatalogSyncScope,