From 9c697dc062076c469b3807f1544a8d3ef2acd10f Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 31 Aug 2026 15:58:43 +0200 Subject: [PATCH] feat: complete catalog fleet management workflow --- backend/src/app.ts | 8 +- backend/src/catalog/memory-repository.ts | 149 +++ backend/src/catalog/metrics.ts | 53 + backend/src/catalog/migrate.ts | 2 + .../007_sensitive_data_suggestion_runs.ts | 73 ++ backend/src/catalog/repository.ts | 255 +++++ .../src/catalog/sensitive-data-suggester.ts | 13 +- .../sensitive-data-suggestion-runner.ts | 110 +++ backend/src/catalog/types.ts | 73 ++ backend/src/routes/catalog-databases.ts | 16 + .../routes/catalog-description-generation.ts | 104 +- backend/test/catalog-databases-routes.test.ts | 170 +++- ...alog-description-generation-routes.test.ts | 103 +- ...description-generation.integration.test.ts | 2 + .../catalog-repository.integration.test.ts | 88 +- .../e2e/database-management-layout.spec.ts | 563 +++++++++++ frontend/src/api/catalog-databases.test.ts | 60 ++ frontend/src/api/catalog-databases.ts | 61 ++ .../catalog-description-generation.test.ts | 51 + frontend/src/api/client.test.ts | 3 + frontend/src/api/client.ts | 6 + .../AppShell.database-management.test.tsx | 88 +- .../src/shell/AppShell.new-session.test.tsx | 5 + .../src/shell/AppShell.session-mgmt.test.tsx | 11 +- frontend/src/shell/AppShell.tsx | 108 ++- .../src/shell/DatabaseManagementPage.test.tsx | 320 +++++- frontend/src/shell/DatabaseManagementPage.tsx | 521 ++++++++-- frontend/src/shell/PiManagement.test.tsx | 8 +- frontend/src/shell/PiManagement.tsx | 28 +- frontend/src/shell/WorkAreaPanel.css | 138 +++ frontend/src/shell/WorkAreaPanel.test.tsx | 67 ++ frontend/src/shell/WorkAreaPanel.tsx | 239 +++++ frontend/src/shell/WorkspaceManager.css | 27 + frontend/src/shell/WorkspaceManager.test.tsx | 17 +- frontend/src/shell/WorkspaceManager.tsx | 50 +- .../database-management/CatalogSyncDrawer.tsx | 140 ++- .../database-management/DatabaseColumns.tsx | 308 ++++-- .../DatabaseFleetQueryErrors.test.tsx | 161 ++++ .../database-management/DatabaseForm.tsx | 54 +- .../database-management/DatabaseGrid.tsx | 216 ++++- .../DatabaseRelationships.tsx | 158 ++- .../database-management/DatabaseTables.tsx | 497 ++++++++-- .../DescriptionGenerationDrawer.test.tsx | 10 +- .../DescriptionGenerationDrawer.tsx | 145 +-- .../FleetActionSelector.test.tsx | 57 ++ .../FleetActionSelector.tsx | 144 +++ .../database-management/FleetLedgerShell.css | 907 ++++++++++++++++++ .../FleetLedgerShell.test.tsx | 70 ++ .../database-management/FleetLedgerShell.tsx | 369 +++++++ .../MetadataGenerationModelSelector.tsx | 26 +- .../SensitiveDataReviewDrawer.tsx | 61 +- ...sitiveDataSuggestionHistoryDrawer.test.tsx | 132 +++ .../SensitiveDataSuggestionHistoryDrawer.tsx | 234 +++++ .../database-management/useModelessDrawer.ts | 1 + frontend/src/test/msw-contract.test.ts | 6 + frontend/src/test/msw.ts | 17 + 56 files changed, 6692 insertions(+), 611 deletions(-) create mode 100644 backend/src/catalog/metrics.ts create mode 100644 backend/src/catalog/migrations/007_sensitive_data_suggestion_runs.ts create mode 100644 backend/src/catalog/sensitive-data-suggestion-runner.ts create mode 100644 frontend/e2e/database-management-layout.spec.ts create mode 100644 frontend/src/api/catalog-databases.test.ts create mode 100644 frontend/src/shell/WorkAreaPanel.css create mode 100644 frontend/src/shell/WorkAreaPanel.test.tsx create mode 100644 frontend/src/shell/WorkAreaPanel.tsx create mode 100644 frontend/src/shell/WorkspaceManager.css create mode 100644 frontend/src/shell/database-management/DatabaseFleetQueryErrors.test.tsx create mode 100644 frontend/src/shell/database-management/FleetActionSelector.test.tsx create mode 100644 frontend/src/shell/database-management/FleetActionSelector.tsx create mode 100644 frontend/src/shell/database-management/FleetLedgerShell.css create mode 100644 frontend/src/shell/database-management/FleetLedgerShell.test.tsx create mode 100644 frontend/src/shell/database-management/FleetLedgerShell.tsx create mode 100644 frontend/src/shell/database-management/SensitiveDataSuggestionHistoryDrawer.test.tsx create mode 100644 frontend/src/shell/database-management/SensitiveDataSuggestionHistoryDrawer.tsx create mode 100644 frontend/src/shell/database-management/useModelessDrawer.ts diff --git a/backend/src/app.ts b/backend/src/app.ts index d3d56c0c..1b8dc64a 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -57,6 +57,7 @@ import { catalogDescriptionConsolidationRoutes } from "./routes/catalog-descript import { PythonModelCompleter, type ModelCompleter } from "./catalog/model-completer.js"; import { DescriptionGenerationWorker } from "./catalog/description-generation-worker.js"; import { SensitiveDataSuggester } from "./catalog/sensitive-data-suggester.js"; +import { SensitiveDataSuggestionRunner } from "./catalog/sensitive-data-suggestion-runner.js"; import { PostgresDescriptionSourceSampler, type DescriptionSourceSampler, @@ -185,6 +186,10 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc metadataGenerationModels, modelCompleter, ); + const sensitiveDataSuggestionRunner = new SensitiveDataSuggestionRunner( + catalogRepository, + sensitiveDataSuggester, + ); const catalogService = deps?.catalogService ?? new CatalogService( catalogRepository, workspaceRegistry, @@ -207,6 +212,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc ); app.addHook("onReady", async () => { await catalogSyncWorker.initialize(); }); app.addHook("onReady", async () => { await descriptionGenerationWorker.initialize(); }); + app.addHook("onReady", async () => { await sensitiveDataSuggestionRunner.initialize(); }); if (!deps?.catalogRepository && catalogRepository.close) { app.addHook("onClose", async () => { await catalogRepository.close?.(); }); } @@ -444,7 +450,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc catalogDescriptionGenerationRoutes(app, { repository: catalogRepository, worker: descriptionGenerationWorker, - sensitiveDataSuggester, + sensitiveDataSuggestionRunner, }); settingsRoutes(app, { cfg: config, listModels, getSettings }); piManagementRoutes(app, { service: piManagement }); diff --git a/backend/src/catalog/memory-repository.ts b/backend/src/catalog/memory-repository.ts index aed47bd6..40a14488 100644 --- a/backend/src/catalog/memory-repository.ts +++ b/backend/src/catalog/memory-repository.ts @@ -1,4 +1,9 @@ import { randomUUID } from "node:crypto"; +import { + createCatalogMetrics, + hasCatalogDescription, + latestCatalogTimestamp, +} from "./metrics.js"; import { CatalogConflictError, CatalogConnectorError, @@ -8,6 +13,7 @@ import { type CatalogDescriptionTarget, type CatalogDatabaseMetadataDeleteTarget, type CatalogMetadataDeleteCounts, + type CatalogMetrics, type CatalogRelationship, type CatalogSchemaDiff, type CatalogSyncCounts, @@ -26,6 +32,10 @@ import { type DescriptionGenerationRun, type DescriptionGenerationRunUpdate, type DescriptionGenerationScope, + type SensitiveDataSuggestionEvent, + type SensitiveDataSuggestionRun, + type SensitiveDataSuggestionRunUpdate, + type SensitiveDataSuggestionScope, type TableSyncRepositoryResult, type WorkspaceDatabase, } from "./types.js"; @@ -42,6 +52,8 @@ export class MemoryCatalogRepository implements CatalogRepository { private readonly relationships = new Map(); private readonly descriptionGenerationRuns = new Map(); private readonly descriptionGenerationEvents = new Map(); + private readonly sensitiveDataSuggestionRuns = new Map(); + private readonly sensitiveDataSuggestionEvents = new Map(); private readonly syncRuns = new Map(); private readonly syncEvents = new Map(); @@ -56,6 +68,34 @@ export class MemoryCatalogRepository implements CatalogRepository { const value = [...this.records.values()].find((record) => record.workspaceId === workspaceId); return value ? clone(value) : undefined; } + async getCatalogMetrics(databaseId?: string): Promise { + if (databaseId !== undefined && !this.records.has(databaseId)) return undefined; + + const selectedDatabaseIds = new Set( + databaseId === undefined ? this.records.keys() : [databaseId], + ); + const tables = [...this.tables.values()] + .filter((table) => selectedDatabaseIds.has(table.databaseId)); + const tableIds = new Set(tables.map((table) => table.id)); + const columns = [...this.columns.values()] + .filter((column) => tableIds.has(column.tableId)); + const relationships = [...this.relationships.values()] + .filter((relationship) => selectedDatabaseIds.has(relationship.databaseId)); + + return createCatalogMetrics(databaseId, { + tables: tables.length, + columns: columns.length, + sensitiveColumns: columns.filter((column) => column.sensitive).length, + relationships: relationships.length, + describedTables: tables.filter(hasCatalogDescription).length, + describedColumns: columns.filter(hasCatalogDescription).length, + }, latestCatalogTimestamp([ + ...[...selectedDatabaseIds].map((id) => this.records.get(id)?.schemaSyncedAt), + ...tables.map((table) => table.updatedAt), + ...columns.map((column) => column.updatedAt), + ...relationships.map((relationship) => relationship.updatedAt), + ])); + } async create(input: DatabaseConfigurationInput): Promise { if ([...this.records.values()].some((record) => record.workspaceId === input.workspaceId)) { throw new CatalogConflictError("Workspace database already exists"); @@ -136,6 +176,11 @@ export class MemoryCatalogRepository implements CatalogRepository { this.descriptionGenerationRuns.delete(runId); this.descriptionGenerationEvents.delete(runId); } + for (const [runId, run] of this.sensitiveDataSuggestionRuns) { + if (run.databaseId !== id) continue; + this.sensitiveDataSuggestionRuns.delete(runId); + this.sensitiveDataSuggestionEvents.delete(runId); + } return this.records.delete(id); } async listTables(databaseId: string): Promise { @@ -378,6 +423,110 @@ export class MemoryCatalogRepository implements CatalogRepository { .map((event) => structuredClone(event)); } + async createSensitiveDataSuggestionRun( + databaseId: string, + scope: SensitiveDataSuggestionScope, + modelId: string, + ): Promise { + const now = new Date().toISOString(); + const run: SensitiveDataSuggestionRun = { + id: randomUUID(), + databaseId, + scope, + modelId, + status: "running", + total: 0, + suggestedSensitive: 0, + suggestedNonSensitive: 0, + createdAt: now, + startedAt: now, + updatedAt: now, + finishedAt: null, + errorSummary: null, + }; + this.sensitiveDataSuggestionRuns.set(run.id, run); + return structuredClone(run); + } + + async getSensitiveDataSuggestionRun( + runId: string, + ): Promise { + const run = this.sensitiveDataSuggestionRuns.get(runId); + return run ? structuredClone(run) : undefined; + } + + async listSensitiveDataSuggestionRuns(limit = 50): Promise { + return [...this.sensitiveDataSuggestionRuns.values()] + .sort((a, b) => b.createdAt.localeCompare(a.createdAt) || b.id.localeCompare(a.id)) + .slice(0, limit) + .map((run) => structuredClone(run)); + } + + async interruptActiveSensitiveDataSuggestionRuns( + errorSummary: string, + ): Promise { + const interrupted: SensitiveDataSuggestionRun[] = []; + for (const run of this.sensitiveDataSuggestionRuns.values()) { + if (run.status !== "running") continue; + const now = new Date().toISOString(); + const updated: SensitiveDataSuggestionRun = { + ...run, + status: "interrupted", + updatedAt: now, + finishedAt: now, + errorSummary, + }; + this.sensitiveDataSuggestionRuns.set(run.id, updated); + interrupted.push(structuredClone(updated)); + } + return interrupted; + } + + async updateSensitiveDataSuggestionRun( + runId: string, + update: SensitiveDataSuggestionRunUpdate, + ): Promise { + const current = this.sensitiveDataSuggestionRuns.get(runId); + if (!current) return undefined; + const updated = { + ...current, + ...structuredClone(update), + updatedAt: new Date().toISOString(), + }; + this.sensitiveDataSuggestionRuns.set(runId, updated); + return structuredClone(updated); + } + + async appendSensitiveDataSuggestionEvent( + runId: string, + level: SensitiveDataSuggestionEvent["level"], + message: string, + ): Promise { + if (!this.sensitiveDataSuggestionRuns.has(runId)) { + throw new CatalogConflictError("Sensitive Data Suggestion Run does not exist"); + } + const events = this.sensitiveDataSuggestionEvents.get(runId) ?? []; + const event: SensitiveDataSuggestionEvent = { + runId, + sequence: events.length + 1, + level, + message, + createdAt: new Date().toISOString(), + }; + events.push(event); + this.sensitiveDataSuggestionEvents.set(runId, events); + return structuredClone(event); + } + + async listSensitiveDataSuggestionEvents( + runId: string, + afterSequence = 0, + ): Promise { + return (this.sensitiveDataSuggestionEvents.get(runId) ?? []) + .filter((event) => event.sequence > afterSequence) + .map((event) => structuredClone(event)); + } + async listRelationships(databaseId: string): Promise { return [...this.relationships.values()].filter((relationship) => relationship.databaseId === databaseId) .sort((a, b) => `${a.sourceTableName}.${a.constraintName}`.localeCompare(`${b.sourceTableName}.${b.constraintName}`)) diff --git a/backend/src/catalog/metrics.ts b/backend/src/catalog/metrics.ts new file mode 100644 index 00000000..10dccc6e --- /dev/null +++ b/backend/src/catalog/metrics.ts @@ -0,0 +1,53 @@ +import type { CatalogMetrics } from "./types.js"; + +export interface CatalogMetricCounts { + tables: number; + columns: number; + sensitiveColumns: number; + relationships: number; + describedTables: number; + describedColumns: number; +} + +export function hasCatalogDescription(target: { + description: string | null; + generatedDescription: string | null; +}): boolean { + return Boolean(target.description?.trim() || target.generatedDescription?.trim()); +} + +export function latestCatalogTimestamp( + values: readonly (string | null | undefined)[], +): string | null { + let latest: number | undefined; + for (const value of values) { + if (!value) continue; + const timestamp = Date.parse(value); + if (!Number.isFinite(timestamp)) continue; + latest = latest === undefined ? timestamp : Math.max(latest, timestamp); + } + return latest === undefined ? null : new Date(latest).toISOString(); +} + +export function createCatalogMetrics( + databaseId: string | undefined, + counts: CatalogMetricCounts, + updatedAt: string | null, +): CatalogMetrics { + const descriptionTargets = counts.tables + counts.columns; + const describedTargets = counts.describedTables + counts.describedColumns; + return { + scope: databaseId === undefined ? "global" : "database", + databaseId: databaseId ?? null, + tables: counts.tables, + columns: counts.columns, + sensitiveColumns: counts.sensitiveColumns, + relationships: counts.relationships, + descriptionTargets, + describedTargets, + descriptionCoverage: descriptionTargets === 0 + ? 0 + : Math.round((describedTargets / descriptionTargets) * 100), + updatedAt, + }; +} diff --git a/backend/src/catalog/migrate.ts b/backend/src/catalog/migrate.ts index b4467c5d..e6a0d179 100644 --- a/backend/src/catalog/migrate.ts +++ b/backend/src/catalog/migrate.ts @@ -9,6 +9,7 @@ import * as catalogSchemaSyncMigration from "./migrations/003_catalog_schema_syn import * as catalogRuntimeSequencePrivilegesMigration from "./migrations/004_catalog_runtime_sequence_privileges.js"; import * as descriptionGenerationRunsMigration from "./migrations/005_description_generation_runs.js"; import * as sensitiveDataFlagMigration from "./migrations/006_sensitive_data_flag.js"; +import * as sensitiveDataSuggestionRunsMigration from "./migrations/007_sensitive_data_suggestion_runs.js"; const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL; const host = process.env.THT_CATALOG_DB_HOST; @@ -40,6 +41,7 @@ const provider: MigrationProvider = { "004_catalog_runtime_sequence_privileges": catalogRuntimeSequencePrivilegesMigration, "005_description_generation_runs": descriptionGenerationRunsMigration, "006_sensitive_data_flag": sensitiveDataFlagMigration, + "007_sensitive_data_suggestion_runs": sensitiveDataSuggestionRunsMigration, }; }, }; diff --git a/backend/src/catalog/migrations/007_sensitive_data_suggestion_runs.ts b/backend/src/catalog/migrations/007_sensitive_data_suggestion_runs.ts new file mode 100644 index 00000000..939f3868 --- /dev/null +++ b/backend/src/catalog/migrations/007_sensitive_data_suggestion_runs.ts @@ -0,0 +1,73 @@ +import { type Kysely, sql } from "kysely"; +import type { CatalogDatabase } from "../repository.js"; + +export async function up(db: Kysely): Promise { + await db.schema.createTable("sensitive_data_suggestion_runs") + .addColumn("id", "uuid", (column) => column.primaryKey()) + .addColumn("database_id", "uuid", (column) => column.notNull() + .references("workspace_databases.id").onDelete("cascade")) + .addColumn("scope", "text", (column) => column.notNull()) + .addColumn("model_id", "text", (column) => column.notNull()) + .addColumn("status", "text", (column) => column.notNull()) + .addColumn("total", "integer", (column) => column.notNull().defaultTo(0)) + .addColumn("suggested_sensitive", "integer", (column) => column.notNull().defaultTo(0)) + .addColumn("suggested_non_sensitive", "integer", (column) => column.notNull().defaultTo(0)) + .addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`)) + .addColumn("started_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`)) + .addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`)) + .addColumn("finished_at", "timestamptz") + .addColumn("error_summary", "text") + .addCheckConstraint( + "sensitive_data_suggestion_runs_scope_check", + sql`scope in ('selected_columns', 'selected_tables', 'all')`, + ) + .addCheckConstraint( + "sensitive_data_suggestion_runs_model_id_check", + sql`model_id ~ '^[a-z][a-z0-9._-]{0,63}$'`, + ) + .addCheckConstraint( + "sensitive_data_suggestion_runs_status_check", + sql`status in ('running', 'completed', 'failed', 'interrupted')`, + ) + .addCheckConstraint( + "sensitive_data_suggestion_runs_counters_check", + sql`total >= 0 + and suggested_sensitive >= 0 + and suggested_non_sensitive >= 0 + and suggested_sensitive + suggested_non_sensitive <= total`, + ) + .addCheckConstraint( + "sensitive_data_suggestion_runs_error_summary_check", + sql`error_summary is null or char_length(error_summary) between 1 and 2000`, + ) + .execute(); + + await db.schema.createIndex("sensitive_data_suggestion_runs_database_created_idx") + .on("sensitive_data_suggestion_runs") + .columns(["database_id", "created_at"]) + .execute(); + + await db.schema.createTable("sensitive_data_suggestion_events") + .addColumn("run_id", "uuid", (column) => column.notNull() + .references("sensitive_data_suggestion_runs.id").onDelete("cascade")) + .addColumn("sequence", "integer", (column) => column.notNull()) + .addColumn("level", "text", (column) => column.notNull()) + .addColumn("message", "text", (column) => column.notNull()) + .addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`)) + .addPrimaryKeyConstraint("sensitive_data_suggestion_events_pkey", ["run_id", "sequence"]) + .addCheckConstraint("sensitive_data_suggestion_events_sequence_check", sql`sequence > 0`) + .addCheckConstraint( + "sensitive_data_suggestion_events_level_check", + sql`level in ('info', 'warning', 'error')`, + ) + .addCheckConstraint( + "sensitive_data_suggestion_events_message_check", + sql`char_length(message) between 1 and 2000`, + ) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable("sensitive_data_suggestion_events").execute(); + await db.schema.dropTable("sensitive_data_suggestion_runs").execute(); +} diff --git a/backend/src/catalog/repository.ts b/backend/src/catalog/repository.ts index 5aa41b0d..d48063ca 100644 --- a/backend/src/catalog/repository.ts +++ b/backend/src/catalog/repository.ts @@ -11,6 +11,7 @@ import { type Transaction, } from "kysely"; import { Pool } from "pg"; +import { createCatalogMetrics } from "./metrics.js"; import { CatalogConflictError, CatalogConnectorError, @@ -21,6 +22,7 @@ import { type CatalogDescriptionTarget, type CatalogDatabaseMetadataDeleteTarget, type CatalogMetadataDeleteCounts, + type CatalogMetrics, type CatalogRelationship, type CatalogSchemaDiff, type CatalogSyncCounts, @@ -40,6 +42,10 @@ import { type DescriptionGenerationScope, type ObservedCatalogTable, type ObservedSchemaSnapshot, + type SensitiveDataSuggestionEvent, + type SensitiveDataSuggestionRun, + type SensitiveDataSuggestionRunUpdate, + type SensitiveDataSuggestionScope, type TableSyncRepositoryResult, type WorkspaceDatabase, } from "./types.js"; @@ -166,6 +172,30 @@ interface DescriptionGenerationEventTable { createdAt: Timestamp; } +interface SensitiveDataSuggestionRunTable { + id: string; + databaseId: string; + scope: SensitiveDataSuggestionScope; + modelId: string; + status: SensitiveDataSuggestionRun["status"]; + total: number; + suggestedSensitive: number; + suggestedNonSensitive: number; + createdAt: Timestamp; + startedAt: Timestamp; + updatedAt: Timestamp; + finishedAt: Timestamp | null; + errorSummary: string | null; +} + +interface SensitiveDataSuggestionEventTable { + runId: string; + sequence: number; + level: SensitiveDataSuggestionEvent["level"]; + message: string; + createdAt: Timestamp; +} + interface CatalogSyncRunTable { id: string; databaseId: string; @@ -213,6 +243,8 @@ export interface CatalogDatabase { catalogRelationshipColumns: CatalogRelationshipColumnTable; descriptionGenerationRuns: DescriptionGenerationRunTable; descriptionGenerationEvents: DescriptionGenerationEventTable; + sensitiveDataSuggestionRuns: SensitiveDataSuggestionRunTable; + sensitiveDataSuggestionEvents: SensitiveDataSuggestionEventTable; catalogSyncRuns: CatalogSyncRunTable; catalogSyncEvents: CatalogSyncEventTable; } @@ -220,6 +252,17 @@ export interface CatalogDatabase { type DbOrTransaction = Kysely | Transaction; type JoinedRow = Selectable & Selectable; +interface CatalogMetricsRow { + databaseCount: number; + tables: number; + columns: number; + sensitiveColumns: number; + relationships: number; + describedTables: number; + describedColumns: number; + updatedAt: Date | string | null; +} + function present(value: T | null): T | undefined { return value === null ? undefined : value; } @@ -338,6 +381,25 @@ function serializeDescriptionGenerationEvent( return { ...row, createdAt: new Date(row.createdAt).toISOString() }; } +function serializeSensitiveDataSuggestionRun( + row: Selectable, +): SensitiveDataSuggestionRun { + const stamp = (value: Date | string | null) => value === null ? null : new Date(value).toISOString(); + return { + ...row, + createdAt: new Date(row.createdAt).toISOString(), + startedAt: new Date(row.startedAt).toISOString(), + updatedAt: new Date(row.updatedAt).toISOString(), + finishedAt: stamp(row.finishedAt), + }; +} + +function serializeSensitiveDataSuggestionEvent( + row: Selectable, +): SensitiveDataSuggestionEvent { + return { ...row, createdAt: new Date(row.createdAt).toISOString() }; +} + function bindingValues(databaseId: string, binding: DatabaseBinding) { return { databaseId, @@ -388,6 +450,83 @@ export class KyselyCatalogRepository implements CatalogRepository { return id ? await this.get(id.id) : undefined; } + async getCatalogMetrics(databaseId?: string): Promise { + const result = await sql` + WITH requested_database AS ( + SELECT ${databaseId ?? null}::uuid AS id + ), + selected_databases AS ( + SELECT workspace_databases.id, workspace_databases.schema_synced_at + FROM workspace_databases + CROSS JOIN requested_database + WHERE requested_database.id IS NULL + OR workspace_databases.id = requested_database.id + ), + table_metrics AS ( + SELECT + count(*)::int AS tables, + count(*) FILTER ( + WHERE nullif(btrim(catalog_tables.description), '') IS NOT NULL + OR nullif(btrim(catalog_tables.generated_description), '') IS NOT NULL + )::int AS "describedTables", + max(catalog_tables.updated_at) AS updated_at + FROM catalog_tables + INNER JOIN selected_databases + ON selected_databases.id = catalog_tables.database_id + ), + column_metrics AS ( + SELECT + count(*)::int AS columns, + count(*) FILTER (WHERE catalog_columns.sensitive)::int AS "sensitiveColumns", + count(*) FILTER ( + WHERE nullif(btrim(catalog_columns.description), '') IS NOT NULL + OR nullif(btrim(catalog_columns.generated_description), '') IS NOT NULL + )::int AS "describedColumns", + max(catalog_columns.updated_at) AS updated_at + FROM catalog_columns + INNER JOIN catalog_tables ON catalog_tables.id = catalog_columns.table_id + INNER JOIN selected_databases + ON selected_databases.id = catalog_tables.database_id + ), + relationship_metrics AS ( + SELECT + count(*)::int AS relationships, + max(catalog_relationships.updated_at) AS updated_at + FROM catalog_relationships + INNER JOIN selected_databases + ON selected_databases.id = catalog_relationships.database_id + ) + SELECT + (SELECT count(*)::int FROM selected_databases) AS "databaseCount", + table_metrics.tables, + column_metrics.columns, + column_metrics."sensitiveColumns", + relationship_metrics.relationships, + table_metrics."describedTables", + column_metrics."describedColumns", + greatest( + (SELECT max(schema_synced_at) FROM selected_databases), + table_metrics.updated_at, + column_metrics.updated_at, + relationship_metrics.updated_at + ) AS "updatedAt" + FROM table_metrics + CROSS JOIN column_metrics + CROSS JOIN relationship_metrics + `.execute(this.db); + const row = result.rows[0]; + if (!row || (databaseId !== undefined && Number(row.databaseCount) === 0)) return undefined; + + return createCatalogMetrics(databaseId, { + tables: Number(row.tables), + columns: Number(row.columns), + sensitiveColumns: Number(row.sensitiveColumns), + relationships: Number(row.relationships), + describedTables: Number(row.describedTables), + describedColumns: Number(row.describedColumns), + }, row.updatedAt === null ? null : new Date(row.updatedAt).toISOString()); + } + async create(input: DatabaseConfigurationInput): Promise { try { return await this.db.transaction().execute(async (trx) => { @@ -775,6 +914,114 @@ export class KyselyCatalogRepository implements CatalogRepository { return rows.map(serializeDescriptionGenerationEvent); } + async createSensitiveDataSuggestionRun( + databaseId: string, + scope: SensitiveDataSuggestionScope, + modelId: string, + ): Promise { + const row = await this.db.insertInto("sensitiveDataSuggestionRuns").values({ + id: randomUUID(), + databaseId, + scope, + modelId, + status: "running", + total: 0, + suggestedSensitive: 0, + suggestedNonSensitive: 0, + finishedAt: null, + errorSummary: null, + }).returningAll().executeTakeFirstOrThrow(); + return serializeSensitiveDataSuggestionRun(row); + } + + async getSensitiveDataSuggestionRun( + runId: string, + ): Promise { + const row = await this.db.selectFrom("sensitiveDataSuggestionRuns") + .selectAll() + .where("id", "=", runId) + .executeTakeFirst(); + return row ? serializeSensitiveDataSuggestionRun(row) : undefined; + } + + async listSensitiveDataSuggestionRuns(limit = 50): Promise { + const rows = await this.db.selectFrom("sensitiveDataSuggestionRuns") + .selectAll() + .orderBy("createdAt", "desc") + .orderBy("id", "desc") + .limit(limit) + .execute(); + return rows.map(serializeSensitiveDataSuggestionRun); + } + + async interruptActiveSensitiveDataSuggestionRuns( + errorSummary: string, + ): Promise { + const rows = await this.db.updateTable("sensitiveDataSuggestionRuns") + .set({ + status: "interrupted", + finishedAt: sql`now()`, + updatedAt: sql`now()`, + errorSummary, + }) + .where("status", "=", "running") + .returningAll() + .execute(); + return rows.map(serializeSensitiveDataSuggestionRun); + } + + async updateSensitiveDataSuggestionRun( + runId: string, + update: SensitiveDataSuggestionRunUpdate, + ): Promise { + const values: any = { ...update, updatedAt: sql`now()` }; + const row = await this.db.updateTable("sensitiveDataSuggestionRuns") + .set(values) + .where("id", "=", runId) + .returningAll() + .executeTakeFirst(); + return row ? serializeSensitiveDataSuggestionRun(row) : undefined; + } + + async appendSensitiveDataSuggestionEvent( + runId: string, + level: SensitiveDataSuggestionEvent["level"], + message: string, + ): Promise { + return await this.db.transaction().execute(async (trx) => { + const run = await trx.selectFrom("sensitiveDataSuggestionRuns") + .select("id") + .where("id", "=", runId) + .forUpdate() + .executeTakeFirst(); + if (!run) throw new CatalogConflictError("Sensitive Data Suggestion Run does not exist"); + const current = await trx.selectFrom("sensitiveDataSuggestionEvents") + .select(sql`coalesce(max(sequence), 0)::int`.as("sequence")) + .where("runId", "=", runId) + .executeTakeFirst(); + const row = await trx.insertInto("sensitiveDataSuggestionEvents").values({ + runId, + sequence: Number(current?.sequence ?? 0) + 1, + level, + message, + }).returningAll().executeTakeFirstOrThrow(); + return serializeSensitiveDataSuggestionEvent(row); + }); + } + + async listSensitiveDataSuggestionEvents( + runId: string, + afterSequence = 0, + ): Promise { + const rows = await this.db.selectFrom("sensitiveDataSuggestionEvents") + .selectAll() + .where("runId", "=", runId) + .where("sequence", ">", afterSequence) + .orderBy("sequence") + .execute(); + return rows.map(serializeSensitiveDataSuggestionEvent); + } + async listRelationships(databaseId: string): Promise { const rows = await this.db.selectFrom("catalogRelationships as relationship") .innerJoin("catalogTables as sourceTable", "sourceTable.id", "relationship.sourceTableId") @@ -1345,6 +1592,7 @@ export class UnavailableCatalogRepository implements CatalogRepository { async list(): Promise { return this.fail(); } async get(): Promise { return this.fail(); } async getByWorkspace(): Promise { return this.fail(); } + async getCatalogMetrics(): Promise { return this.fail(); } async create(): Promise { return this.fail(); } async update(): Promise { return this.fail(); } async recordTest(): Promise { return this.fail(); } @@ -1366,6 +1614,13 @@ export class UnavailableCatalogRepository implements CatalogRepository { async updateDescriptionGenerationRun(): Promise { return this.fail(); } async appendDescriptionGenerationEvent(): Promise { return this.fail(); } async listDescriptionGenerationEvents(): Promise { return this.fail(); } + async createSensitiveDataSuggestionRun(): Promise { return this.fail(); } + async getSensitiveDataSuggestionRun(): Promise { return this.fail(); } + async listSensitiveDataSuggestionRuns(): Promise { return this.fail(); } + async interruptActiveSensitiveDataSuggestionRuns(): Promise { return this.fail(); } + async updateSensitiveDataSuggestionRun(): Promise { return this.fail(); } + async appendSensitiveDataSuggestionEvent(): Promise { return this.fail(); } + async listSensitiveDataSuggestionEvents(): Promise { return this.fail(); } async listRelationships(): Promise { return this.fail(); } async deleteDatabaseMetadata(): Promise { return this.fail(); } async deleteTableMetadata(): Promise { return this.fail(); } diff --git a/backend/src/catalog/sensitive-data-suggester.ts b/backend/src/catalog/sensitive-data-suggester.ts index d6eb17a9..f66fd0ed 100644 --- a/backend/src/catalog/sensitive-data-suggester.ts +++ b/backend/src/catalog/sensitive-data-suggester.ts @@ -1,7 +1,14 @@ import { z } from "zod"; import type { MetadataGenerationModels } from "./metadata-generation-models.js"; import type { ModelCompleter, ModelCompletionMessage } from "./model-completer.js"; -import type { CatalogColumn, CatalogRepository, CatalogTable } from "./types.js"; +import type { + CatalogColumn, + CatalogRepository, + CatalogTable, + SensitiveDataSuggestionScope, +} from "./types.js"; + +export type { SensitiveDataSuggestionScope } from "./types.js"; // The helper accepts at most 64 KiB per message. Keep the same safety margin used by // Description Generation so UTF-8 structural metadata never reaches that hard limit. @@ -16,8 +23,6 @@ const responseSchema = z.object({ }).strict()), }).strict(); -export type SensitiveDataSuggestionScope = "all" | "selected_tables" | "selected_columns"; - interface StructuralColumn { columnId: string; tableId: string; @@ -195,10 +200,12 @@ export class SensitiveDataSuggester { scope: SensitiveDataSuggestionScope, targetIds: readonly string[], signal: AbortSignal, + onPrepared?: (total: number) => void | Promise, ): Promise { const database = await this.repository.get(databaseId); if (!database) throw new SensitiveDataSuggestionTargetNotFoundError("database"); const columns = await this.selectColumns(databaseId, scope, targetIds); + await onPrepared?.(columns.length); const model = this.models.resolve(modelId); const suggestions: SensitiveDataSuggestion[] = []; diff --git a/backend/src/catalog/sensitive-data-suggestion-runner.ts b/backend/src/catalog/sensitive-data-suggestion-runner.ts new file mode 100644 index 00000000..c337f63c --- /dev/null +++ b/backend/src/catalog/sensitive-data-suggestion-runner.ts @@ -0,0 +1,110 @@ +import type { + SensitiveDataSuggestion, +} from "./sensitive-data-suggester.js"; +import { + SensitiveDataSuggester, + SensitiveDataSuggestionTargetNotFoundError, +} from "./sensitive-data-suggester.js"; +import type { + CatalogRepository, + SensitiveDataSuggestionRun, + SensitiveDataSuggestionScope, +} from "./types.js"; + +const interruptedMessage = "Sensitive-field suggestion generation was interrupted by backend restart."; +const failedMessage = "Sensitive-field suggestion generation failed."; + +export interface SensitiveDataSuggestionRunResult { + suggestions: readonly SensitiveDataSuggestion[]; + run: SensitiveDataSuggestionRun; +} + +export class SensitiveDataSuggestionRunner { + constructor( + private readonly repository: CatalogRepository, + private readonly suggester: SensitiveDataSuggester, + ) {} + + async initialize(): Promise { + if (!(await this.repository.available())) return; + const interrupted = await this.repository.interruptActiveSensitiveDataSuggestionRuns( + interruptedMessage, + ); + for (const run of interrupted) { + await this.repository.appendSensitiveDataSuggestionEvent( + run.id, + "warning", + interruptedMessage, + ); + } + } + + async run( + databaseId: string, + modelId: string, + scope: SensitiveDataSuggestionScope, + targetIds: readonly string[], + signal: AbortSignal, + ): Promise { + if (!(await this.repository.get(databaseId))) { + throw new SensitiveDataSuggestionTargetNotFoundError("database"); + } + const started = await this.repository.createSensitiveDataSuggestionRun( + databaseId, + scope, + modelId, + ); + + try { + await this.repository.appendSensitiveDataSuggestionEvent( + started.id, + "info", + "Sensitive-field suggestion generation started.", + ); + const suggestions = await this.suggester.suggest( + databaseId, + modelId, + scope, + targetIds, + signal, + async (total) => { + const prepared = await this.repository.updateSensitiveDataSuggestionRun(started.id, { + total, + }); + if (!prepared) throw new Error("Sensitive Data Suggestion Run disappeared"); + }, + ); + const suggestedSensitive = suggestions.filter((suggestion) => suggestion.sensitive).length; + const suggestedNonSensitive = suggestions.length - suggestedSensitive; + await this.repository.appendSensitiveDataSuggestionEvent( + started.id, + "info", + `Sensitive-field suggestion generation completed for ${suggestions.length} column${ + suggestions.length === 1 ? "" : "s" + }.`, + ); + const completed = await this.repository.updateSensitiveDataSuggestionRun(started.id, { + status: "completed", + total: suggestions.length, + suggestedSensitive, + suggestedNonSensitive, + finishedAt: new Date().toISOString(), + errorSummary: null, + }); + if (!completed) throw new Error("Sensitive Data Suggestion Run disappeared"); + return { suggestions, run: completed }; + } catch (error) { + await this.repository.updateSensitiveDataSuggestionRun(started.id, { + status: "failed", + finishedAt: new Date().toISOString(), + errorSummary: failedMessage, + }).catch(() => undefined); + await this.repository.appendSensitiveDataSuggestionEvent( + started.id, + "error", + failedMessage, + ).catch(() => undefined); + throw error; + } + } +} diff --git a/backend/src/catalog/types.ts b/backend/src/catalog/types.ts index caa36446..809af9f6 100644 --- a/backend/src/catalog/types.ts +++ b/backend/src/catalog/types.ts @@ -38,6 +38,19 @@ export interface WorkspaceDatabase { schemaSyncedAt?: string; } +export interface CatalogMetrics { + scope: "global" | "database"; + databaseId: string | null; + tables: number; + columns: number; + sensitiveColumns: number; + relationships: number; + descriptionTargets: number; + describedTargets: number; + descriptionCoverage: number; + updatedAt: string | null; +} + export interface DatabaseConfigurationInput { workspaceId: string; engine: "postgres"; @@ -201,6 +214,42 @@ export interface DescriptionGenerationEvent { createdAt: string; } +export type SensitiveDataSuggestionScope = "all" | "selected_tables" | "selected_columns"; +export type SensitiveDataSuggestionStatus = "running" | "completed" | "failed" | "interrupted"; + +export interface SensitiveDataSuggestionRun { + id: string; + databaseId: string; + scope: SensitiveDataSuggestionScope; + modelId: string; + status: SensitiveDataSuggestionStatus; + total: number; + suggestedSensitive: number; + suggestedNonSensitive: number; + createdAt: string; + startedAt: string; + updatedAt: string; + finishedAt: string | null; + errorSummary: string | null; +} + +export interface SensitiveDataSuggestionRunUpdate { + status?: SensitiveDataSuggestionStatus; + total?: number; + suggestedSensitive?: number; + suggestedNonSensitive?: number; + finishedAt?: string | null; + errorSummary?: string | null; +} + +export interface SensitiveDataSuggestionEvent { + runId: string; + sequence: number; + level: "info" | "warning" | "error"; + message: string; + createdAt: string; +} + export interface ObservedRelationshipColumn { position: number; sourceColumnName: string; @@ -321,6 +370,7 @@ export interface CatalogRepository { list(): Promise; get(id: string): Promise; getByWorkspace(workspaceId: string): Promise; + getCatalogMetrics(databaseId?: string): Promise; create(input: DatabaseConfigurationInput): Promise; update(id: string, expectedVersion: number, input: DatabaseConfigurationInput): Promise; recordTest(id: string, expectedVersion: number, result: DatabaseTestResult): Promise; @@ -383,6 +433,29 @@ export interface CatalogRepository { runId: string, afterSequence?: number, ): Promise; + createSensitiveDataSuggestionRun( + databaseId: string, + scope: SensitiveDataSuggestionScope, + modelId: string, + ): Promise; + getSensitiveDataSuggestionRun(runId: string): Promise; + listSensitiveDataSuggestionRuns(limit?: number): Promise; + interruptActiveSensitiveDataSuggestionRuns( + errorSummary: string, + ): Promise; + updateSensitiveDataSuggestionRun( + runId: string, + update: SensitiveDataSuggestionRunUpdate, + ): Promise; + appendSensitiveDataSuggestionEvent( + runId: string, + level: SensitiveDataSuggestionEvent["level"], + message: string, + ): Promise; + listSensitiveDataSuggestionEvents( + runId: string, + afterSequence?: number, + ): Promise; listRelationships(databaseId: string): Promise; deleteDatabaseMetadata( databaseIds: readonly string[], diff --git a/backend/src/routes/catalog-databases.ts b/backend/src/routes/catalog-databases.ts index 080a1008..c0c8a443 100644 --- a/backend/src/routes/catalog-databases.ts +++ b/backend/src/routes/catalog-databases.ts @@ -66,6 +66,7 @@ const secretsSchema = z.object({ values: z.partialRecord(z.enum(secretNames), z.string().min(1).max(65_536)).refine((values) => Object.keys(values).length > 0), }).strict(); const versionQuery = z.object({ version: z.coerce.number().int().positive() }); +const metricsQuery = z.object({ databaseId: z.uuid().optional() }).strict(); function safeError(reply: FastifyReply, error: unknown) { if (error instanceof CatalogUnavailableError) { @@ -114,6 +115,21 @@ export function catalogDatabaseRoutes( return { available: await deps.repository.available() }; }); + app.get("/catalog/metrics", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const { databaseId } = metricsQuery.parse(request.query); + const metrics = await deps.repository.getCatalogMetrics(databaseId); + if (!metrics) { + return reply.code(404).send({ + code: "database_not_found", + message: "Database configuration was not found.", + }); + } + return metrics; + } catch (error) { return safeError(reply, error); } + }); + app.get("/catalog/databases", async (request, reply) => { if (!manage(request, reply)) return reply; try { diff --git a/backend/src/routes/catalog-description-generation.ts b/backend/src/routes/catalog-description-generation.ts index 4d3732a2..a79ce21d 100644 --- a/backend/src/routes/catalog-description-generation.ts +++ b/backend/src/routes/catalog-description-generation.ts @@ -13,13 +13,13 @@ import { import { MetadataGenerationModelUnavailableError } from "../catalog/metadata-generation-models.js"; import { ModelCompletionProviderError } from "../catalog/model-completer.js"; import { - SensitiveDataSuggester, SensitiveDataSuggestionDuplicateTargetIdsError, SensitiveDataSuggestionInvalidResponseError, SensitiveDataSuggestionNoEligibleColumnsError, SensitiveDataSuggestionPayloadTooLargeError, SensitiveDataSuggestionTargetNotFoundError, } from "../catalog/sensitive-data-suggester.js"; +import type { SensitiveDataSuggestionRunner } from "../catalog/sensitive-data-suggestion-runner.js"; import { CatalogOperationInProgressError, CatalogUnavailableError, @@ -27,6 +27,8 @@ import { type CatalogRepository, type DescriptionGenerationEvent, type DescriptionGenerationRun, + type SensitiveDataSuggestionEvent, + type SensitiveDataSuggestionRun, } from "../catalog/types.js"; const idSchema = z.uuid(); @@ -107,6 +109,34 @@ function publicRun(run: DescriptionGenerationRun) { }; } +function publicSensitiveDataSuggestionEvent(event: SensitiveDataSuggestionEvent) { + return { + runId: event.runId, + sequence: event.sequence, + level: event.level, + message: event.message, + createdAt: event.createdAt, + }; +} + +function publicSensitiveDataSuggestionRun(run: SensitiveDataSuggestionRun) { + return { + id: run.id, + databaseId: run.databaseId, + scope: run.scope, + modelId: run.modelId, + status: run.status, + total: run.total, + suggestedSensitive: run.suggestedSensitive, + suggestedNonSensitive: run.suggestedNonSensitive, + createdAt: run.createdAt, + startedAt: run.startedAt, + updatedAt: run.updatedAt, + finishedAt: run.finishedAt, + errorSummary: run.errorSummary, + }; +} + function safeError(reply: FastifyReply, error: unknown) { if (error instanceof CatalogUnavailableError) { return reply.code(503).send({ @@ -260,12 +290,31 @@ function safeSuggestionError(reply: FastifyReply, error: unknown) { }); } +function safeSuggestionHistoryError(reply: FastifyReply, error: unknown) { + if (error instanceof CatalogUnavailableError) { + return reply.code(503).send({ + code: "catalog_unavailable", + message: "Sensitive Data Suggestion history is unavailable because the database catalog is unavailable.", + }); + } + if (error instanceof z.ZodError) { + return reply.code(400).send({ + code: "sensitive_data_suggestion_history_request_invalid", + message: "Sensitive Data Suggestion history parameters are invalid.", + }); + } + return reply.code(500).send({ + code: "sensitive_data_suggestion_history_failed", + message: "Sensitive Data Suggestion history could not be loaded.", + }); +} + export function catalogDescriptionGenerationRoutes( app: FastifyInstance, deps: { repository: CatalogRepository; worker: DescriptionGenerationWorker; - sensitiveDataSuggester: SensitiveDataSuggester; + sensitiveDataSuggestionRunner: SensitiveDataSuggestionRunner; }, ): void { app.post("/catalog/databases/:databaseId/sensitive-data-suggestions", async (request, reply) => { @@ -273,19 +322,66 @@ export function catalogDescriptionGenerationRoutes( try { const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId); const input = suggestionSchema.parse(request.body); - const suggestions = await deps.sensitiveDataSuggester.suggest( + const result = await deps.sensitiveDataSuggestionRunner.run( databaseId, input.modelId, input.scope, "targetIds" in input ? input.targetIds : [], new AbortController().signal, ); - return { suggestions }; + return { + suggestions: result.suggestions, + run: publicSensitiveDataSuggestionRun(result.run), + }; } catch (error) { return safeSuggestionError(reply, error); } }); + app.get("/catalog/sensitive-data-suggestion-runs", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const { limit } = historyQuerySchema.parse(request.query); + return (await deps.repository.listSensitiveDataSuggestionRuns(limit)) + .map(publicSensitiveDataSuggestionRun); + } catch (error) { + return safeSuggestionHistoryError(reply, error); + } + }); + + app.get("/catalog/sensitive-data-suggestion-runs/:runId", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const runId = idSchema.parse((request.params as { runId?: unknown }).runId); + const run = await deps.repository.getSensitiveDataSuggestionRun(runId); + if (!run) return reply.code(404).send({ + code: "sensitive_data_suggestion_run_not_found", + message: "Sensitive Data Suggestion Run was not found.", + }); + return publicSensitiveDataSuggestionRun(run); + } catch (error) { + return safeSuggestionHistoryError(reply, error); + } + }); + + app.get("/catalog/sensitive-data-suggestion-runs/:runId/events-list", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const runId = idSchema.parse((request.params as { runId?: unknown }).runId); + const { after } = eventQuerySchema.parse(request.query); + if (!(await deps.repository.getSensitiveDataSuggestionRun(runId))) { + return reply.code(404).send({ + code: "sensitive_data_suggestion_run_not_found", + message: "Sensitive Data Suggestion Run was not found.", + }); + } + return (await deps.repository.listSensitiveDataSuggestionEvents(runId, after)) + .map(publicSensitiveDataSuggestionEvent); + } catch (error) { + return safeSuggestionHistoryError(reply, error); + } + }); + app.post("/catalog/databases/:databaseId/description-generation-runs", async (request, reply) => { if (!manage(request, reply)) return reply; try { diff --git a/backend/test/catalog-databases-routes.test.ts b/backend/test/catalog-databases-routes.test.ts index f3d7002c..051c511f 100644 --- a/backend/test/catalog-databases-routes.test.ts +++ b/backend/test/catalog-databases-routes.test.ts @@ -5,6 +5,7 @@ import { afterEach, expect, test, vi } from "vitest"; import { buildApp } from "../src/app.js"; import { loadConfig } from "../src/config.js"; import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js"; +import type { ObservedSchemaSnapshot } from "../src/catalog/types.js"; import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js"; import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/registry.js"; import type { WorkspaceDescriptor } from "../src/workspaces/schema.js"; @@ -27,7 +28,7 @@ const workspace: WorkspaceDescriptor = { }; const revision: WorkspaceRevision = { id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/tmp/psd.yaml" }; -function setup() { +function setup(environment: Record = {}) { const secretRoot = mkdtempSync(join(tmpdir(), "catalog-secret-")); const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-secret-runtime-")); roots.push(secretRoot, runtimeRoot); @@ -38,7 +39,11 @@ function setup() { listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]), read: vi.fn(async () => ({ workspace, revision })), } as unknown as WorkspaceRegistry; - const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test" }), { + const app = buildApp(loadConfig({ + THT_HARNESS_DIR: "/missing", + NODE_ENV: "test", + ...environment, + }), { thtRunner: {} as never, workspaceRegistry: registry, workspaceSecretStore: secretStore, @@ -56,6 +61,31 @@ const direct = { binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" }, }; +const fleetSnapshot: ObservedSchemaSnapshot = { + schemaVersion: 1, + capabilities: { tables: "available", columns: "available", relationships: "available" }, + tables: [ + { name: "patients", sourceComment: "Clinical patients" }, + { name: "visits", sourceComment: null }, + ], + columns: [ + { tableName: "patients", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null }, + { tableName: "patients", name: "name", ordinalPosition: 2, dataType: "text", isNullable: true, defaultExpression: null, primaryKeyPosition: null, sourceComment: null }, + { tableName: "visits", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null }, + { tableName: "visits", name: "patient_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null }, + ], + relationships: [{ + constraintName: "visits_patient_id_fkey", + sourceTableName: "visits", + targetTableName: "patients", + updateRule: "NO ACTION", + deleteRule: "CASCADE", + deferrable: false, + initiallyDeferred: false, + columns: [{ position: 1, sourceColumnName: "patient_id", targetColumnName: "id" }], + }], +}; + test("lists every YAML workspace and creates its one database configuration", async () => { const { app } = setup(); const initial = await app.inject({ method: "GET", url: "/catalog/databases" }); @@ -120,3 +150,139 @@ test("uses optimistic versions, keeps secrets write-only, and hard-deletes only expect(secretStore.has("psd-clinical", "catalog.dwh.password")).toBe(false); expect((await app.inject({ method: "GET", url: "/catalog/databases" })).json()).toMatchObject([{ configured: false }]); }); + +test("returns exact global and per-database fleet metrics", async () => { + const { app, repository } = setup(); + const database = await repository.create(direct); + await repository.applySchemaSync(database.id, database.version, "all", [], fleetSnapshot); + + const patients = (await repository.listTables(database.id)) + .find((table) => table.name === "patients")!; + await repository.updateTableDescription( + database.id, + patients.id, + patients.version, + "Curated patients", + ); + const patientName = (await repository.listColumns(database.id, patients.id)) + .find((column) => column.name === "name")!; + await repository.updateColumnMetadata( + database.id, + patients.id, + patientName.id, + patientName.version, + null, + "Generated patient name", + true, + ); + + const archive = await repository.create({ + workspaceId: "removed-workspace", + engine: "postgres", + databaseName: "archive", + schema: "public", + binding: { + transport: "postgres_direct", + host: "archive.internal", + port: 5432, + username: "reader", + }, + }); + await repository.applySchemaSync(archive.id, archive.version, "all", [], { + schemaVersion: 1, + capabilities: { tables: "available", columns: "available", relationships: "available" }, + tables: [{ name: "events", sourceComment: null }], + columns: [{ + tableName: "events", + name: "id", + ordinalPosition: 1, + dataType: "bigint", + isNullable: false, + defaultExpression: null, + primaryKeyPosition: 1, + sourceComment: null, + }], + relationships: [], + }); + const events = (await repository.listTables(archive.id))[0]!; + await repository.updateTableMetadata( + archive.id, + events.id, + events.version, + null, + "Generated archive events", + ); + + const scoped = await app.inject({ + method: "GET", + url: `/catalog/metrics?databaseId=${database.id}`, + }); + expect(scoped.statusCode).toBe(200); + expect(scoped.json()).toEqual({ + scope: "database", + databaseId: database.id, + tables: 2, + columns: 4, + sensitiveColumns: 1, + relationships: 1, + descriptionTargets: 6, + describedTargets: 2, + descriptionCoverage: 33, + updatedAt: expect.any(String), + }); + + const global = await app.inject({ method: "GET", url: "/catalog/metrics" }); + expect(global.statusCode).toBe(200); + expect(global.json()).toEqual({ + scope: "global", + databaseId: null, + tables: 3, + columns: 5, + sensitiveColumns: 1, + relationships: 1, + descriptionTargets: 8, + describedTargets: 3, + descriptionCoverage: 38, + updatedAt: expect.any(String), + }); + expect(Number.isNaN(Date.parse(global.json().updatedAt))).toBe(false); +}); + +test("validates fleet metric scope and requires database.manage", async () => { + const { app } = setup(); + const unknown = await app.inject({ + method: "GET", + url: "/catalog/metrics?databaseId=99999999-9999-4999-8999-999999999999", + }); + expect(unknown.statusCode).toBe(404); + expect(unknown.json()).toEqual({ + code: "database_not_found", + message: "Database configuration was not found.", + }); + + const invalid = await app.inject({ + method: "GET", + url: "/catalog/metrics?databaseId=not-a-uuid", + }); + expect(invalid.statusCode).toBe(400); + expect(invalid.json()).toEqual({ + code: "database_invalid", + message: "Database configuration is invalid.", + }); + + const { app: restrictedApp } = setup({ AUTH_MODE: "upstream" }); + const forbidden = await restrictedApp.inject({ + method: "GET", + url: "/catalog/metrics", + headers: { + "x-thoth-principal-issuer": "portal", + "x-thoth-principal-subject": "catalog-reader", + "x-thoth-is-admin": "0", + }, + }); + expect(forbidden.statusCode).toBe(403); + expect(forbidden.json()).toEqual({ + code: "auth_forbidden", + error: "This operation is not permitted", + }); +}); diff --git a/backend/test/catalog-description-generation-routes.test.ts b/backend/test/catalog-description-generation-routes.test.ts index b10a18bb..f7efae5e 100644 --- a/backend/test/catalog-description-generation-routes.test.ts +++ b/backend/test/catalog-description-generation-routes.test.ts @@ -160,7 +160,18 @@ test("suggests sensitive flags from structural metadata without persisting them" }); expect(response.statusCode).toBe(200); - expect(response.json()).toEqual({ + const responseBody = response.json(); + expect(responseBody).toMatchObject({ + run: { + databaseId: database.id, + scope: "all", + modelId: configuredModel.id, + status: "completed", + total: 1, + suggestedSensitive: 1, + suggestedNonSensitive: 0, + errorSummary: null, + }, suggestions: [{ columnId: column.id, tableId: table.id, @@ -173,6 +184,43 @@ test("suggests sensitive flags from structural metadata without persisting them" }); expect(await repository.getColumn(database.id, column.tableId, column.id)) .toMatchObject({ sensitive: false }); + expect(responseBody.run).not.toHaveProperty("suggestions"); + + const history = await app.inject({ + method: "GET", + url: "/catalog/sensitive-data-suggestion-runs?limit=1", + }); + expect(history.statusCode).toBe(200); + expect(history.json()).toEqual([responseBody.run]); + expect(history.body).not.toContain(column.id); + + const detail = await app.inject({ + method: "GET", + url: `/catalog/sensitive-data-suggestion-runs/${responseBody.run.id}`, + }); + expect(detail.statusCode).toBe(200); + expect(detail.json()).toEqual(responseBody.run); + + const events = await app.inject({ + method: "GET", + url: `/catalog/sensitive-data-suggestion-runs/${responseBody.run.id}/events-list`, + }); + expect(events.statusCode).toBe(200); + expect(events.body).not.toContain(column.id); + expect(events.json()).toMatchObject([ + { + runId: responseBody.run.id, + sequence: 1, + level: "info", + message: "Sensitive-field suggestion generation started.", + }, + { + runId: responseBody.run.id, + sequence: 2, + level: "info", + message: "Sensitive-field suggestion generation completed for 1 column.", + }, + ]); const request = modelCompleter.complete.mock.calls[0]![0] as ModelCompletionRequest; const prompt = request.messages.map((message) => message.content).join("\n"); @@ -505,6 +553,42 @@ test("explains a sensitive-data suggestion provider failure without exposing pro expect(response.body).not.toContain("model completion failed"); expect(await repository.getColumn(database.id, column.tableId, column.id)) .toMatchObject({ sensitive: false }); + + const history = await app.inject({ + method: "GET", + url: "/catalog/sensitive-data-suggestion-runs", + }); + expect(history.statusCode).toBe(200); + const [failedRun] = history.json(); + expect(failedRun).toMatchObject({ + databaseId: database.id, + status: "failed", + total: 1, + suggestedSensitive: 0, + suggestedNonSensitive: 0, + errorSummary: "Sensitive-field suggestion generation failed.", + }); + + const events = await app.inject({ + method: "GET", + url: `/catalog/sensitive-data-suggestion-runs/${failedRun.id}/events-list`, + }); + expect(events.statusCode).toBe(200); + expect(events.json()).toMatchObject([ + { + runId: failedRun.id, + sequence: 1, + level: "info", + message: "Sensitive-field suggestion generation started.", + }, + { + runId: failedRun.id, + sequence: 2, + level: "error", + message: "Sensitive-field suggestion generation failed.", + }, + ]); + expect(events.body).not.toContain("model completion failed"); } finally { await app.close(); } @@ -2676,10 +2760,25 @@ test("requires database.manage for every Description Generation route", async () url: "/catalog/description-generation-runs/99999999-9999-4999-8999-999999999999/events", headers, }), + app.inject({ + method: "GET", + url: "/catalog/sensitive-data-suggestion-runs", + headers, + }), + app.inject({ + method: "GET", + url: "/catalog/sensitive-data-suggestion-runs/99999999-9999-4999-8999-999999999999", + headers, + }), + app.inject({ + method: "GET", + url: "/catalog/sensitive-data-suggestion-runs/99999999-9999-4999-8999-999999999999/events-list", + headers, + }), ]); expect(responses.map((response) => response.statusCode)).toEqual([ - 403, 403, 403, 403, 403, 403, 403, + 403, 403, 403, 403, 403, 403, 403, 403, 403, 403, ]); expect(modelCompleter.complete).not.toHaveBeenCalled(); } finally { diff --git a/backend/test/catalog-description-generation.integration.test.ts b/backend/test/catalog-description-generation.integration.test.ts index 771d5e69..830796e3 100644 --- a/backend/test/catalog-description-generation.integration.test.ts +++ b/backend/test/catalog-description-generation.integration.test.ts @@ -12,6 +12,7 @@ import { up as upTables } from "../src/catalog/migrations/002_catalog_tables.js" import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema_sync.js"; import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js"; import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js"; +import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js"; import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js"; import { loadConfig } from "../src/config.js"; import type { WorkspaceRegistry } from "../src/workspaces/registry.js"; @@ -46,6 +47,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a await upSchemaSync(db); await upSensitiveDataFlag(db); await upDescriptionGeneration(db); + await upSensitiveSuggestionRuns(db); const repository = new KyselyCatalogRepository(db); const database = await repository.create({ workspaceId: "psd-clinical", diff --git a/backend/test/catalog-repository.integration.test.ts b/backend/test/catalog-repository.integration.test.ts index ecdf0cda..1a8559fa 100644 --- a/backend/test/catalog-repository.integration.test.ts +++ b/backend/test/catalog-repository.integration.test.ts @@ -11,6 +11,7 @@ import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema import { up as upRuntimeSequencePrivileges } from "../src/catalog/migrations/004_catalog_runtime_sequence_privileges.js"; import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js"; import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js"; +import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js"; const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0; @@ -98,6 +99,33 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo patientName.generatedDescription, true, )).toMatchObject({ sensitive: true }); + expect(await repository.getCatalogMetrics(created.id)).toEqual({ + scope: "database", + databaseId: created.id, + tables: 2, + columns: 4, + sensitiveColumns: 1, + relationships: 0, + descriptionTargets: 6, + describedTargets: 1, + descriptionCoverage: 17, + updatedAt: expect.any(String), + }); + expect(await repository.getCatalogMetrics()).toEqual({ + scope: "global", + databaseId: null, + tables: 2, + columns: 4, + sensitiveColumns: 1, + relationships: 0, + descriptionTargets: 6, + describedTargets: 1, + descriptionCoverage: 17, + updatedAt: expect.any(String), + }); + expect(await repository.getCatalogMetrics( + "99999999-9999-4999-8999-999999999999", + )).toBeUndefined(); const refreshedColumnsSnapshot: ObservedSchemaSnapshot = { ...fullColumnsSnapshot, schemaVersion: 2, @@ -340,7 +368,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates sel } }, 60_000); -test.skipIf(!dockerAvailable)("PostgreSQL repository persists globally exclusive Description Generation Runs and ordered events", async () => { +test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and sensitive suggestion run histories", async () => { const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start(); const db = new Kysely({ dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }), @@ -352,6 +380,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists globally exclusive await upSchemaSync(db); await upSensitiveDataFlag(db); await upDescriptionGeneration(db); + await upSensitiveSuggestionRuns(db); const repository = new KyselyCatalogRepository(db); const firstDatabase = await repository.create({ workspaceId: "generation-one", @@ -519,6 +548,63 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists globally exclusive }), ]); expect(await repository.getActiveDescriptionGenerationRun()).toBeUndefined(); + + const suggestionRun = await repository.createSensitiveDataSuggestionRun( + firstDatabase.id, + "selected_columns", + "openai-mini", + ); + expect(suggestionRun).toMatchObject({ + databaseId: firstDatabase.id, + status: "running", + total: 0, + suggestedSensitive: 0, + suggestedNonSensitive: 0, + startedAt: expect.any(String), + }); + await repository.appendSensitiveDataSuggestionEvent( + suggestionRun.id, + "info", + "Sensitive-field suggestion generation started.", + ); + await repository.appendSensitiveDataSuggestionEvent( + suggestionRun.id, + "info", + "Sensitive-field suggestion generation completed for 2 columns.", + ); + expect(await repository.updateSensitiveDataSuggestionRun(suggestionRun.id, { + status: "completed", + total: 2, + suggestedSensitive: 1, + suggestedNonSensitive: 1, + finishedAt: new Date().toISOString(), + })).toMatchObject({ + status: "completed", + total: 2, + suggestedSensitive: 1, + suggestedNonSensitive: 1, + }); + expect(await repository.listSensitiveDataSuggestionEvents(suggestionRun.id, 1)).toEqual([ + expect.objectContaining({ sequence: 2, level: "info" }), + ]); + expect((await repository.listSensitiveDataSuggestionRuns(1))[0]).toMatchObject({ + id: suggestionRun.id, + }); + + const interruptedSuggestionRun = await repository.createSensitiveDataSuggestionRun( + secondDatabase.id, + "all", + "openai-mini", + ); + expect(await repository.interruptActiveSensitiveDataSuggestionRuns( + "Sensitive-field suggestion generation was interrupted by backend restart.", + )).toEqual([ + expect.objectContaining({ + id: interruptedSuggestionRun.id, + status: "interrupted", + finishedAt: expect.any(String), + }), + ]); } finally { await db.destroy(); await container.stop(); diff --git a/frontend/e2e/database-management-layout.spec.ts b/frontend/e2e/database-management-layout.spec.ts new file mode 100644 index 00000000..83877a16 --- /dev/null +++ b/frontend/e2e/database-management-layout.spec.ts @@ -0,0 +1,563 @@ +import { expect, test, type Page, type Route } from "@playwright/test"; +import { createAuthenticationStack } from "./fixtures/auth-stack.mjs"; + +test.describe.configure({ mode: "serial" }); +test.use({ viewport: { width: 1910, height: 911 } }); + +let stack: Awaited>; + +const databaseId = "11111111-1111-4111-8111-111111111111"; +const database = { + id: databaseId, + workspaceId: "psd-clinical", + workspaceName: "Policlinico San Donato", + workspaceAvailable: true, + configured: true, + engine: "postgres", + databaseName: "warehouse", + schema: "datawarehouse", + version: 3, + createdAt: "2026-08-27T08:00:00Z", + updatedAt: "2026-08-27T09:00:00Z", + binding: { + transport: "postgres_direct", + host: "db.internal", + port: 5432, + username: "reader", + }, + connectionStatus: "reachable", + testedVersion: 3, + secrets: { + password: false, + apiKey: false, + sshPrivateKey: false, + sshPrivateKeyPassphrase: false, + sshKnownHosts: false, + tlsCa: false, + }, +}; + +const table = { + id: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + databaseId, + name: "patients", + sourceComment: "Patients imported from the clinical source", + description: null, + generatedDescription: null, + version: 1, + createdAt: "2026-08-27T08:00:00Z", + updatedAt: "2026-08-27T09:00:00Z", +}; + +function metrics(requestedDatabaseId: string | null) { + return { + scope: requestedDatabaseId ? "database" : "global", + databaseId: requestedDatabaseId, + tables: 1, + columns: 0, + sensitiveColumns: 0, + relationships: 0, + descriptionTargets: 1, + describedTargets: 0, + descriptionCoverage: 0, + updatedAt: "2026-08-31T09:00:00Z", + }; +} + +function json(route: Route, body: unknown, status = 200) { + return route.fulfill({ + status, + contentType: "application/json", + body: JSON.stringify(body), + }); +} + +async function signInAsAdmin(page: Page) { + const account = stack.localAccount("admin"); + await page.getByLabel("Username").fill(account.username); + await page.locator('input[name="password"]').fill(account.password); + await page.getByRole("button", { name: "Sign in", exact: true }).click(); + await expect(page.getByTestId("app-shell")).toBeVisible({ timeout: 30_000 }); +} + +async function expectContextPanelGeometry(page: Page, accessibleName: string) { + const manager = page + .getByTestId("conversation-column") + .getByRole("main", { name: "Database management" }); + const panel = manager.getByRole("dialog", { name: accessibleName }); + const sessionRail = page.getByRole("complementary", { name: "Session navigation" }); + + await expect(panel).toBeVisible(); + expect( + await panel.evaluate((element) => element.hasAttribute("data-catalog-panel")), + ).toBe(true); + await expect( + panel.locator(':scope > [data-catalog-panel-region="header"]'), + ).toHaveCount(1); + await expect( + panel.locator(':scope > [data-catalog-panel-region="body"]'), + ).toHaveCount(1); + const [managerBox, panelBox, sessionRailBox] = await Promise.all([ + manager.boundingBox(), + panel.boundingBox(), + sessionRail.boundingBox(), + ]); + expect(managerBox).not.toBeNull(); + expect(panelBox).not.toBeNull(); + expect(sessionRailBox).not.toBeNull(); + if (!managerBox || !panelBox || !sessionRailBox) return; + + const managerCenter = managerBox.x + managerBox.width / 2; + const panelCenter = panelBox.x + panelBox.width / 2; + expect(Math.abs(panelCenter - managerCenter)).toBeLessThanOrEqual(1); + expect(Math.abs(panelBox.width / managerBox.width - 0.6)).toBeLessThanOrEqual(0.005); + expect(panelBox.x).toBeGreaterThanOrEqual(managerBox.x - 1); + expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(managerBox.x + managerBox.width + 1); + expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(sessionRailBox.x + 1); + expect(panelBox.y).toBeGreaterThanOrEqual(managerBox.y); + expect(panelBox.y + panelBox.height).toBeLessThanOrEqual(managerBox.y + managerBox.height + 1); +} + +async function expectContextPanelHeaderMatchesNewSession( + page: Page, + accessibleName: string, +) { + const manager = page + .getByTestId("conversation-column") + .getByRole("main", { name: "Database management" }); + const panel = manager.getByRole("dialog", { name: accessibleName }); + const header = panel.locator(':scope > [data-catalog-panel-region="header"]'); + const newSession = page + .getByRole("complementary", { name: "Session navigation" }) + .getByRole("button", { name: "New session", exact: true }); + + await expect(header).toHaveCount(1); + await expect(newSession).toBeVisible(); + + const [headerStyle, actionStyle] = await Promise.all([ + header.evaluate((element) => ({ + backgroundColor: getComputedStyle(element).backgroundColor, + backgroundImage: getComputedStyle(element).backgroundImage, + })), + newSession.evaluate((element) => ({ + backgroundColor: getComputedStyle(element).backgroundColor, + })), + ]); + + expect(headerStyle.backgroundImage).toBe("none"); + expect(headerStyle.backgroundColor).not.toBe("rgba(0, 0, 0, 0)"); + expect(headerStyle.backgroundColor).toBe(actionStyle.backgroundColor); +} + +async function expectWorkAreaPanelGeometry(page: Page, accessibleName: string) { + const workArea = page.getByTestId("conversation-column"); + const panel = workArea.getByRole("dialog", { name: accessibleName }); + const sessionRail = page.getByRole("complementary", { name: "Session navigation" }); + + await expect(panel).toBeVisible(); + await expect(panel).toHaveAttribute("aria-modal", "false"); + await expect(panel).toHaveAttribute("data-work-area-panel", ""); + await expect(panel.locator(':scope > [data-work-area-panel-region="header"]')).toHaveCount(1); + await expect(panel.locator(':scope > [data-work-area-panel-region="body"]')).toHaveCount(1); + + const [workAreaBox, panelBox, sessionRailBox] = await Promise.all([ + workArea.boundingBox(), + panel.boundingBox(), + sessionRail.boundingBox(), + ]); + expect(workAreaBox).not.toBeNull(); + expect(panelBox).not.toBeNull(); + expect(sessionRailBox).not.toBeNull(); + if (!workAreaBox || !panelBox || !sessionRailBox) return; + + expect(Math.abs( + panelBox.x + panelBox.width / 2 - (workAreaBox.x + workAreaBox.width / 2), + )).toBeLessThanOrEqual(1); + expect(Math.abs(panelBox.width / workAreaBox.width - 0.6)).toBeLessThanOrEqual(0.005); + expect(panelBox.x).toBeGreaterThanOrEqual(workAreaBox.x - 1); + expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(workAreaBox.x + workAreaBox.width + 1); + expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(sessionRailBox.x + 1); + expect(panelBox.y).toBeGreaterThanOrEqual(workAreaBox.y); + expect(panelBox.y + panelBox.height).toBeLessThanOrEqual(workAreaBox.y + workAreaBox.height + 1); +} + +async function expectWorkAreaPanelHeaderMatchesNewSession( + page: Page, + accessibleName: string, +) { + const panel = page + .getByTestId("conversation-column") + .getByRole("dialog", { name: accessibleName }); + const header = panel.locator(':scope > [data-work-area-panel-region="header"]'); + const newSession = page + .getByRole("complementary", { name: "Session navigation" }) + .getByRole("button", { name: "New session", exact: true }); + const [headerStyle, actionStyle] = await Promise.all([ + header.evaluate((element) => ({ + backgroundColor: getComputedStyle(element).backgroundColor, + backgroundImage: getComputedStyle(element).backgroundImage, + })), + newSession.evaluate((element) => ({ + backgroundColor: getComputedStyle(element).backgroundColor, + })), + ]); + + expect(headerStyle.backgroundImage).toBe("none"); + expect(headerStyle.backgroundColor).toBe(actionStyle.backgroundColor); +} + +test.beforeAll(async () => { + stack = await createAuthenticationStack(); + await stack.useLocalMode(); +}); + +test.afterAll(async () => { + await stack?.close(); +}); + +test("context panels stay inside the manager and the Tables grid sits in a sidebar-colored frame", async ({ page }) => { + const unexpectedCatalogRequests: string[] = []; + const responses: Record = { + "GET /api/catalog/databases": [database], + "GET /api/catalog/metadata-generation/models": { + models: [], + default: null, + }, + "GET /api/catalog/description-generation-runs?limit=50": [], + "GET /api/catalog/sensitive-data-suggestion-runs?limit=50": [], + [`GET /api/catalog/databases/${databaseId}/tables`]: [table], + }; + + await page.route("**/api/catalog/**", async (route) => { + const request = route.request(); + const url = new URL(request.url()); + const key = `${request.method()} ${url.pathname}${url.search}`; + + if (request.method() === "GET" && url.pathname === "/api/catalog/metrics") { + const requestedId = url.searchParams.get("databaseId"); + if (requestedId === null || requestedId === databaseId) { + await json(route, metrics(requestedId)); + return; + } + } + + if (Object.hasOwn(responses, key)) { + await json(route, responses[key]); + return; + } + + unexpectedCatalogRequests.push(key); + await json(route, { code: "unexpected_catalog_request" }, 500); + }); + + await page.goto(stack.publicUrl); + await signInAsAdmin(page); + + await page + .getByRole("button", { name: "Database management", exact: true }) + .click(); + + await expect( + page.getByRole("complementary", { name: "Session navigation" }), + ).toBeVisible(); + await expect( + page.getByTestId("conversation-column").getByRole("main", { + name: "Database management", + }), + ).toBeVisible(); + await expect( + page.getByRole("button", { name: "Back to workspace", exact: true }), + ).toHaveCount(0); + + await page + .getByRole("button", { name: "Edit Policlinico San Donato", exact: true }) + .click(); + await expectContextPanelGeometry(page, "Edit database"); + await expectContextPanelHeaderMatchesNewSession(page, "Edit database"); + await page + .getByRole("dialog", { name: "Edit database" }) + .getByRole("button", { name: "Close database panel" }) + .click(); + + await page + .getByRole("button", { name: "Description history", exact: true }) + .click(); + await expectContextPanelGeometry(page, "Description generation"); + await expectContextPanelHeaderMatchesNewSession(page, "Description generation"); + await page + .getByRole("dialog", { name: "Description generation" }) + .getByRole("button", { name: /close/i }) + .click(); + + await page.setViewportSize({ width: 1280, height: 800 }); + await page + .getByRole("button", { name: "Edit Policlinico San Donato", exact: true }) + .click(); + await expectContextPanelGeometry(page, "Edit database"); + await page + .getByRole("dialog", { name: "Edit database" }) + .getByRole("button", { name: "Close database panel" }) + .click(); + await page.setViewportSize({ width: 1910, height: 911 }); + + await page + .getByRole("button", { + name: "View tables for Policlinico San Donato", + exact: true, + }) + .click(); + + await expect( + page.getByRole("region", { + name: "Tables for Policlinico San Donato", + }), + ).toBeVisible(); + + const applicationBar = page.locator( + 'main[aria-label="Database management"] .thot-fleet-ledger__header', + ); + await expect(applicationBar).toHaveCount(1); + await expect( + applicationBar.getByRole("heading", { name: "Database management" }), + ).toBeVisible(); + await expect( + applicationBar.getByRole("region", { name: "Catalog status" }), + ).toBeVisible(); + await expect( + applicationBar.getByRole("group", { name: "Database management actions" }), + ).toBeVisible(); + await expect(applicationBar).not.toContainText("Thoth catalog · Fleet ledger"); + await expect(applicationBar).not.toContainText("Inspect physical metadata"); + + const applicationBarLayout = await applicationBar.evaluate((element) => { + const title = element.querySelector(".thot-fleet-ledger__heading-copy"); + const status = element.querySelector(".thot-fleet-ledger__header-status"); + const actions = element.querySelector(".thot-fleet-ledger__header-actions"); + if (!title || !status || !actions) { + throw new Error("Application bar regions are missing"); + } + const centerY = (node: Element) => { + const rect = node.getBoundingClientRect(); + return rect.top + rect.height / 2; + }; + return { + backgroundImage: getComputedStyle(element).backgroundImage, + height: element.getBoundingClientRect().height, + centers: [centerY(title), centerY(status), centerY(actions)], + }; + }); + + expect(applicationBarLayout.backgroundImage).toBe("none"); + expect(applicationBarLayout.height).toBeLessThanOrEqual(64); + expect( + Math.max(...applicationBarLayout.centers) - Math.min(...applicationBarLayout.centers), + ).toBeLessThanOrEqual(1); + + await expect(page.getByRole("row", { name: /patients/ })).toBeVisible(); + + await page + .getByRole("button", { name: "Edit description for patients", exact: true }) + .click(); + await expectContextPanelGeometry(page, "Review table description"); + await expectContextPanelHeaderMatchesNewSession(page, "Review table description"); + const tableEditor = page.getByRole("dialog", { name: "Review table description" }); + const tableEditorFooter = tableEditor.locator( + ':scope > [data-catalog-panel-region="footer"]', + ); + await expect(tableEditorFooter).toHaveCount(1); + await expect( + tableEditorFooter.getByRole("button", { name: "Cancel", exact: true }), + ).toBeVisible(); + await expect( + tableEditorFooter.getByRole("button", { name: "Save metadata", exact: true }), + ).toBeVisible(); + await tableEditor + .getByRole("button", { name: "Close table metadata", exact: true }) + .click(); + + const gridRoot = page.locator( + 'main[aria-label="Database management"] .thot-fleet-ledger-grid .ag-root-wrapper', + ); + + await expect(gridRoot).toHaveCount(1); + await expect(gridRoot).toBeVisible(); + + await expect.poll( + () => gridRoot.evaluate((element) => { + const manager = element.closest('main[aria-label="Database management"]'); + if (!manager) throw new Error("Database manager is missing"); + + const managerBottom = manager.getBoundingClientRect().bottom; + const gridBottom = element.getBoundingClientRect().bottom; + + return Math.max( + Math.abs(window.innerHeight - managerBottom), + Math.abs(managerBottom - gridBottom - 30), + ); + }), + { message: "AG Grid must end 30px above the manager bottom" }, + ).toBeLessThanOrEqual(1); + + const gapColors = await gridRoot.evaluate((element) => { + const workspace = element.closest(".thot-fleet-ledger__workspace"); + const tableSurface = element.closest(".thot-fleet-ledger__content"); + const sidebar = document.querySelector('aside[aria-label="Session navigation"]'); + if (!workspace || !tableSurface || !sidebar) { + throw new Error("Application layout is missing"); + } + + return { + gap: getComputedStyle(workspace).backgroundColor, + table: getComputedStyle(tableSurface).backgroundColor, + sidebar: getComputedStyle(sidebar).backgroundColor, + }; + }); + + expect(gapColors.gap).not.toBe("rgba(0, 0, 0, 0)"); + expect(gapColors.gap).toBe(gapColors.sidebar); + expect(gapColors.gap).not.toBe(gapColors.table); + + const workspace = page.locator( + 'main[aria-label="Database management"] .thot-fleet-ledger__workspace', + ); + const tableSurface = page.locator( + 'main[aria-label="Database management"] .thot-fleet-ledger__content', + ); + const sessionNavigation = page.getByRole("complementary", { + name: "Session navigation", + }); + + await expect.poll( + async () => { + const [workspaceBox, tableBox, gridBox, sidebarBox] = await Promise.all([ + workspace.boundingBox(), + tableSurface.boundingBox(), + gridRoot.boundingBox(), + sessionNavigation.boundingBox(), + ]); + if (!workspaceBox || !tableBox || !gridBox || !sidebarBox) { + throw new Error("Database frame geometry is missing"); + } + + return Math.max( + Math.abs(tableBox.x - workspaceBox.x - 20), + Math.abs(tableBox.x + tableBox.width - sidebarBox.x), + Math.abs(tableBox.y + tableBox.height - gridBox.y - gridBox.height), + ); + }, + { + message: + "The table must have a 20px left frame and its right border must stop at the footer", + }, + ).toBeLessThanOrEqual(1); + + const frameBorders = await tableSurface.evaluate((element) => { + const sidebar = document.querySelector('aside[aria-label="Session navigation"]'); + if (!sidebar) throw new Error("Session navigation is missing"); + + return { + tableEnd: Number.parseFloat(getComputedStyle(element).borderInlineEndWidth), + sidebarStart: Number.parseFloat( + getComputedStyle(sidebar).borderInlineStartWidth, + ), + }; + }); + + expect(frameBorders.tableEnd).toBe(1); + expect(frameBorders.sidebarStart).toBe(0); + await expect(page.locator(".thot-session-navigation__boundary")).toHaveCount(0); + + expect(unexpectedCatalogRequests).toEqual([]); +}); + +test("Workspace and Pi management share the centered work-area panel without covering the session rail", async ({ page }) => { + await page.route("**/api/workspace-registry/status", (route) => json(route, { + branch: "main", + head: "a".repeat(40), + ahead: 0, + behind: 0, + degraded: false, + repository: { + host: "git.example.test", + repository: "analytics/thoth-workspaces", + transport: "ssh", + }, + })); + await page.route("**/api/workspaces", (route) => json(route, [])); + await page.route("**/api/pi-management/status", (route) => json(route, { + version: "0.80.3", + ready: true, + credentials: "present", + config: { provider: "zai", model: "glm-5.2", reasoning: "medium" }, + checkedAt: "2026-08-31T09:00:00Z", + })); + await page.route("**/api/pi-management/options", (route) => json(route, { + providers: ["zai"], + models: [{ provider: "zai", id: "glm-5.2" }], + reasoning: ["low", "medium", "high"], + checkedAt: "2026-08-31T09:00:00Z", + })); + + await page.goto(stack.publicUrl); + await signInAsAdmin(page); + + for (const viewport of [ + { width: 1910, height: 911 }, + { width: 1280, height: 800 }, + ]) { + await page.setViewportSize(viewport); + const workspaceTrigger = page.getByRole("button", { + name: "Workspace management", + exact: true, + }); + await workspaceTrigger.click(); + await expectWorkAreaPanelGeometry(page, "Workspace management"); + await expectWorkAreaPanelHeaderMatchesNewSession(page, "Workspace management"); + + const piTrigger = page.getByRole("button", { name: "Pi management", exact: true }); + await piTrigger.click(); + await expect( + page.getByRole("dialog", { name: "Workspace management" }), + ).toHaveCount(0); + await expectWorkAreaPanelGeometry(page, "Pi management"); + await expectWorkAreaPanelHeaderMatchesNewSession(page, "Pi management"); + await page + .getByRole("dialog", { name: "Pi management" }) + .getByRole("button", { name: "Close Pi management" }) + .click(); + await expect(piTrigger).toBeFocused(); + } + + await page.setViewportSize({ width: 720, height: 800 }); + await page.getByRole("button", { name: "Workspace management", exact: true }).click(); + const compactWorkArea = page.getByTestId("conversation-column"); + const compactPanel = compactWorkArea.getByRole("dialog", { name: "Workspace management" }); + const [compactWorkAreaBox, compactPanelBox] = await Promise.all([ + compactWorkArea.boundingBox(), + compactPanel.boundingBox(), + ]); + expect(compactWorkAreaBox).not.toBeNull(); + expect(compactPanelBox).not.toBeNull(); + if (compactWorkAreaBox && compactPanelBox) { + expect(Math.abs(compactPanelBox.width - compactWorkAreaBox.width + 16)).toBeLessThanOrEqual(1); + expect(Math.abs( + compactPanelBox.x + compactPanelBox.width / 2 + - (compactWorkAreaBox.x + compactWorkAreaBox.width / 2), + )).toBeLessThanOrEqual(1); + } + const compactNavigation = compactPanel.getByRole("navigation", { name: "Workspaces" }); + const compactContent = compactPanel.getByRole("region", { + name: "Workspace configuration", + }); + const [compactNavigationBox, compactContentBox] = await Promise.all([ + compactNavigation.boundingBox(), + compactContent.boundingBox(), + ]); + expect(compactNavigationBox).not.toBeNull(); + expect(compactContentBox).not.toBeNull(); + if (compactNavigationBox && compactContentBox) { + expect(compactContentBox.y).toBeGreaterThanOrEqual( + compactNavigationBox.y + compactNavigationBox.height - 1, + ); + } +}); diff --git a/frontend/src/api/catalog-databases.test.ts b/frontend/src/api/catalog-databases.test.ts new file mode 100644 index 00000000..dd59c07c --- /dev/null +++ b/frontend/src/api/catalog-databases.test.ts @@ -0,0 +1,60 @@ +import { http, HttpResponse } from "msw"; +import { expect, test } from "vitest"; +import { server } from "../test/msw"; +import { getCatalogMetrics, type CatalogMetrics } from "./catalog-databases"; + +const databaseId = "d4baf0f5-b9c3-4dc5-aad2-2f5676a36e64"; + +const globalMetrics: CatalogMetrics = { + scope: "global", + databaseId: null, + tables: 163, + columns: 2_275, + sensitiveColumns: 5, + relationships: 21, + descriptionTargets: 2_438, + describedTargets: 1_829, + descriptionCoverage: 75.02, + updatedAt: "2026-08-31T08:45:00.000Z", +}; + +test("loads global catalog metrics without a database filter", async () => { + let requestedUrl: URL | undefined; + server.use(http.get("/api/catalog/metrics", ({ request }) => { + requestedUrl = new URL(request.url); + return HttpResponse.json(globalMetrics); + })); + + await expect(getCatalogMetrics()).resolves.toEqual(globalMetrics); + expect(requestedUrl?.pathname).toBe("/api/catalog/metrics"); + expect(requestedUrl?.searchParams.has("databaseId")).toBe(false); +}); + +test("loads metrics scoped to the requested database", async () => { + const databaseMetrics: CatalogMetrics = { + ...globalMetrics, + scope: "database", + databaseId, + tables: 42, + columns: 508, + sensitiveColumns: 3, + relationships: 8, + descriptionTargets: 550, + describedTargets: 413, + descriptionCoverage: 75.09, + }; + let requestedDatabaseId: string | null = null; + server.use(http.get("/api/catalog/metrics", ({ request }) => { + requestedDatabaseId = new URL(request.url).searchParams.get("databaseId"); + return HttpResponse.json(databaseMetrics); + })); + + await expect(getCatalogMetrics(databaseId)).resolves.toEqual(databaseMetrics); + expect(requestedDatabaseId).toBe(databaseId); +}); + +test("propagates an unsuccessful catalog metrics response", async () => { + server.use(http.get("/api/catalog/metrics", () => new HttpResponse(null, { status: 503 }))); + + await expect(getCatalogMetrics()).rejects.toMatchObject({ status: 503 }); +}); diff --git a/frontend/src/api/catalog-databases.ts b/frontend/src/api/catalog-databases.ts index b792550b..aa0e087b 100644 --- a/frontend/src/api/catalog-databases.ts +++ b/frontend/src/api/catalog-databases.ts @@ -52,6 +52,19 @@ export interface CatalogDatabase { secrets: Record; } +export interface CatalogMetrics { + scope: "global" | "database"; + databaseId: string | null; + tables: number; + columns: number; + sensitiveColumns: number; + relationships: number; + descriptionTargets: number; + describedTargets: number; + descriptionCoverage: number; + updatedAt: string | null; +} + export interface DatabaseConfiguration { workspaceId: string; engine: "postgres"; @@ -109,6 +122,7 @@ export interface SensitiveDataSuggestion { export interface SensitiveDataSuggestions { suggestions: SensitiveDataSuggestion[]; + run: SensitiveDataSuggestionRun; } export type SensitiveDataSuggestionRequest = @@ -116,6 +130,32 @@ export type SensitiveDataSuggestionRequest = | { scope: "selected_tables"; targetIds: string[] } | { scope: "selected_columns"; targetIds: string[] }; +export type SensitiveDataSuggestionStatus = "running" | "completed" | "failed" | "interrupted"; + +export interface SensitiveDataSuggestionRun { + id: string; + databaseId: string; + scope: SensitiveDataSuggestionRequest["scope"]; + modelId: string; + status: SensitiveDataSuggestionStatus; + total: number; + suggestedSensitive: number; + suggestedNonSensitive: number; + createdAt: string; + startedAt: string | null; + updatedAt: string; + finishedAt: string | null; + errorSummary: string | null; +} + +export interface SensitiveDataSuggestionEvent { + runId: string; + sequence: number; + level: "info" | "warning" | "error"; + message: string; + createdAt: string; +} + export interface CatalogRelationshipColumn { position: number; sourceColumnId: string; @@ -245,6 +285,12 @@ export interface DescriptionGenerationEvent { export const listCatalogDatabases = () => apiFetch("/catalog/databases"); +export const getCatalogMetrics = (databaseId?: string) => apiFetch( + databaseId + ? `/catalog/metrics?databaseId=${encodeURIComponent(databaseId)}` + : "/catalog/metrics", +); + export const listMetadataGenerationModels = () => apiFetch("/catalog/metadata-generation/models"); @@ -388,6 +434,21 @@ export const suggestSensitiveFields = ( { method: "POST", body: JSON.stringify({ modelId, ...selection }) }, ); +export const getSensitiveDataSuggestionRun = (runId: string) => + apiFetch( + `/catalog/sensitive-data-suggestion-runs/${encodeURIComponent(runId)}`, + ); + +export const listSensitiveDataSuggestionRuns = (limit = 50) => + apiFetch( + `/catalog/sensitive-data-suggestion-runs?limit=${encodeURIComponent(String(limit))}`, + ); + +export const listSensitiveDataSuggestionEvents = (runId: string, after = 0) => + apiFetch( + `/catalog/sensitive-data-suggestion-runs/${encodeURIComponent(runId)}/events-list?after=${after}`, + ); + export const listCatalogRelationships = (databaseId: string) => apiFetch(`/catalog/databases/${encodeURIComponent(databaseId)}/relationships`); diff --git a/frontend/src/api/catalog-description-generation.test.ts b/frontend/src/api/catalog-description-generation.test.ts index 08568a1b..dc1e146c 100644 --- a/frontend/src/api/catalog-description-generation.test.ts +++ b/frontend/src/api/catalog-description-generation.test.ts @@ -3,9 +3,13 @@ import { server } from "../test/msw"; import { cancelDescriptionGenerationRun, descriptionGenerationEventsUrl, + getSensitiveDataSuggestionRun, listDescriptionGenerationRuns, + listSensitiveDataSuggestionEvents, + listSensitiveDataSuggestionRuns, unlockDescriptionGenerationRun, type DescriptionGenerationRun, + type SensitiveDataSuggestionRun, } from "./catalog-databases"; const historicalRun: DescriptionGenerationRun = { @@ -27,6 +31,22 @@ const historicalRun: DescriptionGenerationRun = { errorSummary: null, }; +const sensitiveSuggestionRun: SensitiveDataSuggestionRun = { + id: "99999999-9999-4999-8999-999999999999", + databaseId: "11111111-1111-4111-8111-111111111111", + scope: "selected_columns", + modelId: "local-qwen", + status: "completed", + total: 4, + suggestedSensitive: 2, + suggestedNonSensitive: 2, + createdAt: "2026-08-28T09:00:00Z", + startedAt: "2026-08-28T09:00:00Z", + updatedAt: "2026-08-28T09:00:01Z", + finishedAt: "2026-08-28T09:00:01Z", + errorSummary: null, +}; + test("lists newest-first persisted description-generation history with a bounded limit", async () => { let requestedLimit: string | null = null; server.use(http.get("/api/catalog/description-generation-runs", ({ request }) => { @@ -74,3 +94,34 @@ test("builds the same-origin description-generation SSE replay URL from an event `/api/catalog/description-generation-runs/${historicalRun.id}/events?after=17`, ); }); + +test("lists, reads, and replays persisted sensitive-suggestion history", async () => { + let requestedLimit: string | null = null; + let requestedAfter: string | null = null; + const event = { + runId: sensitiveSuggestionRun.id, + sequence: 3, + level: "info" as const, + message: "Classified 4 columns", + createdAt: "2026-08-28T09:00:01Z", + }; + server.use( + http.get("/api/catalog/sensitive-data-suggestion-runs", ({ request }) => { + requestedLimit = new URL(request.url).searchParams.get("limit"); + return HttpResponse.json([sensitiveSuggestionRun]); + }), + http.get("/api/catalog/sensitive-data-suggestion-runs/:runId", () => ( + HttpResponse.json(sensitiveSuggestionRun) + )), + http.get("/api/catalog/sensitive-data-suggestion-runs/:runId/events-list", ({ request }) => { + requestedAfter = new URL(request.url).searchParams.get("after"); + return HttpResponse.json([event]); + }), + ); + + await expect(listSensitiveDataSuggestionRuns(50)).resolves.toEqual([sensitiveSuggestionRun]); + await expect(getSensitiveDataSuggestionRun(sensitiveSuggestionRun.id)).resolves.toEqual(sensitiveSuggestionRun); + await expect(listSensitiveDataSuggestionEvents(sensitiveSuggestionRun.id, 2)).resolves.toEqual([event]); + expect(requestedLimit).toBe("50"); + expect(requestedAfter).toBe("2"); +}); diff --git a/frontend/src/api/client.test.ts b/frontend/src/api/client.test.ts index 301e529b..0d50a9bc 100644 --- a/frontend/src/api/client.test.ts +++ b/frontend/src/api/client.test.ts @@ -149,6 +149,9 @@ test.each([ ["catalog_table_not_found", "One or more selected catalog tables were not found."], ["sensitive_data_suggestion_invalid_response", "The model returned an incomplete or invalid classification. No suggestions were applied."], ["sensitive_data_suggestion_provider_unavailable", "The selected model could not complete the request. No suggestions were applied."], + ["sensitive_data_suggestion_history_request_invalid", "Sensitive suggestion history parameters are invalid."], + ["sensitive_data_suggestion_history_failed", "Sensitive suggestion history could not be loaded."], + ["sensitive_data_suggestion_run_not_found", "The sensitive suggestion run was not found."], ])("maps the catalog error code %s to safe local copy", async (code, message) => { const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue( new Response(JSON.stringify({ code, message: "provider detail must not be trusted" }), { diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index d9c6b04e..5cbcebf3 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -30,6 +30,9 @@ const safeErrorCodes = new Set([ "sensitive_data_suggestion_invalid_response", "sensitive_data_suggestion_provider_unavailable", "sensitive_data_suggestion_failed", + "sensitive_data_suggestion_history_request_invalid", + "sensitive_data_suggestion_history_failed", + "sensitive_data_suggestion_run_not_found", "schema_sync_conflict", "schema_introspection_failed", "schema_request_invalid", "schema_operation_failed", "sync_run_not_found", "table_stale", "column_stale", ]); @@ -91,6 +94,9 @@ const localCodeMessages: Record = { sensitive_data_suggestion_invalid_response: "The model returned an incomplete or invalid classification. No suggestions were applied.", sensitive_data_suggestion_provider_unavailable: "The selected model could not complete the request. No suggestions were applied.", sensitive_data_suggestion_failed: "Sensitive-field suggestions failed before review. No changes were applied.", + sensitive_data_suggestion_history_request_invalid: "Sensitive suggestion history parameters are invalid.", + sensitive_data_suggestion_history_failed: "Sensitive suggestion history could not be loaded.", + sensitive_data_suggestion_run_not_found: "The sensitive suggestion run was not found.", schema_sync_conflict: "A schema synchronization is already active or no longer current.", schema_introspection_failed: "The database schema could not be read safely.", schema_request_invalid: "The schema request is invalid.", diff --git a/frontend/src/shell/AppShell.database-management.test.tsx b/frontend/src/shell/AppShell.database-management.test.tsx index cbe7e056..c5dc4047 100644 --- a/frontend/src/shell/AppShell.database-management.test.tsx +++ b/frontend/src/shell/AppShell.database-management.test.tsx @@ -1,4 +1,4 @@ -import { render, screen, waitFor } from "@testing-library/react"; +import { render, screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { http, HttpResponse } from "msw"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; @@ -6,7 +6,7 @@ import { server } from "../test/msw"; import { FakeEventSource } from "../test/fakeEventSource"; import { useSessionStore } from "../store/sessionStore"; import { clearAuthState, setAuthState } from "../auth/authState"; -import { AppShell } from "./AppShell"; +import { AppShell, resolveDatabaseManagementPresentation } from "./AppShell"; function renderShell() { const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); @@ -29,6 +29,7 @@ beforeEach(() => { session: null, }); localStorage.clear(); + window.history.replaceState(null, "", "/"); FakeEventSource.instances = []; (globalThis as any).EventSource = FakeEventSource; useSessionStore.getState().resetSession(); @@ -49,33 +50,66 @@ beforeEach(() => { http.get("/api/workspaces", () => HttpResponse.json([])), http.get("/api/models", () => HttpResponse.json({ models: [] })), http.post("/api/runtime/prewarm", () => new HttpResponse(null, { status: 202 })), + http.get("/api/catalog/metrics", () => HttpResponse.json({ + scope: "global", + databaseId: null, + tables: 0, + columns: 0, + sensitiveColumns: 0, + relationships: 0, + descriptionTargets: 0, + describedTargets: 0, + descriptionCoverage: 0, + updatedAt: null, + })), ); }); -test("opens the database management surface and returns to the core", async () => { - renderShell(); - - const composer = screen.getByRole("textbox", { name: /new question/i }); - await userEvent.type(composer, "Unsent draft"); - - const databaseManagement = screen.getByRole("button", { name: "Database management" }); - await userEvent.click(databaseManagement); - - expect(screen.getByRole("main", { name: "Database management" })).toBeVisible(); - expect(databaseManagement).toHaveAttribute("aria-current", "page"); - expect(databaseManagement).toHaveClass("w-full"); - expect(databaseManagement).not.toHaveClass("bg-sidebar-accent"); - expect(screen.queryByRole("textbox", { name: /new question/i })).not.toBeInTheDocument(); - - await userEvent.click(screen.getByRole("button", { name: "New session" })); - - await waitFor(() => { - expect(screen.queryByRole("main", { name: "Database management" })).not.toBeInTheDocument(); - }); - expect(screen.getByRole("textbox", { name: /new question/i })).toHaveValue("Unsent draft"); +test("uses Fleet Ledger by default and gates the legacy fallback to non-production environments", () => { + expect(resolveDatabaseManagementPresentation({ + isDevelopment: true, + mode: "development", + search: "", + })).toBe("fleet"); + expect(resolveDatabaseManagementPresentation({ + isDevelopment: true, + mode: "development", + search: "?db-ui=legacy", + })).toBe("legacy"); + expect(resolveDatabaseManagementPresentation({ + isDevelopment: false, + mode: "staging", + legacyFlag: "true", + search: "", + })).toBe("legacy"); + expect(resolveDatabaseManagementPresentation({ + isDevelopment: false, + mode: "production", + legacyFlag: "true", + search: "?db-ui=legacy", + })).toBe("fleet"); }); -test("keeps a live core session connected and returns when that session is opened", async () => { +test("replaces the core conversation while keeping the session navigation", async () => { + renderShell(); + + const conversationColumn = screen.getByTestId("conversation-column"); + const sessionNavigation = screen.getByRole("complementary", { name: "Session navigation" }); + const databaseManagement = within(sessionNavigation).getByRole("button", { name: "Database management" }); + await userEvent.click(databaseManagement); + + const manager = screen.getByRole("main", { name: "Database management" }); + expect(manager).toBeVisible(); + expect(conversationColumn).toContainElement(manager); + expect(screen.getByRole("complementary", { name: "Session navigation" })).toBe(sessionNavigation); + expect(sessionNavigation).toBeVisible(); + expect(within(sessionNavigation).getByRole("button", { name: "New session" })).toBeVisible(); + expect(databaseManagement).toHaveAttribute("aria-current", "page"); + expect(screen.queryByRole("button", { name: "Back to workspace" })).not.toBeInTheDocument(); + expect(screen.queryByRole("textbox", { name: /new question/i })).not.toBeInTheDocument(); +}); + +test("keeps a live core session connected while returning from database management", async () => { server.use( http.get("/api/sessions", () => HttpResponse.json([{ id: "s1", @@ -113,6 +147,7 @@ test("keeps a live core session connected and returns when that session is opene expect(source.closed).toBe(false); expect(FakeEventSource.instances).toHaveLength(1); + expect(session).toBeVisible(); await userEvent.click(session); await waitFor(() => { @@ -174,14 +209,15 @@ test("does not leave database management without confirming a dirty form", async await userEvent.clear(schema); await userEvent.type(schema, "reporting"); - await userEvent.click(screen.getByRole("button", { name: "New session" })); + const newSession = screen.getByRole("button", { name: "New session" }); + await userEvent.click(newSession); expect(confirm).toHaveBeenCalledWith("Discard unsaved database changes and leave database management?"); expect(screen.getByRole("main", { name: "Database management" })).toBeVisible(); expect(schema).toHaveValue("reporting"); confirm.mockReturnValue(true); - await userEvent.click(screen.getByRole("button", { name: "New session" })); + await userEvent.click(newSession); await waitFor(() => expect(screen.queryByRole("main", { name: "Database management" })).not.toBeInTheDocument()); confirm.mockRestore(); }); diff --git a/frontend/src/shell/AppShell.new-session.test.tsx b/frontend/src/shell/AppShell.new-session.test.tsx index bc2a87db..9a75d9a3 100644 --- a/frontend/src/shell/AppShell.new-session.test.tsx +++ b/frontend/src/shell/AppShell.new-session.test.tsx @@ -166,6 +166,11 @@ test("opens Workspace management from the right sidebar without interrupting the await userEvent.click(screen.getByRole("button", { name: "Workspace management" })); expect(await screen.findByRole("heading", { name: "Workspace management" })).toBeVisible(); + const dialog = screen.getByRole("dialog", { name: "Workspace management" }); + const workArea = screen.getByTestId("conversation-column"); + const sessionNavigation = screen.getByRole("complementary", { name: "Session navigation" }); + expect(workArea).toContainElement(dialog); + expect(sessionNavigation).not.toContainElement(dialog); expect(screen.getByTestId("app-shell")).toHaveAttribute("data-activity-layout", "closed"); }); diff --git a/frontend/src/shell/AppShell.session-mgmt.test.tsx b/frontend/src/shell/AppShell.session-mgmt.test.tsx index 13af31d8..cc08a04d 100644 --- a/frontend/src/shell/AppShell.session-mgmt.test.tsx +++ b/frontend/src/shell/AppShell.session-mgmt.test.tsx @@ -121,9 +121,14 @@ test("Pi management preserves the open session summary and the model activity ti expect(await screen.findByRole("complementary", { name: "Session summary" })).toHaveTextContent("Domanda originale"); await user.click(screen.getByRole("button", { name: "Pi management" })); expect(await screen.findByRole("heading", { name: "Pi management" })).toBeVisible(); - const hiddenSummary = document.querySelector('aside[aria-label="Session summary"]'); - expect(hiddenSummary).toHaveAttribute("aria-hidden", "true"); - expect(hiddenSummary).toHaveTextContent("Attiva uno"); + const dialog = screen.getByRole("dialog", { name: "Pi management" }); + const workArea = screen.getByTestId("conversation-column"); + const sessionNavigation = screen.getByRole("complementary", { name: "Session navigation" }); + expect(workArea).toContainElement(dialog); + expect(sessionNavigation).not.toContainElement(dialog); + const preservedSummary = screen.getByRole("complementary", { name: "Session summary" }); + expect(preservedSummary).not.toHaveAttribute("aria-hidden", "true"); + expect(preservedSummary).toHaveTextContent("Attiva uno"); await user.click(screen.getByRole("button", { name: "Close Pi management" })); await waitFor(() => { expect(screen.getByRole("complementary", { name: "Session summary" })).toHaveTextContent("Domanda originale"); diff --git a/frontend/src/shell/AppShell.tsx b/frontend/src/shell/AppShell.tsx index fefe489a..3f66e79e 100644 --- a/frontend/src/shell/AppShell.tsx +++ b/frontend/src/shell/AppShell.tsx @@ -5,6 +5,7 @@ import { CentralStatus } from "./CentralStatus"; import { ModelActivityPanel } from "./ModelActivityPanel"; import { WorkspaceManager } from "./WorkspaceManager"; import { PiManagement } from "./PiManagement"; +import { WorkAreaPanelHost } from "./WorkAreaPanel"; import { useActivityPanelResize } from "./useActivityPanelResize"; import { useSessionPanelResize } from "./useSessionPanelResize"; import { NavSessions } from "./NavSessions"; @@ -41,6 +42,35 @@ interface AppShellProps { } type ActiveSurface = "core" | "database-management"; +type ActiveManagementPanel = "workspace" | "pi" | null; + +type DatabaseManagementPresentation = "legacy" | "fleet"; + +interface DatabaseManagementPresentationEnvironment { + isDevelopment: boolean; + mode: string; + legacyFlag?: string | boolean; + search: string; +} + +/** + * Fleet Ledger is the release-safe default. The old surface can only be + * requested explicitly from a development server or a staging build, so an + * accidentally retained flag can never switch the production UI back. + */ +export function resolveDatabaseManagementPresentation({ + isDevelopment, + mode, + legacyFlag, + search, +}: DatabaseManagementPresentationEnvironment): DatabaseManagementPresentation { + const legacyAllowed = isDevelopment || mode.trim().toLowerCase() === "staging"; + if (!legacyAllowed) return "fleet"; + + const legacyRequestedByQuery = new URLSearchParams(search).get("db-ui")?.toLowerCase() === "legacy"; + const legacyRequestedByEnvironment = String(legacyFlag ?? "").trim().toLowerCase() === "true"; + return legacyRequestedByQuery || legacyRequestedByEnvironment ? "legacy" : "fleet"; +} export function AppShell({ canLogout }: AppShellProps) { const authenticatedUser = useAuthUser(); @@ -123,12 +153,17 @@ export function AppShell({ canLogout }: AppShellProps) { const queryClient = useQueryClient(); const [showActivity, setShowActivity] = useState(false); const [activeSurface, setActiveSurface] = useState("core"); + const databaseManagementPresentation = resolveDatabaseManagementPresentation({ + isDevelopment: import.meta.env.DEV, + mode: import.meta.env.MODE, + legacyFlag: import.meta.env.VITE_DB_MANAGEMENT_LEGACY, + search: window.location.search, + }); const databaseNavigationRef = useRef({ dirty: false, busy: false }); const updateDatabaseNavigationState = useCallback((state: { dirty: boolean; busy: boolean }) => { databaseNavigationRef.current = state; }, []); - const [workspaceManagerOpen, setWorkspaceManagerOpen] = useState(false); - const [piManagementOpen, setPiManagementOpen] = useState(false); + const [activeManagementPanel, setActiveManagementPanel] = useState(null); const [activeOpen, setActiveOpen] = useState(true); const [archiveOpen, setArchiveOpen] = useState(false); const [renameTarget, setRenameTarget] = useState(null); @@ -218,6 +253,7 @@ export function AppShell({ canLogout }: AppShellProps) { const s = sessions.find((x) => x.id === id); if (!s) return; setActiveSurface("core"); + setActiveManagementPanel(null); // A session with a live Pi runtime opens straight into its live view: doResume // reconnects to the already-active runtime and replays its pending gate, so an // in-progress session never shows an empty screen that reads as "stopped". Cold or @@ -239,6 +275,7 @@ export function AppShell({ canLogout }: AppShellProps) { async function doResume(id: string, databaseExitApproved = false) { if (!databaseExitApproved && !canLeaveDatabaseManagement()) return; setActiveSurface("core"); + setActiveManagementPanel(null); const guard = captureAuthOperation({ sessionId: id, disposalEpoch: operationEpochRef.current }); if (!guard) return; const token = ++resumeInvocationRef.current; @@ -522,6 +559,7 @@ export function AppShell({ canLogout }: AppShellProps) { function startNewSession() { if (!canLeaveDatabaseManagement()) return; setActiveSurface("core"); + setActiveManagementPanel(null); invalidateResumeIntent(); newSessionOperationRef.current = null; resetSession(); @@ -623,7 +661,7 @@ export function AppShell({ canLogout }: AppShellProps) { )} - {showActivity && setShowActivity(false)} onOpenWorkspaceManager={() => setWorkspaceManagerOpen(true)} hidden={activeSurface !== "core"} />} + {showActivity && setShowActivity(false)} onOpenWorkspaceManager={() => setActiveManagementPanel("workspace")} hidden={activeSurface !== "core"} />} {activeSurface === "core" && showActivity && desktopSplit && (
+ {activeSurface === "database-management" && ( )}
-
+ setActiveManagementPanel(null)} + canManageWorkspace={canManageWorkspace} + canManageSecrets={canManageWorkspaceSecrets} + /> + {canManagePi && ( + setActiveManagementPanel(null)} + /> + )} + {/* Right session rail */} - {!showActivity && ( -