From 453174603832d83b1072a0bb3d6be93fa9c10a2e Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 2 Sep 2026 11:38:47 +0200 Subject: [PATCH] feat: refine metadata catalog workflows --- PROJECT_STATE.md | 7 +- backend/src/app.ts | 4 +- .../catalog/description-generation-worker.ts | 125 ++++++--- .../src/catalog/description-source-sampler.ts | 175 +++++++++--- backend/src/catalog/memory-repository.ts | 10 +- backend/src/catalog/migrate.ts | 2 + .../catalog/migrations/009_ai_token_usage.ts | 20 ++ backend/src/catalog/model-completer.ts | 12 +- backend/src/catalog/repository.ts | 12 + .../src/catalog/sensitive-data-suggester.ts | 12 +- .../sensitive-data-suggestion-runner.ts | 26 ++ backend/src/catalog/types.ts | 12 + .../routes/catalog-description-generation.ts | 6 + ...alog-description-generation-routes.test.ts | 109 ++++++-- ...description-generation.integration.test.ts | 10 +- ...catalog-description-source-sampler.test.ts | 71 ++++- .../catalog-repository.integration.test.ts | 5 + .../test/metadata-generation-models.test.ts | 7 +- backend/test/model-completer.test.ts | 4 +- deploy/psd/thothii-installation.yaml.example | 6 + docs/general/pi-configuration.md | 1 + ...026-08-26-metadata-catalog-from-thothai.md | 14 +- ...talog-description-generation-acceptance.md | 1 + .../e2e/database-management-layout.spec.ts | 95 ++++++- .../prototypes/database-management/fleet.tsx | 1 - frontend/src/api/catalog-databases.ts | 6 + frontend/src/components/ui/button.tsx | 2 +- frontend/src/shell/AppShell.auth.test.tsx | 1 + frontend/src/shell/AppShell.tsx | 26 +- .../src/shell/DatabaseManagementPage.test.tsx | 136 +++++++-- frontend/src/shell/DatabaseManagementPage.tsx | 100 ++++++- .../database-management/CatalogSyncDrawer.tsx | 7 +- .../database-management/DatabaseColumns.tsx | 38 +-- .../DatabaseFleetQueryErrors.test.tsx | 1 - .../database-management/DatabaseForm.tsx | 5 +- .../database-management/DatabaseGrid.tsx | 260 +++++++----------- .../database-management/DatabaseSyncMenu.tsx | 7 +- .../database-management/DatabaseTables.tsx | 103 ++++--- .../DescriptionGenerationDrawer.test.tsx | 3 + .../DescriptionGenerationDrawer.tsx | 102 ++++--- .../FleetActionSelector.test.tsx | 14 + .../FleetActionSelector.tsx | 8 +- .../database-management/FleetLedgerShell.css | 104 ++++++- .../RecentRunsSuccessStyle.test.ts | 13 + ...sitiveDataSuggestionHistoryDrawer.test.tsx | 5 + .../SensitiveDataSuggestionHistoryDrawer.tsx | 106 ++++--- harness/tht/internal/litellm_completion.py | 36 ++- .../installation_metadata_generation_test.go | 8 +- 48 files changed, 1345 insertions(+), 493 deletions(-) create mode 100644 backend/src/catalog/migrations/009_ai_token_usage.ts diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index b7b37cf7..cecb114d 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -108,10 +108,9 @@ and staging only: `?db-ui=legacy` is honored in Vite development or when prototype on port `5173` also remains temporary until owner acceptance of the integrated surface, after which both migration aids can be removed. -Schema refresh is one durable asynchronous engine with database-table, database-column, -selected-table-column, relationship, and full-database actions. Database-level menus expose the -table, all-column, relationship, and full scopes separately; selecting tables exposes column -synchronization plus manual column and relationship cleanup for that subset. Database selections +Schema refresh is one durable asynchronous engine with database-table, selected-table-column, +relationship, and full-database actions. Database-level menus expose only the table, relationship, +and full scopes; selecting tables exposes column synchronization plus manual column and relationship cleanup for that subset. Database selections also expose manual table and relationship cleanup. Cleanup selections are atomic and share the one-active-operation-per-database exclusion with synchronization. Runs have leases and restart recovery, atomic apply, destructive-diff confirmation with re-scan, cancellation before diff --git a/backend/src/app.ts b/backend/src/app.ts index 01c5c7e9..bd030242 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -59,7 +59,7 @@ import { DescriptionGenerationWorker } from "./catalog/description-generation-wo import { SensitiveDataSuggester } from "./catalog/sensitive-data-suggester.js"; import { SensitiveDataSuggestionRunner } from "./catalog/sensitive-data-suggestion-runner.js"; import { - PostgresDescriptionSourceSampler, + ConcreteDescriptionSourceSampler, type DescriptionSourceSampler, } from "./catalog/description-source-sampler.js"; import { catalogDescriptionGenerationRoutes } from "./routes/catalog-description-generation.js"; @@ -177,7 +177,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc { connectTimeoutMs: config.workspaceDiagnosticTimeoutMs }, ); const descriptionSourceSampler = deps?.descriptionSourceSampler - ?? new PostgresDescriptionSourceSampler(catalogPostgresAccess); + ?? new ConcreteDescriptionSourceSampler(catalogPostgresAccess, workspaceSecretStore); const descriptionGenerationWorker = new DescriptionGenerationWorker( catalogRepository, workspaceRegistry, diff --git a/backend/src/catalog/description-generation-worker.ts b/backend/src/catalog/description-generation-worker.ts index 233cf22e..3cf64e6d 100644 --- a/backend/src/catalog/description-generation-worker.ts +++ b/backend/src/catalog/description-generation-worker.ts @@ -1,7 +1,7 @@ import { z } from "zod"; import type { WorkspaceRegistry } from "../workspaces/registry.js"; import type { MetadataGenerationModels, ResolvedMetadataGenerationModel } from "./metadata-generation-models.js"; -import type { ModelCompleter, ModelCompletionMessage } from "./model-completer.js"; +import type { ModelCompleter, ModelCompletionMessage, ModelCompletionResult } from "./model-completer.js"; import { ModelCompletionCancelledError, ModelCompletionProviderError, @@ -38,6 +38,7 @@ const MAX_SAMPLE_FIELDS_PER_ROW = 4; const MAX_SAMPLE_COLUMNS = 4; const MAX_REPRESENTATIVE_VALUES_PER_REQUEST = 5; const MAX_TARGET_SAMPLE_JSON_BYTES = 8 * 1024; +const MAX_COMPLETION_ATTEMPTS_PER_BATCH = 2; const generatedOutcomeSchema = z.object({ targetId: z.uuid(), outcome: z.literal("generated"), @@ -55,7 +56,9 @@ const completionResponseSchema = z.object({ results: z.array(outcomeSchema).min(1).max(MAX_TARGETS_PER_BATCH), }).strict(); -class InvalidModelOutcomeError extends Error {} +class InvalidModelJsonError extends Error {} +class InvalidModelSchemaError extends Error {} +class MissingModelTargetsError extends Error {} class DescriptionGenerationBatchError extends Error { constructor( readonly failure: unknown, @@ -143,6 +146,9 @@ interface DescriptionGenerationCounters { generated: number; nonGeneratable: number; failed: number; + inputTokens: number; + cacheReadTokens: number; + outputTokens: number; consecutiveTechnicalFailures: number; } @@ -152,6 +158,9 @@ function persistedCounters(counters: DescriptionGenerationCounters) { generated: counters.generated, nonGeneratable: counters.nonGeneratable, failed: counters.failed, + inputTokens: counters.inputTokens, + cacheReadTokens: counters.cacheReadTokens, + outputTokens: counters.outputTokens, }; } @@ -625,36 +634,47 @@ function messagesFor( } function parseOutcomes(content: string, expectedTargetIds: readonly string[]): Map { + const trimmed = content.trim(); + const fenced = /^```(?:json)?[ \t]*\r?\n([\s\S]*?)\r?\n```$/iu.exec(trimmed); + let parsed: unknown; try { - const trimmed = content.trim(); - const fenced = /^```(?:json)?[ \t]*\r?\n([\s\S]*?)\r?\n```$/iu.exec(trimmed); - const outcomes = completionResponseSchema.parse(JSON.parse(fenced?.[1] ?? trimmed)).results; - const expected = new Set(expectedTargetIds); - if (outcomes.length !== expectedTargetIds.length || expected.size !== expectedTargetIds.length) { - throw new InvalidModelOutcomeError(); - } - const mapped = new Map(); - for (const outcome of outcomes) { - if (!expected.has(outcome.targetId) || mapped.has(outcome.targetId)) { - throw new InvalidModelOutcomeError(); - } - mapped.set(outcome.targetId, outcome.outcome === "generated" - ? { ...outcome, description: outcome.description.trim() } - : outcome); - } - if (mapped.size !== expected.size) throw new InvalidModelOutcomeError(); - return mapped; - } catch (error) { - if (error instanceof InvalidModelOutcomeError) throw error; - throw new InvalidModelOutcomeError(); + parsed = JSON.parse(fenced?.[1] ?? trimmed); + } catch { + throw new InvalidModelJsonError(); } + + const response = completionResponseSchema.safeParse(parsed); + if (!response.success) throw new InvalidModelSchemaError(); + + const outcomes = response.data.results; + const expected = new Set(expectedTargetIds); + if (outcomes.length !== expectedTargetIds.length || expected.size !== expectedTargetIds.length) { + throw new MissingModelTargetsError(); + } + const mapped = new Map(); + for (const outcome of outcomes) { + if (!expected.has(outcome.targetId) || mapped.has(outcome.targetId)) { + throw new MissingModelTargetsError(); + } + mapped.set(outcome.targetId, outcome.outcome === "generated" + ? { ...outcome, description: outcome.description.trim() } + : outcome); + } + if (mapped.size !== expected.size) throw new MissingModelTargetsError(); + return mapped; } function safeFailure(error: unknown): string { if (error instanceof DescriptionGenerationFailureStreakError) return error.message; const failure = error instanceof DescriptionGenerationBatchError ? error.failure : error; if (failure instanceof ModelCompletionProviderError) return "The model provider request failed."; - if (failure instanceof InvalidModelOutcomeError) return "The model response was invalid."; + if (failure instanceof InvalidModelJsonError) return "The model response was not valid JSON."; + if (failure instanceof InvalidModelSchemaError) { + return "The model response did not match the required schema."; + } + if (failure instanceof MissingModelTargetsError) { + return "The model response was missing one or more requested targets."; + } return "Description generation failed."; } @@ -907,6 +927,9 @@ export class DescriptionGenerationWorker { generated: 0, nonGeneratable: 0, failed: 0, + inputTokens: 0, + cacheReadTokens: 0, + outputTokens: 0, consecutiveTechnicalFailures: 0, }; await this.processTargets(run, database, plan.columnTargets, model, counters, signal); @@ -973,20 +996,41 @@ export class DescriptionGenerationWorker { ); } throwIfCancelled(signal); - let outcomes: Map; - try { - const content = await this.completer.complete({ - model, - messages: messagesFor(database, batch, run.language, sourceSamples), - signal, - }); - throwIfCancelled(signal); - outcomes = parseOutcomes(content, batch.map((target) => ( - target.kind === "column" ? target.column.id : target.table.id - ))); - } catch (error) { - if (error instanceof ModelCompletionCancelledError) throw error; - const batchError = new DescriptionGenerationBatchError(error, batch.map(failureTarget)); + const expectedTargetIds = batch.map((target) => ( + target.kind === "column" ? target.column.id : target.table.id + )); + const messages = messagesFor(database, batch, run.language, sourceSamples); + let outcomes: Map | undefined; + let terminalFailure: unknown; + for (let attempt = 1; attempt <= MAX_COMPLETION_ATTEMPTS_PER_BATCH; attempt += 1) { + try { + const completion = await this.completer.complete({ model, messages, signal }); + const result: ModelCompletionResult = typeof completion === "string" + ? { content: completion, usage: { input: 0, cacheRead: 0, output: 0 } } + : completion; + counters.inputTokens += result.usage.input; + counters.cacheReadTokens += result.usage.cacheRead; + counters.outputTokens += result.usage.output; + throwIfCancelled(signal); + outcomes = parseOutcomes(result.content, expectedTargetIds); + break; + } catch (error) { + if (error instanceof ModelCompletionCancelledError) throw error; + terminalFailure = error; + if (attempt < MAX_COMPLETION_ATTEMPTS_PER_BATCH) { + await this.appendEvent( + run.id, + "warning", + `${safeFailure(error)} Retrying batch (attempt ${attempt + 1} of ${MAX_COMPLETION_ATTEMPTS_PER_BATCH}).`, + ); + } + } + } + if (!outcomes) { + const batchError = new DescriptionGenerationBatchError( + terminalFailure, + batch.map(failureTarget), + ); counters.processed += batch.length; counters.failed += batch.length; counters.consecutiveTechnicalFailures += 1; @@ -1009,7 +1053,10 @@ export class DescriptionGenerationWorker { const targetId = target.kind === "column" ? target.column.id : target.table.id; const outcome = outcomes.get(targetId); if (!outcome) { - throw new DescriptionGenerationBatchError(new InvalidModelOutcomeError(), [failureTarget(target)]); + throw new DescriptionGenerationBatchError( + new MissingModelTargetsError(), + [failureTarget(target)], + ); } const generatedDescription = outcome.outcome === "generated" ? outcome.description diff --git a/backend/src/catalog/description-source-sampler.ts b/backend/src/catalog/description-source-sampler.ts index 17f56657..3db0cf9b 100644 --- a/backend/src/catalog/description-source-sampler.ts +++ b/backend/src/catalog/description-source-sampler.ts @@ -1,5 +1,8 @@ +import { readFile } from "node:fs/promises"; +import type { WorkspaceSecretStore } from "../workspaces/secret-store.js"; import type { CatalogPostgresAccess } from "./postgres-access.js"; -import type { WorkspaceDatabase } from "./types.js"; +import { CATALOG_SECRET_IDS } from "./secrets.js"; +import { CatalogConnectorError, type WorkspaceDatabase } from "./types.js"; const MAX_SOURCE_ROWS = 5; const MAX_REPRESENTATIVE_VALUES = 5; @@ -79,15 +82,82 @@ function distinctKey(value: Exclude): string return `${typeof value}:${String(value)}`; } -/** PostgreSQL-wire sampler. REST bindings remain unsupported by CatalogPostgresAccess. */ -export class PostgresDescriptionSourceSampler implements DescriptionSourceSampler { - constructor(private readonly access: CatalogPostgresAccess) {} +function columnsFor(target: DescriptionSourceSamplingTarget): string[] { + return [...new Set(target.columnNames)].slice(0, MAX_SOURCE_COLUMNS_PER_TARGET); +} + +function normalizedSample( + target: DescriptionSourceSamplingTarget, + columnNames: readonly string[], + sourceRows: readonly Record[], +): DescriptionTargetSourceSample { + const rows = sourceRows.slice(0, MAX_SOURCE_ROWS).map((row) => ({ + fields: columnNames.flatMap((name) => { + const value = normalizeValue(row[name]); + return value === undefined ? [] : [{ name, value }]; + }), + })); + const valuesByColumn = new Map[]>(); + const seenByColumn = new Map>(); + let representativeValueCount = 0; + for (const row of rows) { + for (const field of row.fields) { + if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break; + if (field.value === null) continue; + const seen = seenByColumn.get(field.name) ?? new Set(); + const key = distinctKey(field.value); + if (seen.has(key)) continue; + seen.add(key); + seenByColumn.set(field.name, seen); + const values = valuesByColumn.get(field.name) ?? []; + values.push(field.value); + valuesByColumn.set(field.name, values); + representativeValueCount += 1; + } + if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break; + } + return { + targetId: target.targetId, + tableName: target.tableName, + rows, + representativeValues: columnNames.flatMap((column) => { + const values = valuesByColumn.get(column); + return values && values.length > 0 ? [{ column, values }] : []; + }), + }; +} + +function samplingSql( + database: WorkspaceDatabase, + target: DescriptionSourceSamplingTarget, + columns: readonly string[], +): string { + const projections = columns.map((columnName) => { + const identifier = quoteIdentifier(columnName); + return `LEFT((${identifier})::text, ${MAX_SOURCE_VALUE_BYTES}) AS ${identifier}`; + }); + return [ + `SELECT ${projections.join(", ")}`, + `FROM ${quoteIdentifier(database.schema)}.${quoteIdentifier(target.tableName)}`, + `LIMIT ${MAX_SOURCE_ROWS}`, + ].join(" "); +} + +/** Bounded source sampler that follows the database's PostgreSQL-wire or REST binding. */ +export class ConcreteDescriptionSourceSampler implements DescriptionSourceSampler { + constructor( + private readonly access: CatalogPostgresAccess, + private readonly secretStore?: Pick, + ) {} async sample( database: WorkspaceDatabase, targets: readonly DescriptionSourceSamplingTarget[], signal: AbortSignal, ): Promise { + if (database.binding.transport === "rest_api") { + return await this.sampleRest(database, targets, signal); + } const client = await this.access.connect(database, signal); let transactionOpen = false; try { @@ -95,7 +165,7 @@ export class PostgresDescriptionSourceSampler implements DescriptionSourceSample transactionOpen = true; const samples: DescriptionTargetSourceSample[] = []; for (const target of targets) { - const columnNames = [...new Set(target.columnNames)].slice(0, MAX_SOURCE_COLUMNS_PER_TARGET); + const columnNames = columnsFor(target); if (columnNames.length === 0) { samples.push({ targetId: target.targetId, @@ -115,44 +185,7 @@ export class PostgresDescriptionSourceSampler implements DescriptionSourceSample "LIMIT $2", ].join(" "); const result = await client.query(sql, [MAX_SOURCE_VALUE_BYTES, MAX_SOURCE_ROWS]); - const rows = result.rows.slice(0, MAX_SOURCE_ROWS).map((row) => ({ - fields: columnNames.flatMap((name) => { - const value = normalizeValue(row[name]); - return value === undefined ? [] : [{ name, value }]; - }), - })); - const valuesByColumn = new Map< - string, - Exclude[] - >(); - const seenByColumn = new Map>(); - let representativeValueCount = 0; - for (const row of rows) { - for (const field of row.fields) { - if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break; - if (field.value === null) continue; - const seen = seenByColumn.get(field.name) ?? new Set(); - const key = distinctKey(field.value); - if (seen.has(key)) continue; - seen.add(key); - seenByColumn.set(field.name, seen); - const values = valuesByColumn.get(field.name) ?? []; - values.push(field.value); - valuesByColumn.set(field.name, values); - representativeValueCount += 1; - } - if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break; - } - const representativeValues = columnNames.flatMap((column) => { - const values = valuesByColumn.get(column); - return values && values.length > 0 ? [{ column, values }] : []; - }); - samples.push({ - targetId: target.targetId, - tableName: target.tableName, - rows, - representativeValues, - }); + samples.push(normalizedSample(target, columnNames, result.rows)); } return samples; } finally { @@ -160,4 +193,60 @@ export class PostgresDescriptionSourceSampler implements DescriptionSourceSample await client.end().catch(() => undefined); } } + + private async sampleRest( + database: WorkspaceDatabase, + targets: readonly DescriptionSourceSamplingTarget[], + signal: AbortSignal, + ): Promise { + if (!this.secretStore) throw new CatalogConnectorError("REST source sampling is not configured"); + const auth = database.binding.restAuth ?? "bearer"; + const materialized = this.secretStore.materialize( + database.workspaceId, + auth === "none" ? [] : [CATALOG_SECRET_IDS.apiKey], + ); + try { + const headers: Record = { "content-type": "application/json" }; + if (auth !== "none") { + const credentialFile = materialized.files.get(CATALOG_SECRET_IDS.apiKey); + if (!credentialFile) throw new CatalogConnectorError("REST API key is not configured"); + const credential = (await readFile(credentialFile, "utf8")).trim(); + if (auth === "bearer") headers.authorization = `Bearer ${credential}`; + else headers["x-api-key"] = credential; + } + const baseUrl = database.binding.baseUrl?.replace(/\/+$/, ""); + if (!baseUrl) throw new CatalogConnectorError("Database binding is incomplete"); + const samples: DescriptionTargetSourceSample[] = []; + for (const target of targets) { + const columnNames = columnsFor(target); + if (columnNames.length === 0) { + samples.push(normalizedSample(target, columnNames, [])); + continue; + } + const response = await fetch(`${baseUrl}/rpc/run_query`, { + method: "POST", + headers, + body: JSON.stringify({ query_text: samplingSql(database, target, columnNames) }), + signal, + }); + if (!response.ok) throw new CatalogConnectorError("REST source sampling failed"); + const body: unknown = await response.json(); + if (!Array.isArray(body) + || body.some((row) => !row || typeof row !== "object" || Array.isArray(row))) { + throw new CatalogConnectorError("REST source sampling response is invalid"); + } + samples.push(normalizedSample( + target, + columnNames, + body as Array>, + )); + } + return samples; + } catch (error) { + if (error instanceof CatalogConnectorError) throw error; + throw new CatalogConnectorError("REST source sampling failed"); + } finally { + materialized.release(); + } + } } diff --git a/backend/src/catalog/memory-repository.ts b/backend/src/catalog/memory-repository.ts index a373250f..e8cc5ee2 100644 --- a/backend/src/catalog/memory-repository.ts +++ b/backend/src/catalog/memory-repository.ts @@ -335,7 +335,10 @@ export class MemoryCatalogRepository implements CatalogRepository { processed: 0, generated: 0, nonGeneratable: 0, - failed: 0, + failed: 0, + inputTokens: 0, + cacheReadTokens: 0, + outputTokens: 0, createdAt: now, startedAt: null, updatedAt: now, @@ -444,7 +447,10 @@ export class MemoryCatalogRepository implements CatalogRepository { status: "running", total: 0, suggestedSensitive: 0, - suggestedNonSensitive: 0, + suggestedNonSensitive: 0, + inputTokens: 0, + cacheReadTokens: 0, + outputTokens: 0, createdAt: now, startedAt: now, updatedAt: now, diff --git a/backend/src/catalog/migrate.ts b/backend/src/catalog/migrate.ts index 8b6c7459..d04623a6 100644 --- a/backend/src/catalog/migrate.ts +++ b/backend/src/catalog/migrate.ts @@ -11,6 +11,7 @@ import * as descriptionGenerationRunsMigration from "./migrations/005_descriptio import * as sensitiveDataFlagMigration from "./migrations/006_sensitive_data_flag.js"; import * as sensitiveDataSuggestionRunsMigration from "./migrations/007_sensitive_data_suggestion_runs.js"; import * as catalogLogicalRelationshipsMigration from "./migrations/008_catalog_logical_relationships.js"; +import * as aiTokenUsageMigration from "./migrations/009_ai_token_usage.js"; const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL; const host = process.env.THT_CATALOG_DB_HOST; @@ -44,6 +45,7 @@ const provider: MigrationProvider = { "006_sensitive_data_flag": sensitiveDataFlagMigration, "007_sensitive_data_suggestion_runs": sensitiveDataSuggestionRunsMigration, "008_catalog_logical_relationships": catalogLogicalRelationshipsMigration, + "009_ai_token_usage": aiTokenUsageMigration, }; }, }; diff --git a/backend/src/catalog/migrations/009_ai_token_usage.ts b/backend/src/catalog/migrations/009_ai_token_usage.ts new file mode 100644 index 00000000..828d7d7b --- /dev/null +++ b/backend/src/catalog/migrations/009_ai_token_usage.ts @@ -0,0 +1,20 @@ +import type { Kysely } from "kysely"; +import type { CatalogDatabase } from "../repository.js"; + +export async function up(db: Kysely): Promise { + for (const table of ["description_generation_runs", "sensitive_data_suggestion_runs"] as const) { + await db.schema.alterTable(table) + .addColumn("input_tokens", "integer", (col) => col.notNull().defaultTo(0)) + .addColumn("cache_read_tokens", "integer", (col) => col.notNull().defaultTo(0)) + .addColumn("output_tokens", "integer", (col) => col.notNull().defaultTo(0)) + .execute(); + } +} + +export async function down(db: Kysely): Promise { + for (const table of ["sensitive_data_suggestion_runs", "description_generation_runs"] as const) { + await db.schema.alterTable(table) + .dropColumn("input_tokens").dropColumn("cache_read_tokens").dropColumn("output_tokens") + .execute(); + } +} diff --git a/backend/src/catalog/model-completer.ts b/backend/src/catalog/model-completer.ts index 48bb8acf..6136b1e6 100644 --- a/backend/src/catalog/model-completer.ts +++ b/backend/src/catalog/model-completer.ts @@ -4,7 +4,7 @@ import type { ResolvedMetadataGenerationModel } from "./metadata-generation-mode const MAX_HELPER_OUTPUT_BYTES = 64 * 1024; const helperOutputSchema = z.discriminatedUnion("ok", [ - z.object({ ok: z.literal(true), content: z.string() }).strict(), + z.object({ ok: z.literal(true), content: z.string(), usage: z.object({ input: z.number().int().nonnegative(), cacheRead: z.number().int().nonnegative(), output: z.number().int().nonnegative() }).strict().optional() }).strict(), z.object({ ok: z.literal(false), error: z.literal("provider_failure") }).strict(), ]); @@ -18,10 +18,12 @@ export interface ModelCompletionRequest { messages: readonly ModelCompletionMessage[]; signal: AbortSignal; } +export interface ModelCompletionUsage { input: number; cacheRead: number; output: number; } +export interface ModelCompletionResult { content: string; usage: ModelCompletionUsage; } /** The provider boundary used by Description Generation. */ export interface ModelCompleter { - complete(request: ModelCompletionRequest): Promise; + complete(request: ModelCompletionRequest): Promise; } export class ModelCompletionProviderError extends Error { @@ -47,7 +49,7 @@ export class PythonModelCompleter implements ModelCompleter { terminationGraceMs?: number; }) {} - async complete(request: ModelCompletionRequest): Promise { + async complete(request: ModelCompletionRequest): Promise { if (request.signal.aborted) throw new ModelCompletionCancelledError(); const payload = { model: `${request.model.provider}/${request.model.model}`, @@ -62,7 +64,7 @@ export class PythonModelCompleter implements ModelCompleter { ...(request.model.disableThinking === true ? { disable_thinking: true } : {}), }; - return await new Promise((resolve, reject) => { + return await new Promise((resolve, reject) => { const child = spawn( this.options.pythonExecutable, ["-m", this.options.helperModule ?? "tht.internal.litellm_completion"], @@ -135,7 +137,7 @@ export class PythonModelCompleter implements ModelCompleter { if (!output.ok) return fail(); settled = true; cleanup(); - resolve(output.content); + resolve({ content: output.content, usage: output.usage ?? { input: 0, cacheRead: 0, output: 0 } }); } catch { fail(); } diff --git a/backend/src/catalog/repository.ts b/backend/src/catalog/repository.ts index 79be4018..1ba24f28 100644 --- a/backend/src/catalog/repository.ts +++ b/backend/src/catalog/repository.ts @@ -171,6 +171,9 @@ interface DescriptionGenerationRunTable { generated: number; nonGeneratable: number; failed: number; + inputTokens: number; + cacheReadTokens: number; + outputTokens: number; createdAt: Timestamp; startedAt: Timestamp | null; updatedAt: Timestamp; @@ -195,6 +198,9 @@ interface SensitiveDataSuggestionRunTable { total: number; suggestedSensitive: number; suggestedNonSensitive: number; + inputTokens: number; + cacheReadTokens: number; + outputTokens: number; createdAt: Timestamp; startedAt: Timestamp; updatedAt: Timestamp; @@ -816,6 +822,9 @@ export class KyselyCatalogRepository implements CatalogRepository { generated: 0, nonGeneratable: 0, failed: 0, + inputTokens: 0, + cacheReadTokens: 0, + outputTokens: 0, startedAt: null, finishedAt: null, errorSummary: null, @@ -943,6 +952,9 @@ export class KyselyCatalogRepository implements CatalogRepository { total: 0, suggestedSensitive: 0, suggestedNonSensitive: 0, + inputTokens: 0, + cacheReadTokens: 0, + outputTokens: 0, finishedAt: null, errorSummary: null, }).returningAll().executeTakeFirstOrThrow(); diff --git a/backend/src/catalog/sensitive-data-suggester.ts b/backend/src/catalog/sensitive-data-suggester.ts index f66fd0ed..8fbd099b 100644 --- a/backend/src/catalog/sensitive-data-suggester.ts +++ b/backend/src/catalog/sensitive-data-suggester.ts @@ -1,6 +1,6 @@ import { z } from "zod"; import type { MetadataGenerationModels } from "./metadata-generation-models.js"; -import type { ModelCompleter, ModelCompletionMessage } from "./model-completer.js"; +import type { ModelCompleter, ModelCompletionMessage, ModelCompletionResult, ModelCompletionUsage } from "./model-completer.js"; import type { CatalogColumn, CatalogRepository, @@ -201,6 +201,8 @@ export class SensitiveDataSuggester { targetIds: readonly string[], signal: AbortSignal, onPrepared?: (total: number) => void | Promise, + onProgress?: (processed: number, suggestions: readonly SensitiveDataSuggestion[]) => void | Promise, + onUsage?: (usage: ModelCompletionUsage) => void | Promise, ): Promise { const database = await this.repository.get(databaseId); if (!database) throw new SensitiveDataSuggestionTargetNotFoundError("database"); @@ -212,11 +214,16 @@ export class SensitiveDataSuggester { for (const batch of batchesFor(database, columns)) { let received: Map | undefined; for (let attempt = 0; attempt < 2 && !received; attempt += 1) { - const content = await this.completer.complete({ + const completion = await this.completer.complete({ model, signal, messages: [systemMessage, { role: "user", content: userContent(database, batch) }], }); + const result: ModelCompletionResult = typeof completion === "string" + ? { content: completion, usage: { input: 0, cacheRead: 0, output: 0 } } + : completion; + await onUsage?.(result.usage); + const content = result.content; try { const parsed = responseSchema.parse(JSON.parse(content)); const expected = new Set(batch.map((column) => column.columnId)); @@ -240,6 +247,7 @@ export class SensitiveDataSuggester { currentSensitive: column.currentSensitive, sensitive: received!.get(column.columnId)!.sensitive, }))); + await onProgress?.(suggestions.length, suggestions.slice(-batch.length)); } return suggestions; } diff --git a/backend/src/catalog/sensitive-data-suggestion-runner.ts b/backend/src/catalog/sensitive-data-suggestion-runner.ts index c337f63c..f5b14287 100644 --- a/backend/src/catalog/sensitive-data-suggestion-runner.ts +++ b/backend/src/catalog/sensitive-data-suggestion-runner.ts @@ -10,6 +10,7 @@ import type { SensitiveDataSuggestionRun, SensitiveDataSuggestionScope, } from "./types.js"; +import type { ModelCompletionUsage } from "./model-completer.js"; const interruptedMessage = "Sensitive-field suggestion generation was interrupted by backend restart."; const failedMessage = "Sensitive-field suggestion generation failed."; @@ -73,6 +74,31 @@ export class SensitiveDataSuggestionRunner { }); if (!prepared) throw new Error("Sensitive Data Suggestion Run disappeared"); }, + async (processed, batch) => { + const suggestedSensitive = batch.filter((suggestion) => suggestion.sensitive).length; + const suggestedNonSensitive = batch.length - suggestedSensitive; + const current = await this.repository.getSensitiveDataSuggestionRun(started.id); + if (!current) throw new Error("Sensitive Data Suggestion Run disappeared"); + const progress = await this.repository.updateSensitiveDataSuggestionRun(started.id, { + suggestedSensitive: current.suggestedSensitive + suggestedSensitive, + suggestedNonSensitive: current.suggestedNonSensitive + suggestedNonSensitive, + }); + if (!progress) throw new Error("Sensitive Data Suggestion Run disappeared"); + await this.repository.appendSensitiveDataSuggestionEvent( + started.id, + "info", + `Classified ${processed} of ${progress.total} columns.`, + ); + }, + async (usage: ModelCompletionUsage) => { + const current = await this.repository.getSensitiveDataSuggestionRun(started.id); + if (!current) throw new Error("Sensitive Data Suggestion Run disappeared"); + await this.repository.updateSensitiveDataSuggestionRun(started.id, { + inputTokens: current.inputTokens + usage.input, + cacheReadTokens: current.cacheReadTokens + usage.cacheRead, + outputTokens: current.outputTokens + usage.output, + }); + }, ); const suggestedSensitive = suggestions.filter((suggestion) => suggestion.sensitive).length; const suggestedNonSensitive = suggestions.length - suggestedSensitive; diff --git a/backend/src/catalog/types.ts b/backend/src/catalog/types.ts index 86fdf9d2..42a14049 100644 --- a/backend/src/catalog/types.ts +++ b/backend/src/catalog/types.ts @@ -234,6 +234,9 @@ export interface DescriptionGenerationRun { generated: number; nonGeneratable: number; failed: number; + inputTokens: number; + cacheReadTokens: number; + outputTokens: number; createdAt: string; startedAt: string | null; updatedAt: string; @@ -250,6 +253,9 @@ export interface DescriptionGenerationRunUpdate { startedAt?: string | null; finishedAt?: string | null; errorSummary?: string | null; + inputTokens?: number; + cacheReadTokens?: number; + outputTokens?: number; } export interface DescriptionGenerationEvent { @@ -272,6 +278,9 @@ export interface SensitiveDataSuggestionRun { total: number; suggestedSensitive: number; suggestedNonSensitive: number; + inputTokens: number; + cacheReadTokens: number; + outputTokens: number; createdAt: string; startedAt: string; updatedAt: string; @@ -286,6 +295,9 @@ export interface SensitiveDataSuggestionRunUpdate { suggestedNonSensitive?: number; finishedAt?: string | null; errorSummary?: string | null; + inputTokens?: number; + cacheReadTokens?: number; + outputTokens?: number; } export interface SensitiveDataSuggestionEvent { diff --git a/backend/src/routes/catalog-description-generation.ts b/backend/src/routes/catalog-description-generation.ts index a79ce21d..fc45f944 100644 --- a/backend/src/routes/catalog-description-generation.ts +++ b/backend/src/routes/catalog-description-generation.ts @@ -101,6 +101,9 @@ function publicRun(run: DescriptionGenerationRun) { generated: run.generated, nonGeneratable: run.nonGeneratable, failed: run.failed, + inputTokens: run.inputTokens, + cacheReadTokens: run.cacheReadTokens, + outputTokens: run.outputTokens, createdAt: run.createdAt, startedAt: run.startedAt, updatedAt: run.updatedAt, @@ -129,6 +132,9 @@ function publicSensitiveDataSuggestionRun(run: SensitiveDataSuggestionRun) { total: run.total, suggestedSensitive: run.suggestedSensitive, suggestedNonSensitive: run.suggestedNonSensitive, + inputTokens: run.inputTokens, + cacheReadTokens: run.cacheReadTokens, + outputTokens: run.outputTokens, createdAt: run.createdAt, startedAt: run.startedAt, updatedAt: run.updatedAt, diff --git a/backend/test/catalog-description-generation-routes.test.ts b/backend/test/catalog-description-generation-routes.test.ts index f7efae5e..43008e25 100644 --- a/backend/test/catalog-description-generation-routes.test.ts +++ b/backend/test/catalog-description-generation-routes.test.ts @@ -218,6 +218,12 @@ test("suggests sensitive flags from structural metadata without persisting them" runId: responseBody.run.id, sequence: 2, level: "info", + message: "Classified 1 of 1 columns.", + }, + { + runId: responseBody.run.id, + sequence: 3, + level: "info", message: "Sensitive-field suggestion generation completed for 1 column.", }, ]); @@ -671,6 +677,7 @@ test("generates one selected Catalog Column from a single JSON code fence", asyn errorSummary: null, }); expect(Object.keys(start.json()).sort()).toEqual([ + "cacheReadTokens", "createdAt", "databaseId", "errorSummary", @@ -678,9 +685,11 @@ test("generates one selected Catalog Column from a single JSON code fence", asyn "finishedAt", "generated", "id", + "inputTokens", "language", "modelId", "nonGeneratable", + "outputTokens", "processed", "scope", "startedAt", @@ -1272,7 +1281,7 @@ test("Stop aborts source sampling before any model request", async () => { test("an isolated exhausted technical batch failure allows completion with errors", async () => { const modelCompleter: ModelCompleter = { complete: vi.fn(async (request) => { - if (vi.mocked(modelCompleter.complete).mock.calls.length === 1) { + if (vi.mocked(modelCompleter.complete).mock.calls.length <= 2) { throw new ModelCompletionProviderError(); } const context = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n")); @@ -1320,7 +1329,7 @@ test("an isolated exhausted technical batch failure allows completion with error failed: 10, errorSummary: "Description generation completed with errors.", }); - expect(modelCompleter.complete).toHaveBeenCalledTimes(2); + expect(modelCompleter.complete).toHaveBeenCalledTimes(3); const updated = new Map( (await repository.listColumns(database.id, table.id)).map((column) => [column.id, column]), ); @@ -1346,10 +1355,10 @@ test("success resets the technical-failure streak and the third later failure st const modelCompleter: ModelCompleter = { complete: vi.fn(async (request) => { const call = vi.mocked(modelCompleter.complete).mock.calls.length; - if ([1, 2, 4, 5, 6].includes(call)) { + if ([1, 2, 4, 5, 6, 7, 8, 9].includes(call)) { throw Object.assign(new ModelCompletionProviderError(), { message: sensitiveDiagnostic }); } - if (call > 6) throw new Error("a later batch must not start"); + if (call > 9) throw new Error("a later batch must not start"); const context = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n")); return JSON.stringify({ results: context.targets.map((target: { targetId: string }) => ({ @@ -1389,25 +1398,25 @@ test("success resets the technical-failure streak and the third later failure st expect(run).toMatchObject({ status: "failed", total: 61, - processed: 60, + processed: 50, generated: 10, nonGeneratable: 0, - failed: 50, + failed: 40, errorSummary: "Description generation stopped after three consecutive technical batch failures.", }); - expect(modelCompleter.complete).toHaveBeenCalledTimes(6); + expect(modelCompleter.complete).toHaveBeenCalledTimes(9); const requests = vi.mocked(modelCompleter.complete).mock.calls.map(([request]) => request); expect(new Set(requests.map((request) => request.signal)).size).toBe(1); expect(new Set(requests.map((request) => request.model.id))).toEqual(new Set([configuredModel.id])); const updated = new Map( (await repository.listColumns(database.id, table.id)).map((column) => [column.id, column]), ); - targetIds.slice(20, 30).forEach((targetId) => { + targetIds.slice(10, 20).forEach((targetId) => { expect(updated.get(targetId)?.generatedDescription).toBe("Successful reset batch."); }); - expect(updated.get(targetIds[60]!)?.generatedDescription).toBeNull(); + expect(updated.get(targetIds[50]!)?.generatedDescription).toBeNull(); const events = await repository.listDescriptionGenerationEvents(run.id); - expect(events.filter((event) => event.message.includes("model provider request failed"))).toHaveLength(5); + expect(events.filter((event) => event.message.includes("model provider request failed"))).toHaveLength(8); expect(events.at(-1)).toEqual(expect.objectContaining({ level: "error", message: "Description generation stopped after three consecutive technical batch failures.", @@ -1660,6 +1669,7 @@ test("Description Generation history is newest-first, bounded, and exposes only expect(response.statusCode).toBe(200); expect(response.json().map((run: { id: string }) => run.id)).toEqual(ids.slice(1).reverse()); expect(Object.keys(response.json()[0]).sort()).toEqual([ + "cacheReadTokens", "createdAt", "databaseId", "errorSummary", @@ -1667,9 +1677,11 @@ test("Description Generation history is newest-first, bounded, and exposes only "finishedAt", "generated", "id", + "inputTokens", "language", "modelId", "nonGeneratable", + "outputTokens", "processed", "scope", "startedAt", @@ -1945,7 +1957,7 @@ test("retains completed batch writes when a later batch response is malformed", }); const { run } = await waitForTerminalRun(app, start.json().id); - expect(modelCompleter.complete).toHaveBeenCalledTimes(2); + expect(modelCompleter.complete).toHaveBeenCalledTimes(3); expect(run).toMatchObject({ status: "completed_with_errors", total: 11, @@ -1972,9 +1984,14 @@ test("retains completed batch writes when a later batch response is malformed", expect(events.slice(2, 12).map((event) => event.message)).toEqual( orderedIds.slice(0, 10).map((targetId) => `Generated description for Catalog Column ${targetId}.`), ); + expect(events.find((event) => event.level === "warning" && event.message.includes("Retrying batch"))).toEqual( + expect.objectContaining({ + message: "The model response did not match the required schema. Retrying batch (attempt 2 of 2).", + }), + ); expect(events.find((event) => event.level === "error")).toEqual(expect.objectContaining({ level: "error", - message: `The model response was invalid. Affected Catalog Column target: ${orderedIds[10]}.`, + message: `The model response did not match the required schema. Affected Catalog Column target: ${orderedIds[10]}.`, })); } finally { await app.close(); @@ -2186,7 +2203,7 @@ test("Generate Missing skips prior partial results and includes null, empty, and const metadata = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n")); if (mode === "partial") { partialCall += 1; - if (partialCall === 2) throw new ModelCompletionProviderError(); + if (partialCall === 2 || partialCall === 3) throw new ModelCompletionProviderError(); return JSON.stringify({ results: metadata.targets.map((target: { targetId: string }) => ({ targetId: target.targetId, @@ -2512,6 +2529,58 @@ test("localizes valid non-generatable Catalog Column results in English", async } }); +test("retries invalid JSON once and completes the batch when the second response is valid", async () => { + const modelCompleter: ModelCompleter = { + complete: vi.fn(async (request) => { + if (vi.mocked(modelCompleter.complete).mock.calls.length === 1) { + return { content: "not-json", usage: { input: 11, cacheRead: 3, output: 2 } }; + } + const context = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n")); + return { + content: JSON.stringify({ + results: context.targets.map((target: { targetId: string }) => ({ + targetId: target.targetId, + outcome: "generated", + description: "Generated after the application retry.", + })), + }), + usage: { input: 7, cacheRead: 1, output: 5 }, + }; + }), + }; + const { app, repository, database, table, column } = await setup(modelCompleter); + try { + const start = await app.inject({ + method: "POST", + url: `/catalog/databases/${database.id}/description-generation-runs`, + payload: { modelId: configuredModel.id, scope: "selected_columns", targetIds: [column.id] }, + }); + const { run } = await waitForTerminalRun(app, start.json().id); + + expect(run).toMatchObject({ + status: "completed", + processed: 1, + generated: 1, + failed: 0, + inputTokens: 18, + cacheReadTokens: 4, + outputTokens: 7, + }); + expect(modelCompleter.complete).toHaveBeenCalledTimes(2); + expect(await repository.getColumn(database.id, table.id, column.id)).toMatchObject({ + generatedDescription: "Generated after the application retry.", + }); + expect(await repository.listDescriptionGenerationEvents(run.id)).toEqual(expect.arrayContaining([ + expect.objectContaining({ + level: "warning", + message: "The model response was not valid JSON. Retrying batch (attempt 2 of 2).", + }), + ])); + } finally { + await app.close(); + } +}); + test("fails safely when the provider fails and redacts provider diagnostics", async () => { const sensitiveDiagnostic = "test-provider-secret private prompt raw provider payload"; const modelCompleter: ModelCompleter = { @@ -2528,6 +2597,7 @@ test("fails safely when the provider fails and redacts provider diagnostics", as }); const { run } = await waitForTerminalRun(app, start.json().id); + expect(modelCompleter.complete).toHaveBeenCalledTimes(2); expect(run).toMatchObject({ status: "completed_with_errors", processed: 1, @@ -2565,7 +2635,7 @@ test.each([ ["duplicate mappings", (targetIds: readonly string[]) => ({ results: [ { targetId: targetIds[0], outcome: "generated", description: "First valid value" }, { targetId: targetIds[0], outcome: "non_generatable" }, - ] })], + ] }), "The model response was missing one or more requested targets."], ["unknown mappings", (targetIds: readonly string[]) => ({ results: [ { targetId: targetIds[0], outcome: "non_generatable" }, { @@ -2573,15 +2643,15 @@ test.each([ outcome: "generated", description: "Unknown target value", }, - ] })], + ] }), "The model response was missing one or more requested targets."], ["missing mappings", (targetIds: readonly string[]) => ({ results: [ { targetId: targetIds[0], outcome: "generated", description: "Only one result" }, - ] })], + ] }), "The model response was missing one or more requested targets."], ["malformed mappings", (targetIds: readonly string[]) => ({ results: [ { targetId: targetIds[0], outcome: "generated", description: "First valid value" }, { targetId: targetIds[1], outcome: "generated", description: " " }, - ] })], -] as const)("rejects %s without applying any result from the batch", async (_name, responseFor) => { + ] }), "The model response did not match the required schema."], +] as const)("rejects %s without applying any result from the batch", async (_name, responseFor, failureMessage) => { let selectedColumnIds: string[] = []; const modelCompleter: ModelCompleter = { complete: vi.fn(async () => JSON.stringify(responseFor(selectedColumnIds))), @@ -2625,6 +2695,7 @@ test.each([ }); const { run } = await waitForTerminalRun(app, start.json().id); + expect(modelCompleter.complete).toHaveBeenCalledTimes(2); expect(run).toMatchObject({ status: "completed_with_errors", processed: 2, @@ -2644,7 +2715,7 @@ test.each([ expect((await repository.listDescriptionGenerationEvents(run.id)).find((event) => event.level === "error")).toEqual( expect.objectContaining({ level: "error", - message: `The model response was invalid. Affected Catalog Column targets: ${selectedColumnIds.join(", ")}.`, + message: `${failureMessage} Affected Catalog Column targets: ${selectedColumnIds.join(", ")}.`, }), ); } finally { diff --git a/backend/test/catalog-description-generation.integration.test.ts b/backend/test/catalog-description-generation.integration.test.ts index 830796e3..b3ed2ecd 100644 --- a/backend/test/catalog-description-generation.integration.test.ts +++ b/backend/test/catalog-description-generation.integration.test.ts @@ -13,6 +13,7 @@ import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js"; import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js"; import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js"; +import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js"; import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js"; import { loadConfig } from "../src/config.js"; import type { WorkspaceRegistry } from "../src/workspaces/registry.js"; @@ -48,6 +49,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a await upSensitiveDataFlag(db); await upDescriptionGeneration(db); await upSensitiveSuggestionRuns(db); + await upAiTokenUsage(db); const repository = new KyselyCatalogRepository(db); const database = await repository.create({ workspaceId: "psd-clinical", @@ -124,7 +126,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a description: "Elenco dei pazienti e dei loro dati clinici.", }] }); } - if (call === 4) { + if (call === 5) { return JSON.stringify({ results: [ { targetId: birthDate.id, @@ -138,14 +140,14 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a }, ] }); } - if (call === 5) { + if (call === 6) { return JSON.stringify({ results: [{ targetId: table.id, outcome: "generated", description: "Descrizione rigenerata della tabella pazienti.", }] }); } - if (call === 6) { + if (call === 7) { return JSON.stringify({ results: [{ targetId: birthDate.id, outcome: "generated", @@ -340,7 +342,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a expect(await repository.getColumn(database.id, table.id, birthDate.id)).toMatchObject({ generatedDescription: "Descrizione recuperata della data di nascita.", }); - expect(modelCompleter.complete).toHaveBeenCalledTimes(6); + expect(modelCompleter.complete).toHaveBeenCalledTimes(7); expect(JSON.stringify(vi.mocked(modelCompleter.complete).mock.calls)).toContain(persistedSampleSecret); const runIds = [ diff --git a/backend/test/catalog-description-source-sampler.test.ts b/backend/test/catalog-description-source-sampler.test.ts index 07434e48..38fb0722 100644 --- a/backend/test/catalog-description-source-sampler.test.ts +++ b/backend/test/catalog-description-source-sampler.test.ts @@ -1,6 +1,9 @@ import { expect, test, vi } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { - PostgresDescriptionSourceSampler, + ConcreteDescriptionSourceSampler, type DescriptionSourceSamplingTarget, } from "../src/catalog/description-source-sampler.js"; import type { @@ -8,6 +11,8 @@ import type { CatalogPostgresAccess, } from "../src/catalog/postgres-access.js"; import type { WorkspaceDatabase } from "../src/catalog/types.js"; +import type { WorkspaceSecretStore } from "../src/workspaces/secret-store.js"; +import { CATALOG_SECRET_IDS } from "../src/catalog/secrets.js"; const database: WorkspaceDatabase = { id: "11111111-1111-4111-8111-111111111111", @@ -51,7 +56,7 @@ test("samples at most five source rows and five distinct non-null examples in a const access: CatalogPostgresAccess = { connect: vi.fn(async () => ({ query, end }) as CatalogDatabaseClient), }; - const sampler = new PostgresDescriptionSourceSampler(access); + const sampler = new ConcreteDescriptionSourceSampler(access); const controller = new AbortController(); const samples = await sampler.sample(database, [target], controller.signal); @@ -92,13 +97,71 @@ test("samples at most five source rows and five distinct non-null examples in a expect(end).toHaveBeenCalledOnce(); }); +test("samples source rows through the configured REST run_query binding", async () => { + const root = mkdtempSync(join(tmpdir(), "tht-source-rest-")); + const credentialFile = join(root, "api-key"); + writeFileSync(credentialFile, "test-api-key\n", { mode: 0o600 }); + const release = vi.fn(); + const secretStore = { + materialize: vi.fn(() => ({ + files: new Map([[CATALOG_SECRET_IDS.apiKey, credentialFile]]), + release, + })), + } as unknown as WorkspaceSecretStore; + const fetchMock = vi.fn(async () => new Response(JSON.stringify([ + { 'status"code': "active", ward: null }, + { 'status"code': "pending", ward: "A" }, + ]), { status: 200, headers: { "content-type": "application/json" } })); + vi.stubGlobal("fetch", fetchMock); + const access: CatalogPostgresAccess = { + connect: vi.fn(async () => { throw new Error("PostgreSQL access must not be used"); }), + }; + const sampler = new ConcreteDescriptionSourceSampler(access, secretStore); + const restDatabase: WorkspaceDatabase = { + ...database, + binding: { + transport: "rest_api", + baseUrl: "https://dwh.example.test/root/", + restPath: "/health", + restAuth: "x-api-key", + }, + }; + + try { + await expect(sampler.sample(restDatabase, [target], new AbortController().signal)).resolves.toEqual([{ + targetId: target.targetId, + tableName: target.tableName, + rows: [ + { fields: [{ name: 'status"code', value: "active" }, { name: "ward", value: null }] }, + { fields: [{ name: 'status"code', value: "pending" }, { name: "ward", value: "A" }] }, + ], + representativeValues: [ + { column: 'status"code', values: ["active", "pending"] }, + { column: "ward", values: ["A"] }, + ], + }]); + expect(access.connect).not.toHaveBeenCalled(); + expect(fetchMock).toHaveBeenCalledWith("https://dwh.example.test/root/rpc/run_query", expect.objectContaining({ + method: "POST", + headers: { "content-type": "application/json", "x-api-key": "test-api-key" }, + body: JSON.stringify({ + query_text: 'SELECT LEFT(("status""code")::text, 256) AS "status""code", LEFT(("ward")::text, 256) AS "ward" FROM "clinical""data"."patient""facts" LIMIT 5', + }), + })); + expect(release).toHaveBeenCalledOnce(); + } finally { + vi.unstubAllGlobals(); + rmSync(root, { recursive: true, force: true }); + } +}); + test("does not issue a SELECT when a protected target has no source columns", async () => { const query = vi.fn(async () => ({ rows: [] })); const end = vi.fn(async () => undefined); const access: CatalogPostgresAccess = { connect: vi.fn(async () => ({ query, end }) as CatalogDatabaseClient), }; - const sampler = new PostgresDescriptionSourceSampler(access); + const sampler = new ConcreteDescriptionSourceSampler(access); const samples = await sampler.sample(database, [{ targetId: target.targetId, @@ -129,7 +192,7 @@ test("rolls back and closes the source connection when sampling fails", async () const access: CatalogPostgresAccess = { connect: vi.fn(async () => ({ query, end }) as CatalogDatabaseClient), }; - const sampler = new PostgresDescriptionSourceSampler(access); + const sampler = new ConcreteDescriptionSourceSampler(access); const controller = new AbortController(); await expect(sampler.sample(database, [target], controller.signal)).rejects.toThrow(); diff --git a/backend/test/catalog-repository.integration.test.ts b/backend/test/catalog-repository.integration.test.ts index 9879cd0c..2cc9d3cb 100644 --- a/backend/test/catalog-repository.integration.test.ts +++ b/backend/test/catalog-repository.integration.test.ts @@ -13,6 +13,7 @@ import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_des import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js"; import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js"; import { up as upLogicalRelationships } from "../src/catalog/migrations/008_catalog_logical_relationships.js"; +import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js"; const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0; @@ -28,6 +29,9 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo await upSchemaSync(db); await upSensitiveDataFlag(db); await upLogicalRelationships(db); + await upDescriptionGeneration(db); + await upSensitiveSuggestionRuns(db); + await upAiTokenUsage(db); await sql`CREATE ROLE thothii_catalog_runtime`.execute(db); await upRuntimeSequencePrivileges(db); const sequencePrivilege = await sql<{ allowed: boolean }>` @@ -386,6 +390,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se await upLogicalRelationships(db); await upDescriptionGeneration(db); await upSensitiveSuggestionRuns(db); + await upAiTokenUsage(db); const repository = new KyselyCatalogRepository(db); const firstDatabase = await repository.create({ workspaceId: "generation-one", diff --git a/backend/test/metadata-generation-models.test.ts b/backend/test/metadata-generation-models.test.ts index cf0dbaea..ef6e4bcd 100644 --- a/backend/test/metadata-generation-models.test.ts +++ b/backend/test/metadata-generation-models.test.ts @@ -127,7 +127,7 @@ test("resolves only a configured selection for the later generation boundary", ( expect(() => models.resolve("unknown-model")).toThrow(MetadataGenerationModelUnavailableError); }); -test("loads DeepSeek, GLM, and an explicit keyless Qwen endpoint from installation setup", () => { +test("loads DeepSeek models, GLM, and an explicit keyless Qwen endpoint from installation setup", () => { const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration: default: glm-53 models: @@ -135,6 +135,10 @@ test("loads DeepSeek, GLM, and an explicit keyless Qwen endpoint from installati label: DeepSeek V4 Pro litellm: {provider: deepseek, model: deepseek-v4-pro} apiKeyEnv: DEEPSEEK_API_KEY + - id: deepseek-v4-flash + label: DeepSeek V4 Flash + litellm: {provider: deepseek, model: deepseek-v4-flash} + apiKeyEnv: DEEPSEEK_API_KEY - id: glm-53 label: GLM 5.3 litellm: @@ -156,6 +160,7 @@ test("loads DeepSeek, GLM, and an explicit keyless Qwen endpoint from installati expect(models.catalog()).toEqual({ models: [ { id: "deepseek-v4-pro", label: "DeepSeek V4 Pro" }, + { id: "deepseek-v4-flash", label: "DeepSeek V4 Flash" }, { id: "glm-53", label: "GLM 5.3" }, { id: "qwen-36", label: "Qwen 3.6" }, ], diff --git a/backend/test/model-completer.test.ts b/backend/test/model-completer.test.ts index 41bd8f50..58ef9a7d 100644 --- a/backend/test/model-completer.test.ts +++ b/backend/test/model-completer.test.ts @@ -64,7 +64,7 @@ sys.stdout.write(json.dumps({"ok": True, "content": "Descrizione italiana"})) signal: new AbortController().signal, }); - expect(content).toBe("Descrizione italiana"); + expect(content).toEqual({ content: "Descrizione italiana", usage: { input: 0, cacheRead: 0, output: 0 } }); const captured = JSON.parse(readFileSync(join(roots[0]!, "request.json"), "utf8")); expect(captured.request).toEqual({ model: "openai/gpt-4.1-mini", @@ -100,7 +100,7 @@ sys.stdout.write(json.dumps({"ok": True, "content": "Descrizione Qwen"})) }, messages: [{ role: "user", content: "Describe invented metadata." }], signal: new AbortController().signal, - })).resolves.toBe("Descrizione Qwen"); + })).resolves.toEqual({ content: "Descrizione Qwen", usage: { input: 0, cacheRead: 0, output: 0 } }); expect(JSON.parse(readFileSync(join(roots[0]!, "request.json"), "utf8"))).toEqual({ model: "openai/qwen3.6-35b-a3b", diff --git a/deploy/psd/thothii-installation.yaml.example b/deploy/psd/thothii-installation.yaml.example index adc92433..4025732f 100644 --- a/deploy/psd/thothii-installation.yaml.example +++ b/deploy/psd/thothii-installation.yaml.example @@ -17,6 +17,12 @@ metadataGeneration: provider: deepseek model: deepseek-v4-pro apiKeyEnv: DEEPSEEK_API_KEY + - id: deepseek-v4-flash + label: DeepSeek V4 Flash + litellm: + provider: deepseek + model: deepseek-v4-flash + apiKeyEnv: DEEPSEEK_API_KEY - id: glm-53 label: GLM 5.3 litellm: diff --git a/docs/general/pi-configuration.md b/docs/general/pi-configuration.md index 7bbc40a9..e2ba854f 100644 --- a/docs/general/pi-configuration.md +++ b/docs/general/pi-configuration.md @@ -108,6 +108,7 @@ not supported by this installation until the declarative contract is extended ge | Model | Level | Why | Visibility | |---|---|---|---| | `deepseek/deepseek-v4-pro` | Built-in Pi | Known public API, already in the build | All projects | +| `deepseek/deepseek-v4-flash` | Built-in Pi | Known public API, already in the build | All projects | | `zai/glm-5.3` | `deploy/pi/models.json` | Custom OpenAI-compatible endpoint | ThothII installation | | `local-qwen/qwen3.6-35b-a3b` | `deploy/pi/models.json` | Locally configured OpenAI-compatible endpoint | ThothII installation | diff --git a/docs/plans/2026-08-26-metadata-catalog-from-thothai.md b/docs/plans/2026-08-26-metadata-catalog-from-thothai.md index 6474313a..b3e37674 100644 --- a/docs/plans/2026-08-26-metadata-catalog-from-thothai.md +++ b/docs/plans/2026-08-26-metadata-catalog-from-thothai.md @@ -179,12 +179,12 @@ canonico; i percorsi e i comportamenti descrivono il sorgente disponibile il 202 e Schema Sync. Il renderer e il runtime delle sessioni NL→SQL restano fuori scope e continuano a rifiutarla finché non verrà deciso il relativo cutover. 55. I menu di azione a livello Workspace Database espongono separatamente `Synchronize tables`, - `Synchronize all columns`, `Synchronize relationships` e `Synchronize all`. Su una selezione di + `Synchronize relationships` e `Synchronize all`. Su una selezione di database lo scope scelto viene avviato per ogni database idoneo; non viene sostituito implicitamente con una sincronizzazione completa. -56. Per lo scope Columns, `tableIds` vuoto significa tutte le Catalog Table correnti del Workspace - Database; `tableIds` valorizzato limita invece la riconciliazione alle tabelle indicate. La grid - Tables espone `Synchronize columns` sulle tabelle selezionate. +56. Lo scope Columns è disponibile dalla grid Tables e limita la riconciliazione alle tabelle + selezionate; la pagina Columns non espone azioni di sincronizzazione. La grid Tables espone + `Synchronize columns` sulle tabelle selezionate. ## Correzione del modello mentale corrente @@ -570,9 +570,9 @@ Le griglie che dispongono di azioni massive usano checkbox e una toolbar contest menu `Actions` e cancellazione della selezione. La selezione identifica ID espliciti, può essere accumulata attraverso i filtri e viene azzerata dopo successo, nuova sincronizzazione o uscita dalla pagina; un'azione è all-or-nothing se un elemento non è idoneo. I menu a livello database -espongono gli scope fisici come azioni distinte: `Synchronize tables`, `Synchronize all columns`, -`Synchronize relationships` e `Synchronize all`. La grid Tables espone invece `Synchronize -columns` per le tabelle selezionate. Le selezioni database aggiungono `Delete all tables` e +espongono gli scope fisici come azioni distinte: `Synchronize tables`, `Synchronize relationships` +e `Synchronize all`. La grid Tables espone invece `Synchronize columns` per le tabelle selezionate; +la pagina Columns non espone sincronizzazione. Le selezioni database aggiungono `Delete all tables` e `Delete all relationships`; le selezioni tabelle aggiungono `Delete all columns` e `Delete all relationships`. Queste operazioni sono atomiche, richiedono conferma e non modificano database esterno, binding, configurazione o segreti. Test connection resta un'azione distinta; griglie senza diff --git a/docs/testing/2026-08-29-ai-catalog-description-generation-acceptance.md b/docs/testing/2026-08-29-ai-catalog-description-generation-acceptance.md index 102a7235..5c664830 100644 --- a/docs/testing/2026-08-29-ai-catalog-description-generation-acceptance.md +++ b/docs/testing/2026-08-29-ai-catalog-description-generation-acceptance.md @@ -17,6 +17,7 @@ The installation-level model catalog contained these entries, with `glm-53` as t | Selection ID | LiteLLM route | Authentication | | --- | --- | --- | | `deepseek-v4-pro` | `deepseek/deepseek-v4-pro` | Protected `DEEPSEEK_API_KEY` value already used by core | +| `deepseek-v4-flash` | `deepseek/deepseek-v4-flash` | Protected `DEEPSEEK_API_KEY` value already used by core | | `glm-53` | `openai/glm-5.3` on the Z.AI coding endpoint | Protected `ZAI_API_KEY` value already used by core | | `qwen-36` | `openai/qwen3.6-35b-a3b` on the VPN-only AritmoLab endpoint | No operator API key | diff --git a/frontend/e2e/database-management-layout.spec.ts b/frontend/e2e/database-management-layout.spec.ts index 95df0ab4..5e24cb0e 100644 --- a/frontend/e2e/database-management-layout.spec.ts +++ b/frontend/e2e/database-management-layout.spec.ts @@ -72,6 +72,28 @@ const table = { updatedAt: "2026-08-27T09:00:00Z", }; +const descriptionRun = { + id: "77777777-7777-4777-8777-777777777777", + databaseId, + scope: "selected_columns", + modelId: "glm-53", + language: "it", + status: "completed", + total: 5, + processed: 5, + generated: 4, + nonGeneratable: 1, + failed: 0, + inputTokens: 1200, + cacheReadTokens: 80, + outputTokens: 420, + createdAt: "2026-08-31T09:00:00Z", + startedAt: "2026-08-31T09:00:01Z", + updatedAt: "2026-08-31T09:00:04Z", + finishedAt: "2026-08-31T09:00:04Z", + errorSummary: null, +}; + function metrics(requestedDatabaseId: string | null) { return { scope: requestedDatabaseId ? "database" : "global", @@ -133,7 +155,8 @@ async function expectContextPanelGeometry(page: Page, accessibleName: string) { const managerCenter = managerBox.x + managerBox.width / 2; const panelCenter = panelBox.x + panelBox.width / 2; expect(Math.abs(panelCenter - managerCenter)).toBeLessThanOrEqual(1); - expect(Math.abs(panelBox.width / managerBox.width - 0.6)).toBeLessThanOrEqual(0.005); + const expectedPanelWidth = Math.min(1200, managerBox.width - (managerBox.width <= 768 ? 24 : 32)); + expect(Math.abs(panelBox.width - expectedPanelWidth)).toBeLessThanOrEqual(1); expect(panelBox.x).toBeGreaterThanOrEqual(managerBox.x - 1); expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(managerBox.x + managerBox.width + 1); expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(sessionRailBox.x + 1); @@ -297,7 +320,8 @@ async function expectWorkAreaPanelGeometry(page: Page, accessibleName: string) { expect(Math.abs( panelBox.x + panelBox.width / 2 - (workAreaBox.x + workAreaBox.width / 2), )).toBeLessThanOrEqual(1); - expect(Math.abs(panelBox.width / workAreaBox.width - 0.6)).toBeLessThanOrEqual(0.005); + const expectedPanelWidth = Math.min(1200, workAreaBox.width - (workAreaBox.width <= 768 ? 24 : 32)); + expect(Math.abs(panelBox.width - expectedPanelWidth)).toBeLessThanOrEqual(1); expect(panelBox.x).toBeGreaterThanOrEqual(workAreaBox.x - 1); expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(workAreaBox.x + workAreaBox.width + 1); expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(sessionRailBox.x + 1); @@ -344,12 +368,20 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb "GET /api/catalog/metadata-generation/models": { models: [ { id: "deepseek-v4-pro", label: "DeepSeek V4 Pro" }, + { id: "deepseek-v4-flash", label: "DeepSeek V4 Flash" }, { id: "glm-53", label: "GLM 5.3" }, { id: "qwen-36", label: "AritmoLab Qwen 3.6 35B A3B" }, ], default: "glm-53", }, - "GET /api/catalog/description-generation-runs?limit=50": [], + "GET /api/catalog/description-generation-runs?limit=50": [descriptionRun], + [`GET /api/catalog/description-generation-runs/${descriptionRun.id}`]: descriptionRun, + [`GET /api/catalog/description-generation-runs/${descriptionRun.id}/events-list?after=0`]: [{ + sequence: 1, + level: "info", + message: "Description generation completed.", + createdAt: "2026-08-31T09:00:04Z", + }], "GET /api/catalog/sensitive-data-suggestion-runs?limit=50": [], [`GET /api/catalog/databases/${databaseId}/tables`]: [table], }; @@ -387,6 +419,7 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb const adminNavigationRail = page.getByRole("complementary", { name: "Session navigation" }); const administration = adminNavigationRail.getByRole("button", { name: "Administration", exact: true }); + await expect(administration).toHaveCSS("font-family", /Manrope/); await expect(administration).toHaveAttribute("aria-expanded", "false"); await expect(adminNavigationRail.getByRole("region", { name: "Administration" })).toHaveCount(0); await administration.click(); @@ -435,6 +468,7 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb await expect(metadataModelSelector).toHaveValue("glm-53"); await expect(metadataModelSelector.locator("option")).toHaveText([ "DeepSeek V4 Pro", + "DeepSeek V4 Flash", "GLM 5.3", "AritmoLab Qwen 3.6 35B A3B", ]); @@ -488,10 +522,55 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb .click(); await expectContextPanelGeometry(page, "Description generation"); await expectContextPanelHeaderUsesPrimary(page, "Description generation"); - await page - .getByRole("dialog", { name: "Description generation" }) - .getByRole("button", { name: /close/i }) - .click(); + const descriptionPanel = page.getByRole("dialog", { name: "Description generation" }); + const descriptionBody = descriptionPanel.locator( + ':scope > [data-catalog-panel-region="body"]', + ); + const progressTable = descriptionPanel.getByRole("table", { + name: "Description generation progress", + }); + const eventLog = descriptionPanel.getByRole("log", { + name: "Description generation events", + }); + const recentRuns = descriptionPanel.getByRole("region", { + name: "Description generation history", + }); + await expect(progressTable.getByRole("columnheader")).toHaveCount(5); + await expect(eventLog).toBeVisible(); + await expect(recentRuns).toBeVisible(); + + const [bodyBox, eventLogBox, recentRunsBox] = await Promise.all([ + descriptionBody.boundingBox(), + eventLog.boundingBox(), + recentRuns.boundingBox(), + ]); + expect(bodyBox).not.toBeNull(); + expect(eventLogBox).not.toBeNull(); + expect(recentRunsBox).not.toBeNull(); + if (bodyBox && eventLogBox && recentRunsBox) { + const eventBottomGutter = bodyBox.y + bodyBox.height - eventLogBox.y - eventLogBox.height; + const recentBottomGutter = bodyBox.y + bodyBox.height - recentRunsBox.y - recentRunsBox.height; + expect(eventBottomGutter).toBeGreaterThanOrEqual(10); + expect(recentBottomGutter).toBeGreaterThanOrEqual(10); + expect(eventLogBox.x + eventLogBox.width).toBeLessThan(recentRunsBox.x); + } + + await page.setViewportSize({ width: 720, height: 800 }); + const [compactManagerBox, compactDescriptionPanelBox] = await Promise.all([ + page.getByTestId("conversation-column").getByRole("main", { + name: "Database management", + }).boundingBox(), + descriptionPanel.boundingBox(), + ]); + expect(compactManagerBox).not.toBeNull(); + expect(compactDescriptionPanelBox).not.toBeNull(); + if (compactManagerBox && compactDescriptionPanelBox) { + expect(compactManagerBox.width - compactDescriptionPanelBox.width).toBeGreaterThanOrEqual(20); + } + expect(await descriptionBody.evaluate((element) => getComputedStyle(element).display)).not.toBe("grid"); + + await page.setViewportSize({ width: 1910, height: 911 }); + await descriptionPanel.getByRole("button", { name: /close/i }).click(); const applicationBar = page.locator( 'main[aria-label="Database management"] .thot-fleet-ledger__header', @@ -734,7 +813,7 @@ test("Workspace and Pi management share the centered work-area panel without cov expect(compactWorkAreaBox).not.toBeNull(); expect(compactPanelBox).not.toBeNull(); if (compactWorkAreaBox && compactPanelBox) { - expect(Math.abs(compactPanelBox.width - compactWorkAreaBox.width + 16)).toBeLessThanOrEqual(1); + expect(Math.abs(compactPanelBox.width - compactWorkAreaBox.width + 24)).toBeLessThanOrEqual(1); expect(Math.abs( compactPanelBox.x + compactPanelBox.width / 2 - (compactWorkAreaBox.x + compactWorkAreaBox.width / 2), diff --git a/frontend/prototypes/database-management/fleet.tsx b/frontend/prototypes/database-management/fleet.tsx index eff902e9..95059b8d 100644 --- a/frontend/prototypes/database-management/fleet.tsx +++ b/frontend/prototypes/database-management/fleet.tsx @@ -269,7 +269,6 @@ export function FleetSelectionActions({ props, direction = "row" }: { props: Var if (state.inventoryLevel === "databases") actions = [ { id: "databases-test", group: "Connection", label: "Test connections", icon: , disabled, onClick: run(`Testing ${count} selected connection${count === 1 ? "" : "s"}`) }, { id: "databases-sync-tables", group: "Synchronization", label: "Synchronize tables", icon: , disabled, onClick: run("Table synchronization started", "running") }, - { id: "databases-sync-columns", group: "Synchronization", label: "Synchronize all columns", icon: , disabled, onClick: run("Column synchronization started", "running") }, { id: "databases-sync-relationships", group: "Synchronization", label: "Synchronize relationships", icon: , disabled, onClick: run("Relationship synchronization started", "running") }, { id: "databases-sync-all", group: "Synchronization", label: "Synchronize all", icon: , disabled, onClick: run("Full schema synchronization started", "running") }, { id: "databases-generate-all", group: "Descriptions", label: "Generate all descriptions", icon: , disabled, onClick: run("Generate all confirmation opened") }, diff --git a/frontend/src/api/catalog-databases.ts b/frontend/src/api/catalog-databases.ts index 6bac5f5c..f9016e02 100644 --- a/frontend/src/api/catalog-databases.ts +++ b/frontend/src/api/catalog-databases.ts @@ -157,6 +157,9 @@ export interface SensitiveDataSuggestionRun { total: number; suggestedSensitive: number; suggestedNonSensitive: number; + inputTokens?: number; + cacheReadTokens?: number; + outputTokens?: number; createdAt: string; startedAt: string | null; updatedAt: string; @@ -297,6 +300,9 @@ export interface DescriptionGenerationRun { generated: number; nonGeneratable: number; failed: number; + inputTokens?: number; + cacheReadTokens?: number; + outputTokens?: number; createdAt: string; startedAt: string | null; updatedAt: string; diff --git a/frontend/src/components/ui/button.tsx b/frontend/src/components/ui/button.tsx index 5894f7c8..67f606aa 100644 --- a/frontend/src/components/ui/button.tsx +++ b/frontend/src/components/ui/button.tsx @@ -4,7 +4,7 @@ import { cva, type VariantProps } from "class-variance-authority" import { cn } from "@/lib/utils" const buttonVariants = cva( - "group/button inline-flex shrink-0 items-center justify-center rounded-md border border-transparent bg-clip-padding text-sm font-semibold tracking-[0.005em] whitespace-nowrap transition-all outline-none select-none focus-visible:ring-3 focus-visible:ring-ring/25 disabled:pointer-events-none disabled:opacity-50 aria-invalid:border-destructive aria-invalid:ring-3 aria-invalid:ring-destructive/20 dark:aria-invalid:border-destructive/50 dark:aria-invalid:ring-destructive/40 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4", + "group/button inline-flex shrink-0 items-center justify-center rounded-md border border-transparent bg-clip-padding font-sans text-sm font-semibold tracking-[0.005em] whitespace-nowrap transition-all outline-none select-none focus-visible:ring-3 focus-visible:ring-ring/25 disabled:pointer-events-none disabled:opacity-50 aria-invalid:border-destructive aria-invalid:ring-3 aria-invalid:ring-destructive/20 dark:aria-invalid:border-destructive/50 dark:aria-invalid:ring-destructive/40 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4", { variants: { variant: { diff --git a/frontend/src/shell/AppShell.auth.test.tsx b/frontend/src/shell/AppShell.auth.test.tsx index 650de459..330e33a8 100644 --- a/frontend/src/shell/AppShell.auth.test.tsx +++ b/frontend/src/shell/AppShell.auth.test.tsx @@ -67,6 +67,7 @@ describe("authenticated shell permissions", () => { const sessionNavigation = screen.getByRole("complementary", { name: "Session navigation" }); const trigger = await within(sessionNavigation).findByRole("button", { name: "Administration" }); + expect(trigger).toHaveClass("font-sans"); expect(trigger).toHaveAttribute("aria-expanded", "false"); expect(within(sessionNavigation).queryByRole("region", { name: "Administration" })).not.toBeInTheDocument(); diff --git a/frontend/src/shell/AppShell.tsx b/frontend/src/shell/AppShell.tsx index be00b3f4..505ffa9d 100644 --- a/frontend/src/shell/AppShell.tsx +++ b/frontend/src/shell/AppShell.tsx @@ -45,6 +45,23 @@ interface AppShellProps { type ActiveSurface = "core" | "database-management"; type ActiveManagementPanel = "workspace" | "pi" | null; const SESSION_SCOPES: readonly SessionScope[] = ["mine", "all"]; +const APP_NAVIGATION_STORAGE_KEY = "thothii:app-navigation"; + +function readPersistedNavigation(): { surface: ActiveSurface; panel: ActiveManagementPanel } { + if (import.meta.env.MODE === "test") return { surface: "core", panel: null }; + try { + const parsed = JSON.parse(window.localStorage.getItem(APP_NAVIGATION_STORAGE_KEY) ?? "null") as { + surface?: unknown; + panel?: unknown; + } | null; + return { + surface: parsed?.surface === "database-management" ? "database-management" : "core", + panel: parsed?.panel === "workspace" || parsed?.panel === "pi" ? parsed.panel : null, + }; + } catch { + return { surface: "core", panel: null }; + } +} function sessionScopeTabClass(selected: boolean): string { const base = "relative -mb-px flex h-8 w-full cursor-pointer items-center justify-center rounded-t-md border border-b-2 px-2 text-xs font-semibold tracking-[0.005em] outline-none focus-visible:z-10 focus-visible:ring-3 focus-visible:ring-[oklch(var(--nav-active-border)/0.28)]"; @@ -166,7 +183,8 @@ export function AppShell({ canLogout }: AppShellProps) { const queryClient = useQueryClient(); const [showActivity, setShowActivity] = useState(false); - const [activeSurface, setActiveSurface] = useState("core"); + const persistedNavigation = readPersistedNavigation(); + const [activeSurface, setActiveSurface] = useState(persistedNavigation.surface); const databaseManagementPresentation = resolveDatabaseManagementPresentation({ isDevelopment: import.meta.env.DEV, mode: import.meta.env.MODE, @@ -177,7 +195,7 @@ export function AppShell({ canLogout }: AppShellProps) { const updateDatabaseNavigationState = useCallback((state: { dirty: boolean; busy: boolean }) => { databaseNavigationRef.current = state; }, []); - const [activeManagementPanel, setActiveManagementPanel] = useState(null); + const [activeManagementPanel, setActiveManagementPanel] = useState(persistedNavigation.panel); const [adminNavigationValue, setAdminNavigationValue] = useState([]); const [activeOpen, setActiveOpen] = useState(true); const [archiveOpen, setArchiveOpen] = useState(false); @@ -188,6 +206,10 @@ export function AppShell({ canLogout }: AppShellProps) { const [collapsedGroups, setCollapsedGroups] = useState>({}); const [renameGroupTarget, setRenameGroupTarget] = useState(null); const operationEpochRef = useRef(0); + useEffect(() => { + if (import.meta.env.MODE === "test") return; + window.localStorage.setItem(APP_NAVIGATION_STORAGE_KEY, JSON.stringify({ surface: activeSurface, panel: activeManagementPanel })); + }, [activeManagementPanel, activeSurface]); useEffect(() => () => { operationEpochRef.current += 1; }, []); const groups = useMemo( diff --git a/frontend/src/shell/DatabaseManagementPage.test.tsx b/frontend/src/shell/DatabaseManagementPage.test.tsx index a80013c0..172c9c76 100644 --- a/frontend/src/shell/DatabaseManagementPage.test.tsx +++ b/frontend/src/shell/DatabaseManagementPage.test.tsx @@ -197,7 +197,6 @@ function registerCompletedSyncRun(run: CatalogSyncRun) { const synchronizationScopes = [ { scope: "tables", label: "Synchronize tables" }, - { scope: "columns", label: "Synchronize all columns" }, { scope: "relationships", label: "Synchronize relationships" }, { scope: "all", label: "Synchronize all" }, ] as const; @@ -234,9 +233,7 @@ test("renders Fleet Ledger with real metrics, conceptual row tooltips, and persi expect(screen.queryByText("Inspect physical metadata, curate descriptions and control catalog operations.")).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Back to workspace" })).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: /Add database/i })).not.toBeInTheDocument(); - expect(screen.getByRole("columnheader", { name: /Revision \/ Evidence/ })).toBeVisible(); - expect(screen.getByRole("columnheader", { name: /NL→SQL runtime/ })).toBeVisible(); - expect(screen.getByRole("columnheader", { name: /Metadata Catalog/ })).toBeVisible(); + expect(screen.getByRole("columnheader", { name: /Catalog status/ })).toBeVisible(); const summary = screen.getByRole("region", { name: "Fleet summary" }); expect(await within(summary).findByText("2,275")).toBeVisible(); expect(within(summary).getByText("75%")).toBeVisible(); @@ -261,13 +258,14 @@ test("renders Fleet Ledger with real metrics, conceptual row tooltips, and persi await user.click(within(restoredDatabaseRow).getByRole("checkbox", { name: /toggle row selection/i })); const actionPicker = screen.getByRole("combobox", { name: "Batch action" }); expect(within(actionPicker).getByRole("option", { name: /^Synchronize all(?:,|$)/ })).toBeVisible(); + expect(within(actionPicker).queryByRole("option", { name: "Synchronize all columns" })).not.toBeInTheDocument(); await user.selectOptions(actionPicker, "generate-missing"); expect(screen.getAllByRole("combobox", { name: "Metadata-generation LLM model" })).toHaveLength(1); expect(modelSelector).toHaveValue("local-qwen"); }); -test("shows repository, NL→SQL runtime, and Metadata Catalog states independently", async () => { +test("shows synchronization, description, and sensitive-data states independently", async () => { const configured = makeDatabase({ connectionStatus: "reachable", testedVersion: 3 }); const needsRuntimeConfiguration = makeDatabase({ ...unconfigured, @@ -281,21 +279,58 @@ test("shows repository, NL→SQL runtime, and Metadata Catalog states independen renderPage({ rows: [configured, needsRuntimeConfiguration, orphan], presentation: "fleet" }); const configuredRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); - expect(within(configuredRow).getByText("Active revision aaaaaaa")).toBeVisible(); - expect(within(configuredRow).getByText("Evidence materialized · filesystem")).toBeVisible(); - expect(within(configuredRow).getByText("Ready")).toBeVisible(); - expect(within(configuredRow).getByText("Configured")).toBeVisible(); - expect(within(configuredRow).getByText("Connection reachable")).toBeVisible(); + expect(within(configuredRow).getByText("Not synchronized")).toBeVisible(); + expect(within(configuredRow).getByText(/Descriptions ·/)).toBeVisible(); + expect(within(configuredRow).getByText(/Sensitive ·/)).toBeVisible(); const needsConfigurationRow = screen.getByRole("row", { name: /Research laboratory/ }); - expect(within(needsConfigurationRow).getByText("Evidence credentials required · http")).toBeVisible(); - expect(within(needsConfigurationRow).getByText("Configuration required")).toBeVisible(); - expect(within(needsConfigurationRow).getByText("Not configured")).toBeVisible(); + expect(within(needsConfigurationRow).getByText("Not synchronized")).toBeVisible(); + expect(within(needsConfigurationRow).getByText(/Descriptions ·/)).toBeVisible(); const orphanRow = screen.getByRole("row", { name: /Retired workspace/ }); - expect(within(orphanRow).getByText("Workspace missing")).toBeVisible(); - expect(within(orphanRow).getByText("Unavailable")).toBeVisible(); - expect(within(orphanRow).getByText("Orphaned configuration")).toBeVisible(); + expect(within(orphanRow).getByText("Not synchronized")).toBeVisible(); +}); + +test("shows the configured access type and endpoint in catalog status", async () => { + const direct = makeDatabase(); + const rest = makeDatabase({ + id: "33333333-3333-4333-8333-333333333333", + workspaceId: "rest-lab", + workspaceName: "REST laboratory", + binding: { + transport: "rest_api", + baseUrl: "https://203.0.113.18/dwh/", + restPath: "/health", + restAuth: "x-api-key", + }, + }); + const ssh = makeDatabase({ + id: "44444444-4444-4444-8444-444444444444", + workspaceId: "ssh-lab", + workspaceName: "SSH laboratory", + binding: { + transport: "ssh_tunnel", + sshHost: "bastion.example.test", + sshPort: 2222, + sshUsername: "operator", + sshTargetHost: "postgres.internal", + sshTargetPort: 5432, + username: "reader", + }, + }); + renderPage({ rows: [direct, rest, ssh], presentation: "fleet" }); + + const directRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); + expect(within(directRow).getByText("Direct PostgreSQL")).toBeVisible(); + expect(within(directRow).getByText("db.internal:5432")).toBeVisible(); + + const restRow = screen.getByRole("row", { name: /REST laboratory/ }); + expect(within(restRow).getByText("REST")).toBeVisible(); + expect(within(restRow).getByText("203.0.113.18")).toBeVisible(); + + const sshRow = screen.getByRole("row", { name: /SSH laboratory/ }); + expect(within(sshRow).getByText("SSH")).toBeVisible(); + expect(within(sshRow).getByText("bastion.example.test:2222")).toBeVisible(); }); test("keeps Fleet Ledger visible when a catalog response omits the Evidence projection", async () => { @@ -306,7 +341,7 @@ test("keeps Fleet Ledger visible when a catalog response omits the Evidence proj expect(await screen.findByRole("heading", { name: "Database management" })).toBeVisible(); const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); - expect(within(databaseRow).getByText("Evidence state unavailable")).toBeVisible(); + expect(within(databaseRow).getByText("Not synchronized")).toBeVisible(); }); test("opens the relationship map directly from a Fleet database and restores focus on return", async () => { @@ -359,7 +394,7 @@ test("offers catalog configuration directly on an unconfigured Fleet workspace", expect(within(drawer).getByLabelText("Workspace")).toHaveValue("Research laboratory"); }); -test("keeps only the local back action while browsing Fleet Ledger columns", async () => { +test("keeps the table back action in the Fleet Ledger breadcrumb while browsing columns", async () => { const user = userEvent.setup(); server.use( http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([patientsTable])), @@ -372,9 +407,17 @@ test("keeps only the local back action while browsing Fleet Ledger columns", asy name: "View tables for Policlinico San Donato", })); expect(await screen.findByRole("button", { name: "Back to databases" })).toBeVisible(); + expect(screen.getByRole("combobox", { name: "Table action" })).toBeDisabled(); + expect(screen.queryByRole("button", { name: "Sync history" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Description history" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Sensitive history" })).not.toBeInTheDocument(); await user.click(await screen.findByRole("button", { name: "View columns for patients" })); - expect(await screen.findByRole("button", { name: "Back to tables" })).toBeVisible(); + const tableBackAction = await screen.findByRole("button", { name: "Back to tables" }); + expect(tableBackAction).toBeVisible(); + expect(screen.getByRole("combobox", { name: "Column action" })).toBeDisabled(); + expect(screen.getByRole("navigation", { name: "Database hierarchy" }).parentElement) + .toContainElement(tableBackAction); await waitFor(() => { expect(screen.queryByRole("button", { name: "Back to databases" })).not.toBeInTheDocument(); }); @@ -427,7 +470,7 @@ test("reopens database synchronization history when no run is active", async () renderPage({ rows: [makeDatabase()], presentation: "fleet" }); const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); - await user.click(within(databaseRow).getByRole("button", { name: "View synchronization history for Policlinico San Donato" })); + await user.click(within(databaseRow).getByText("Policlinico San Donato")); const drawer = await screen.findByRole("dialog", { name: "Schema synchronization" }); expect(drawer).toBeVisible(); @@ -628,6 +671,7 @@ test.each(synchronizationScopes)( await user.click(screen.getByRole("menuitem", { name: label })); await waitFor(() => expect(startBody).toEqual({ version: 3, scope, tableIds: [] })); + expect(screen.getByText("1 selected")).toBeVisible(); }, ); @@ -1001,15 +1045,16 @@ test("selected database Actions confirms and deletes catalog tables for the full ], target: "tables", })); - await waitFor(() => expect(screen.queryByText("2 selected")).not.toBeInTheDocument()); - await waitFor(() => expect(screen.getByRole("textbox", { name: "Search databases" })).toHaveFocus()); + expect(screen.getByText("2 selected")).toBeVisible(); + await user.click(screen.getByRole("button", { name: "Clear" })); + expect(screen.queryByText("2 selected")).not.toBeInTheDocument(); }); -test("presents completed synchronization steps as success and skips unneeded confirmation", async () => { +test("presents completed database synchronization steps as success and skips unneeded confirmation", async () => { const user = userEvent.setup(); const run = { - ...makeSyncRun("columns"), - counts: { tables: 163, columns: 2_275 }, + ...makeSyncRun("all"), + counts: { tables: 163, columns: 2_275, relationships: 18 }, }; registerCompletedSyncRun(run); server.use(http.post("/api/catalog/databases/:databaseId/sync-runs", () => ( @@ -1022,7 +1067,7 @@ test("presents completed synchronization steps as success and skips unneeded con const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); await user.click(within(databaseRow).getByRole("checkbox", { name: /toggle row selection/i })); await user.click(screen.getByRole("button", { name: "Actions" })); - await user.click(await screen.findByRole("menuitem", { name: "Synchronize all columns" })); + await user.click(await screen.findByRole("menuitem", { name: "Synchronize all" })); expect(await screen.findByRole("heading", { name: "Succeeded" })).toBeVisible(); const steps = screen.getByRole("list", { name: "Synchronization steps" }); @@ -1039,7 +1084,7 @@ test("presents completed synchronization steps as success and skips unneeded con expect(steps.querySelectorAll(".text-primary")).toHaveLength(0); }); -test("database Overview exposes every synchronization scope", async () => { +test("database Overview exposes the database synchronization scopes", async () => { const user = userEvent.setup(); let startBody: unknown; const run = makeSyncRun("relationships"); @@ -1520,6 +1565,37 @@ test("starts description generation for multiple selected tables", async () => { expect(screen.getByText("2 selected")).toBeVisible(); }); +test("generates descriptions for every column in selected tables", async () => { + const user = userEvent.setup(); + const queuedRun = makeDescriptionGenerationRun({ scope: "selected_columns", total: 1 }); + let startBody: unknown; + server.use( + http.get("/api/catalog/metadata-generation/models", () => HttpResponse.json({ models: [{ id: "local-qwen", label: "Local Qwen" }], default: "local-qwen" })), + http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([patientsTable])), + http.get(`/api/catalog/databases/:databaseId/tables/${patientsTable.id}/columns`, () => HttpResponse.json([patientIdColumn])), + http.post("/api/catalog/databases/:databaseId/description-generation-runs", async ({ request }) => { + startBody = await request.json(); + return HttpResponse.json(queuedRun, { status: 202 }); + }), + http.get("/api/catalog/description-generation-runs/:runId", () => HttpResponse.json(queuedRun)), + http.get("/api/catalog/description-generation-runs/:runId/events-list", () => HttpResponse.json([])), + ); + renderPage({ rows: [makeDatabase({ connectionStatus: "reachable", testedVersion: 3 })] }); + + await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" })); + await user.click(screen.getByRole("tab", { name: "Tables" })); + await user.click(within(await screen.findByRole("row", { name: /patients/ })).getByRole("checkbox")); + await user.click(screen.getByRole("button", { name: "Actions" })); + await user.click(await screen.findByRole("menuitem", { name: "Generate descriptions for all columns" })); + + await waitFor(() => expect(startBody).toEqual({ + modelId: "local-qwen", + scope: "selected_columns", + targetIds: [patientIdColumn.id], + })); + expect(await screen.findByText("Column description generation started for 1 column")).toBeVisible(); +}); + test("observes an active run from another browser and reopens a terminal run from history", async () => { const user = userEvent.setup(); const activeRun = makeDescriptionGenerationRun({ @@ -1913,6 +1989,12 @@ test("starts one selected column with the configured default model", async () => await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" })); await user.click(screen.getByRole("tab", { name: "Tables" })); await user.click(await screen.findByRole("button", { name: "View columns for patients" })); + expect(screen.queryByRole("button", { name: "Sync columns" })).not.toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: /Select/i })).toBeVisible(); + expect(await screen.findByRole("checkbox", { name: "Sensitive data for id" })).toHaveAttribute( + "title", + "Mark this column as sensitive; this does not select it for actions.", + ); const columnRow = (await screen.findAllByRole("row", { name: /Patient identifier/ })) .find((row) => within(row).queryByRole("checkbox", { name: /toggle row selection/i })); expect(columnRow).toBeDefined(); diff --git a/frontend/src/shell/DatabaseManagementPage.tsx b/frontend/src/shell/DatabaseManagementPage.tsx index 12f7d9b2..a8b82a0a 100644 --- a/frontend/src/shell/DatabaseManagementPage.tsx +++ b/frontend/src/shell/DatabaseManagementPage.tsx @@ -80,6 +80,20 @@ const SYNC_STARTED_MESSAGES: Record = { relationships: "Relationship synchronization started", all: "Full schema synchronization started", }; +const DATABASE_SCREEN_STORAGE_KEY = "thothii:database-management-screen"; + +function readPersistedDatabaseScreen(): DatabaseScreen { + if (import.meta.env.MODE === "test") return { kind: "list" }; + try { + const parsed = JSON.parse(window.localStorage.getItem(DATABASE_SCREEN_STORAGE_KEY) ?? "null") as { kind?: unknown; workspaceId?: unknown } | null; + if ((parsed?.kind === "tables" || parsed?.kind === "relationships") && typeof parsed.workspaceId === "string") { + return { kind: parsed.kind, workspaceId: parsed.workspaceId }; + } + } catch { + // Ignore malformed persisted navigation and return to the safe list view. + } + return { kind: "list" }; +} interface Props { canManage: boolean; @@ -156,12 +170,13 @@ export function DatabaseManagementPage({ (run) => run.status === "running", ); - const [screen, setScreen] = useState({ kind: "list" }); + const [screen, setScreen] = useState(readPersistedDatabaseScreen); const [selectedMetadataModel, setSelectedMetadataModel] = useState(""); const [draft, setDraft] = useState(null); const [baseline, setBaseline] = useState(""); const [formSource, setFormSource] = useState(null); const [search, setSearch] = useState(""); + const [selectedDatabaseId, setSelectedDatabaseId] = useState(null); const [busyAction, setBusyAction] = useState(null); const [stale, setStale] = useState(false); const [staleBannerOpen, setStaleBannerOpen] = useState(true); @@ -171,6 +186,11 @@ export function DatabaseManagementPage({ busy: false, }); const [tableNestedNavigationActive, setTableNestedNavigationActive] = useState(false); + const [tableNestedNavigationBack, setTableNestedNavigationBack] = useState<(() => void) | null>(null); + const handleTableNestedNavigationChange = useCallback((active: boolean, onBack?: () => void) => { + setTableNestedNavigationActive(active); + setTableNestedNavigationBack(() => (active ? onBack ?? null : null)); + }, []); const [activeSyncRun, setActiveSyncRun] = useState(null); const [syncHistoryDatabaseId, setSyncHistoryDatabaseId] = useState(null); const [syncDrawerOpen, setSyncDrawerOpen] = useState(false); @@ -184,6 +204,12 @@ export function DatabaseManagementPage({ suggestions: SensitiveDataSuggestion[]; } | null>(null); + useEffect(() => { + if (import.meta.env.MODE === "test") return; + const persisted = screen.kind === "tables" || screen.kind === "relationships" ? screen : { kind: "list" }; + window.localStorage.setItem(DATABASE_SCREEN_STORAGE_KEY, JSON.stringify(persisted)); + }, [screen]); + const originRef = useRef(null); const searchInputRef = useRef(null); const formHeadingRef = useRef(null); @@ -284,6 +310,7 @@ export function DatabaseManagementPage({ setPartialSecretFailure(null); setTablesNavigationState({ dirty: false, busy: false }); setTableNestedNavigationActive(false); + setTableNestedNavigationBack(null); restoreListFocus(); }, [restoreListFocus]); @@ -346,6 +373,7 @@ export function DatabaseManagementPage({ setPartialSecretFailure(null); setTablesNavigationState({ dirty: false, busy: false }); setTableNestedNavigationActive(false); + setTableNestedNavigationBack(null); setScreen({ kind: "tables", workspaceId: row.workspaceId }); }, []); @@ -358,6 +386,8 @@ export function DatabaseManagementPage({ setStale(false); setPartialSecretFailure(null); setTablesNavigationState({ dirty: false, busy: false }); + setTableNestedNavigationActive(false); + setTableNestedNavigationBack(null); setScreen({ kind: "relationships", workspaceId: row.workspaceId }); }, []); @@ -856,6 +886,11 @@ export function DatabaseManagementPage({ } }, [invalidateCatalogMetrics, queryClient]); + const clearActiveCatalogTables = useCallback(async () => { + if (!activeRow?.id) return; + await deleteSelectedMetadata([activeRow], "tables"); + }, [activeRow, deleteSelectedMetadata]); + const syncDatabase = useCallback(async (scope: CatalogSyncScope) => { if (!activeRow?.id || !activeRow.configured) return; try { @@ -1038,13 +1073,12 @@ export function DatabaseManagementPage({ onOpenOverview={() => openOverview(activeRow)} onOpenRelationships={() => openRelationships(activeRow)} onNavigationStateChange={setTablesNavigationState} - onNestedNavigationChange={setTableNestedNavigationActive} + onNestedNavigationChange={handleTableNestedNavigationChange} onRunStarted={rememberSyncRun} onOpenSync={() => openSync(activeRow)} - onOpenDescriptionHistory={openDescriptionGenerationHistory} - onOpenSensitiveHistory={openSensitiveDataSuggestionHistory} onDescriptionGenerationRunStarted={rememberDescriptionGenerationRun} onSuggestSensitive={suggestActiveDatabaseSensitiveFields} + onClearCatalogTables={clearActiveCatalogTables} onCatalogMetricsChanged={invalidateCatalogMetrics} /> ) : screen.kind === "relationships" && relationshipsVisible ? ( @@ -1088,12 +1122,14 @@ export function DatabaseManagementPage({ onEdit={editRow} onDelete={deleteRow} onOpenSync={openSync} + onSelectionChange={(selected) => setSelectedDatabaseId(selected.length === 1 ? selected[0].id ?? null : null)} onTestSelected={testSelected} onSyncSelected={syncSelected} selectedMetadataModel={selectedMetadataModelAvailable ? selectedMetadataModel : null} descriptionGenerationActive={descriptionGenerationActive} onGenerateDescriptions={generateDatabaseDescriptions} onSuggestSensitive={suggestDatabaseSensitiveFields} + sensitiveDataSuggestionRuns={sensitiveDataSuggestionRuns} onDeleteMetadataSelected={deleteSelectedMetadata} onRefresh={refreshList} /> @@ -1125,6 +1161,45 @@ export function DatabaseManagementPage({ actions={( <> {fleetMetadataModelControl} + + + )} /> @@ -1144,9 +1219,17 @@ export function DatabaseManagementPage({ navigation={( tableNestedNavigationBack?.(), + disabled: navigationBusy || !tableNestedNavigationBack, + } + : screen.kind === "tables" + ? { label: "Back to databases", onBack: showList, disabled: navigationBusy } + : undefined} /> )} children={{fleetContent}} @@ -1246,10 +1329,12 @@ export function DatabaseManagementPage({ onEdit={editRow} onDelete={deleteRow} onOpenSync={openSync} + onSelectionChange={(selected) => setSelectedDatabaseId(selected.length === 1 ? selected[0].id ?? null : null)} onTestSelected={testSelected} onSyncSelected={syncSelected} selectedMetadataModel={selectedMetadataModelAvailable ? selectedMetadataModel : null} descriptionGenerationActive={descriptionGenerationActive} + sensitiveDataSuggestionRuns={sensitiveDataSuggestionRuns} onGenerateDescriptions={generateDatabaseDescriptions} onSuggestSensitive={suggestDatabaseSensitiveFields} onDeleteMetadataSelected={deleteSelectedMetadata} @@ -1304,10 +1389,9 @@ export function DatabaseManagementPage({ onNavigationStateChange={setTablesNavigationState} onRunStarted={rememberSyncRun} onOpenSync={() => openSync(activeRow)} - onOpenDescriptionHistory={openDescriptionGenerationHistory} - onOpenSensitiveHistory={openSensitiveDataSuggestionHistory} onDescriptionGenerationRunStarted={rememberDescriptionGenerationRun} onSuggestSensitive={suggestActiveDatabaseSensitiveFields} + onClearCatalogTables={clearActiveCatalogTables} onCatalogMetricsChanged={invalidateCatalogMetrics} /> ) : null} diff --git a/frontend/src/shell/database-management/CatalogSyncDrawer.tsx b/frontend/src/shell/database-management/CatalogSyncDrawer.tsx index 41171808..8887089d 100644 --- a/frontend/src/shell/database-management/CatalogSyncDrawer.tsx +++ b/frontend/src/shell/database-management/CatalogSyncDrawer.tsx @@ -285,7 +285,12 @@ export function CatalogSyncDrawer({ open ariaLabel="Schema synchronization" eyebrow="Schema synchronization" - title={run ? `${stateLabel(run)[0].toUpperCase()}${stateLabel(run).slice(1)}` : "Synchronization history"} + title={run ? ( + + {!terminal(run) ? + ) : "Synchronization history"} description={run ? `${run.scope} · ${elapsed(run, now)}` : undefined} onClose={onClose} closeLabel="Close synchronization drawer" diff --git a/frontend/src/shell/database-management/DatabaseColumns.tsx b/frontend/src/shell/database-management/DatabaseColumns.tsx index c2897469..f8deffaa 100644 --- a/frontend/src/shell/database-management/DatabaseColumns.tsx +++ b/frontend/src/shell/database-management/DatabaseColumns.tsx @@ -30,9 +30,7 @@ interface Props { descriptionGenerationActive: boolean; onDescriptionGenerationRunStarted: (run: DescriptionGenerationRun) => void; onNavigationStateChange: (state: DatabaseNavigationState) => void; - onSync: () => void; onSuggestSensitive: (selection: SensitiveDataSuggestionRequest, scopeLabel: string) => Promise; - bindingReady?: boolean; catalogOperationActive?: boolean; onCatalogMetricsChanged?: () => void | Promise; presentation?: "legacy" | "fleet"; @@ -79,6 +77,7 @@ function SensitiveCell({ data, context }: ICellRendererParams context.onSensitiveChange(data, event.target.checked)} onClick={(event) => event.stopPropagation()} /> @@ -94,9 +93,7 @@ export function DatabaseColumns({ descriptionGenerationActive, onDescriptionGenerationRunStarted, onNavigationStateChange, - onSync, onSuggestSensitive, - bindingReady = true, catalogOperationActive = false, onCatalogMetricsChanged, presentation = "legacy", @@ -291,7 +288,6 @@ export function DatabaseColumns({ | "generate-descriptions" | "consolidate-descriptions" | "suggest-sensitive" - | "sync-columns" | "save-sensitive"; const fleetActions: readonly FleetActionOption[] = [ @@ -347,23 +343,6 @@ export function DatabaseColumns({ ? "Another action is running." : undefined, }, - { - id: "sync-columns", - label: "Synchronize columns", - group: "Synchronization", - scopeLabel: "All columns in this table", - runLabel: "Synchronize", - disabled: !canManage || !bindingReady || catalogOperationActive || busy, - disabledReason: !canManage - ? "You do not have permission to synchronize the catalog." - : !bindingReady - ? "Test the current database binding first." - : catalogOperationActive - ? "Wait for the active catalog operation to finish." - : busy - ? "Another action is running." - : undefined, - }, { id: "save-sensitive", label: "Save sensitive-field changes", @@ -386,7 +365,6 @@ export function DatabaseColumns({ if (action === "generate-descriptions") await generateDescriptions(); else if (action === "consolidate-descriptions") await consolidateDescriptions(); else if (action === "suggest-sensitive") await suggestSensitive(); - else if (action === "sync-columns") onSync(); else await saveSensitive(); }; @@ -397,7 +375,7 @@ export function DatabaseColumns({ const columns = useMemo[]>(() => [ { field: "ordinalPosition", headerName: "#", width: 64, maxWidth: 64, filter: "agNumberColumnFilter" }, - { field: "sensitive", headerName: "Sensitive", minWidth: 110, width: 110, sortable: false, filter: false, resizable: false, cellRenderer: SensitiveCell }, + { field: "sensitive", headerName: "Sensitive", headerTooltip: "Mark this column as sensitive; this does not select it for actions.", minWidth: 110, width: 110, sortable: false, filter: false, resizable: false, cellRenderer: SensitiveCell }, { field: "name", headerName: "Name", minWidth: 190, flex: 1, cellClass: "font-mono text-xs" }, { field: "dataType", headerName: "Type", minWidth: 150, flex: 0.8, cellClass: "font-mono text-xs" }, { headerName: "Keys", minWidth: 125, width: 125, sortable: false, filter: false, cellRenderer: KeyCell }, @@ -512,7 +490,6 @@ export function DatabaseColumns({ {data.length} {changedSensitiveColumns.length > 0 ? : null} - ) )} @@ -529,6 +506,17 @@ export function DatabaseColumns({ defaultColDef={{ sortable: true, filter: true, resizable: true }} getRowId={({ data: row }) => row.id} rowSelection={{ mode: "multiRow", selectAll: "filtered", enableClickSelection: false }} + selectionColumnDef={{ + headerName: "Select", + headerTooltip: "Select columns for actions", + width: 72, + minWidth: 72, + maxWidth: 72, + pinned: "left", + sortable: false, + resizable: false, + suppressMovable: true, + }} onSelectionChanged={({ api }) => setSelectedIds(api.getSelectedRows().map((column) => column.id))} rowHeight={44} headerHeight={38} diff --git a/frontend/src/shell/database-management/DatabaseFleetQueryErrors.test.tsx b/frontend/src/shell/database-management/DatabaseFleetQueryErrors.test.tsx index b6c47306..dabada5e 100644 --- a/frontend/src/shell/database-management/DatabaseFleetQueryErrors.test.tsx +++ b/frontend/src/shell/database-management/DatabaseFleetQueryErrors.test.tsx @@ -115,7 +115,6 @@ test("Fleet columns distinguish a failed query from an empty catalog and retry i descriptionGenerationActive={false} onDescriptionGenerationRunStarted={noop} onNavigationStateChange={noop} - onSync={noop} onSuggestSensitive={noopAsync} />, ); diff --git a/frontend/src/shell/database-management/DatabaseForm.tsx b/frontend/src/shell/database-management/DatabaseForm.tsx index af198f26..a22e2306 100644 --- a/frontend/src/shell/database-management/DatabaseForm.tsx +++ b/frontend/src/shell/database-management/DatabaseForm.tsx @@ -12,7 +12,6 @@ import { Button } from "../../components/ui/button"; import type { CatalogDatabase, CatalogSecretName, - CatalogSyncScope, CatalogSyncRun, DatabaseBinding, DatabaseTransport, @@ -23,7 +22,7 @@ import type { DatabaseFormMode, } from "./model"; import { statusLabel } from "./model"; -import { DatabaseSyncMenu } from "./DatabaseSyncMenu"; +import { DatabaseSyncMenu, type DatabaseSyncScope } from "./DatabaseSyncMenu"; const inputClass = "h-9 w-full rounded-md border border-input bg-card px-3 text-sm outline-none transition focus:border-primary/60 focus:ring-3 focus:ring-ring/15 read-only:bg-muted/40 read-only:text-muted-foreground disabled:bg-muted disabled:text-muted-foreground"; const labelClass = "grid min-w-0 gap-1.5 text-xs font-medium text-foreground"; @@ -135,7 +134,7 @@ interface DatabaseFormProps { onRetrySecrets: () => void; onOpenTables: (origin: HTMLButtonElement) => void; onOpenRelationships: () => void; - onSync: (scope: CatalogSyncScope) => void; + onSync: (scope: DatabaseSyncScope) => void; onOpenSync: () => void; onOpenDescriptionHistory?: () => void; onOpenSensitiveHistory?: () => void; diff --git a/frontend/src/shell/database-management/DatabaseGrid.tsx b/frontend/src/shell/database-management/DatabaseGrid.tsx index cd33f2f7..8e7e3cda 100644 --- a/frontend/src/shell/database-management/DatabaseGrid.tsx +++ b/frontend/src/shell/database-management/DatabaseGrid.tsx @@ -1,4 +1,5 @@ import { useEffect, useMemo, useRef, useState, type RefObject } from "react"; +import { useQueries } from "@tanstack/react-query"; import { Menu } from "@base-ui/react/menu"; import { AgGridReact } from "ag-grid-react"; import { @@ -11,13 +12,15 @@ import "ag-grid-community/styles/ag-grid.css"; import "ag-grid-community/styles/ag-theme-alpine.css"; import { ChevronDown, Eye, History, Info, Link2, Pencil, RefreshCw, Rows3, Sparkles, Trash2, X } from "lucide-react"; import { Button } from "../../components/ui/button"; +import { getCatalogMetrics } from "../../api/catalog-databases"; import type { CatalogDatabase, + CatalogMetrics, + SensitiveDataSuggestionRun, CatalogDatabaseMetadataDeleteTarget, - CatalogSyncScope, DescriptionGenerationScope, } from "../../api/catalog-databases"; -import { databaseSyncItemClass, databaseSyncScopes } from "./DatabaseSyncMenu"; +import { databaseSyncItemClass, databaseSyncScopes, type DatabaseSyncScope } from "./DatabaseSyncMenu"; import { FleetActionSelector, type FleetActionOption } from "./FleetActionSelector"; import { NO_METADATA_GENERATION_LLM_MODEL_MESSAGE } from "./MetadataGenerationModelSelector"; @@ -37,10 +40,12 @@ interface DatabaseGridProps { onEdit: (row: CatalogDatabase, origin: HTMLButtonElement) => void; onDelete: (row: CatalogDatabase, origin: HTMLButtonElement) => void; onOpenSync: (row: CatalogDatabase) => void; + onSelectionChange?: (rows: CatalogDatabase[]) => void; onTestSelected: (rows: CatalogDatabase[]) => Promise; - onSyncSelected: (rows: CatalogDatabase[], scope: CatalogSyncScope) => Promise; + onSyncSelected: (rows: CatalogDatabase[], scope: DatabaseSyncScope) => Promise; selectedMetadataModel: string | null; descriptionGenerationActive: boolean; + sensitiveDataSuggestionRuns?: SensitiveDataSuggestionRun[]; onGenerateDescriptions: ( rows: CatalogDatabase[], scope: Extract, @@ -68,7 +73,6 @@ interface DatabaseGridContext { type DatabaseFleetAction = | "test" | "sync-tables" - | "sync-columns" | "sync-relationships" | "sync-all" | "generate-all" @@ -124,87 +128,63 @@ function StateCell({ ); } -function evidenceStatus(row: CatalogDatabase): { label: string; tone: StateTone } { - if (!row.workspaceEvidence) { - return { label: "Evidence state unavailable", tone: "warning" }; - } - switch (row.workspaceEvidence.state) { - case "materialized_current_revision": - return { label: "Evidence materialized", tone: "success" }; - case "configuration_required": - return { label: "Evidence credentials required", tone: "warning" }; - case "configured_unverified": - return { label: "Evidence configured, unverified", tone: "info" }; - case "workspace_unavailable": - return { label: "Evidence unavailable", tone: "danger" }; - default: - return { label: "No Evidence declared", tone: "neutral" }; - } +function endpoint(host: string | undefined, port: number | undefined): string { + if (!host) return "Endpoint not configured"; + return port ? `${host}:${port}` : host; } -function RevisionEvidenceCell({ row }: { row: CatalogDatabase }) { - if (!row.workspaceRevision) { - return ; +function accessSummary(row: CatalogDatabase): { label: string; detail: string } { + const transport = row.configured + ? row.binding.transport + : row.runtimeBinding?.transport ?? row.binding.transport; + if (transport === "rest_api") { + if (!row.binding.baseUrl) return { label: "REST", detail: "Endpoint not configured" }; + try { + return { label: "REST", detail: new URL(row.binding.baseUrl).host }; + } catch { + return { label: "REST", detail: row.binding.baseUrl }; + } } - const evidence = evidenceStatus(row); - const source = row.workspaceEvidence?.sourceType ? ` · ${row.workspaceEvidence.sourceType}` : ""; + if (transport === "ssh_tunnel") { + return { label: "SSH", detail: endpoint(row.binding.sshHost, row.binding.sshPort) }; + } + return { + label: "Direct PostgreSQL", + detail: endpoint(row.binding.host, row.binding.port), + }; +} + +function coverageStatus(total: number, complete: number): { label: string; detail: string; tone: StateTone } { + if (total === 0) return { label: "Not started", detail: "No catalog items", tone: "danger" }; + if (complete >= total) return { label: "Complete", detail: `${complete}/${total}`, tone: "success" }; + if (complete > 0) return { label: "Partial", detail: `${complete}/${total}`, tone: "warning" }; + return { label: "Not started", detail: `0/${total}`, tone: "danger" }; +} + +function CatalogStatusCells({ row, metrics, sensitiveRun }: { row: CatalogDatabase; metrics?: CatalogMetrics; sensitiveRun?: SensitiveDataSuggestionRun }) { + const access = accessSummary(row); + const synchronized = row.schemaSyncedVersion === row.version; + const syncStatus = !row.schemaSyncedVersion + ? { label: "Not synchronized", detail: "Schema not synchronized", tone: "danger" as const } + : synchronized + ? { label: "Synchronized", detail: "Current schema", tone: "success" as const } + : { label: "Partial", detail: "Schema revision changed", tone: "warning" as const }; + const descriptions = coverageStatus(metrics?.descriptionTargets ?? 0, metrics?.describedTargets ?? 0); + const sensitiveProcessed = sensitiveRun ? sensitiveRun.suggestedSensitive + sensitiveRun.suggestedNonSensitive : 0; + const sensitive = !sensitiveRun + ? { label: "Not started", detail: "No analysis run", tone: "danger" as const } + : sensitiveRun.status === "completed" && sensitiveProcessed >= sensitiveRun.total + ? { label: "Complete", detail: `${sensitiveProcessed}/${sensitiveRun.total}`, tone: "success" as const } + : ["failed", "interrupted"].includes(sensitiveRun.status) + ? { label: "Incomplete", detail: `${sensitiveProcessed}/${sensitiveRun.total}`, tone: "danger" as const } + : { label: "Partial", detail: `${sensitiveProcessed}/${sensitiveRun.total}`, tone: "warning" as const }; return ( - - ); -} - -function transportLabel(row: CatalogDatabase): string { - const transport = row.runtimeBinding?.transport; - if (transport === "postgres_direct") return "Direct PostgreSQL"; - if (transport === "rest_api") return "REST API"; - if (transport === "ssh_tunnel") return "SSH tunnel"; - return "No runtime binding"; -} - -function RuntimeBindingCell({ row }: { row: CatalogDatabase }) { - if (!row.runtimeBinding) { - return ; - } - if (!row.runtimeBinding.sessionTransportSupported) { - return ; - } - if (row.runtimeBinding.configurationState === "configuration_required") { - return ; - } - return ; -} - -function catalogConnectionDetail(row: CatalogDatabase): { label: string; className?: string } { - if (row.activeSyncRun) return { label: "Metadata sync in progress", className: "text-primary" }; - if (row.testedVersion !== undefined && row.testedVersion !== row.version) { - return { label: "Connection retest required", className: "text-amber-700 dark:text-amber-300" }; - } - if (row.connectionStatus === "reachable") return { label: "Connection reachable" }; - if (row.connectionStatus === "failed") { - return { label: "Connection test failed", className: "text-destructive" }; - } - return { label: "Connection not tested" }; -} - -function MetadataCatalogCell({ row }: { row: CatalogDatabase }) { - if (!row.workspaceAvailable) { - return ; - } - if (!row.configured) { - return ; - } - const connection = catalogConnectionDetail(row); - return ( - +
+ + + + +
); } @@ -259,11 +239,6 @@ function DatabaseActionsCell({ return (
event.stopPropagation()}> - {data.activeSyncRun || (context.presentation === "fleet" && data.configured && data.id) ? ( - - ) : null} {context.presentation === "fleet" && context.onOpenTables ? ( @@ -833,7 +769,9 @@ export function DatabaseGrid({ onSelectionChanged={({ api }) => { if (pendingDelete && action === null) setPendingDelete(null); if (pendingGenerationScope && action === null) setPendingGenerationScope(null); - setSelectedRows(api.getSelectedRows()); + const rows = api.getSelectedRows(); + setSelectedRows(rows); + onSelectionChange?.(rows); }} rowHeight={52} headerHeight={38} diff --git a/frontend/src/shell/database-management/DatabaseSyncMenu.tsx b/frontend/src/shell/database-management/DatabaseSyncMenu.tsx index e0a2800b..af7eff7a 100644 --- a/frontend/src/shell/database-management/DatabaseSyncMenu.tsx +++ b/frontend/src/shell/database-management/DatabaseSyncMenu.tsx @@ -3,14 +3,15 @@ import { ChevronDown, RefreshCw } from "lucide-react"; import { buttonVariants } from "../../components/ui/button"; import type { CatalogSyncScope } from "../../api/catalog-databases"; +export type DatabaseSyncScope = Exclude; + export const databaseSyncItemClass = [ "flex cursor-default select-none items-center rounded-md px-3 py-2 text-sm outline-none", "data-[highlighted]:bg-muted data-[disabled]:opacity-45", ].join(" "); -export const databaseSyncScopes: ReadonlyArray<{ scope: CatalogSyncScope; label: string }> = [ +export const databaseSyncScopes: ReadonlyArray<{ scope: DatabaseSyncScope; label: string }> = [ { scope: "tables", label: "Synchronize tables" }, - { scope: "columns", label: "Synchronize all columns" }, { scope: "relationships", label: "Synchronize relationships" }, { scope: "all", label: "Synchronize all" }, ]; @@ -18,7 +19,7 @@ export const databaseSyncScopes: ReadonlyArray<{ scope: CatalogSyncScope; label: interface DatabaseSyncMenuProps { disabled: boolean; disabledReason?: string; - onSelect: (scope: CatalogSyncScope) => void; + onSelect: (scope: DatabaseSyncScope) => void; } export function DatabaseSyncMenu({ disabled, disabledReason, onSelect }: DatabaseSyncMenuProps) { diff --git a/frontend/src/shell/database-management/DatabaseTables.tsx b/frontend/src/shell/database-management/DatabaseTables.tsx index 0112c399..b6d2b0bd 100644 --- a/frontend/src/shell/database-management/DatabaseTables.tsx +++ b/frontend/src/shell/database-management/DatabaseTables.tsx @@ -1,15 +1,16 @@ -import { useEffect, useMemo, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { Menu } from "@base-ui/react/menu"; import { useQuery, useQueryClient } from "@tanstack/react-query"; import { AgGridReact } from "ag-grid-react"; import type { ColDef, ICellRendererParams } from "ag-grid-community"; -import { ArrowLeft, ChevronDown, Columns3, History, Pencil, RefreshCw, Save, Sparkles, Trash2, X } from "lucide-react"; +import { ArrowLeft, ChevronDown, Columns3, Pencil, RefreshCw, Save, Sparkles, Trash2, X } from "lucide-react"; import { toast } from "sonner"; import { Button } from "../../components/ui/button"; import { ApiError, apiErrorMessage } from "../../api/client"; import { consolidateCatalogDescriptions, deleteCatalogTableMetadata, + listCatalogColumns, listCatalogTables, startCatalogSync, startDescriptionGenerationRun, @@ -37,11 +38,10 @@ interface Props { onOpenOverview: () => void; onOpenRelationships: () => void; onNavigationStateChange: (state: DatabaseNavigationState) => void; - onNestedNavigationChange?: (active: boolean) => void; + onNestedNavigationChange?: (active: boolean, onBack?: () => void) => void; onRunStarted: (run: CatalogSyncRun) => void; onOpenSync: () => void; - onOpenDescriptionHistory?: () => void; - onOpenSensitiveHistory?: () => void; + onClearCatalogTables?: () => Promise; onDescriptionGenerationRunStarted: (run: DescriptionGenerationRun) => void; onSuggestSensitive: (selection: SensitiveDataSuggestionRequest, scopeLabel: string) => Promise; onCatalogMetricsChanged?: () => void | Promise; @@ -95,8 +95,7 @@ export function DatabaseTables({ onNestedNavigationChange, onRunStarted, onOpenSync, - onOpenDescriptionHistory, - onOpenSensitiveHistory, + onClearCatalogTables, onDescriptionGenerationRunStarted, onSuggestSensitive, onCatalogMetricsChanged, @@ -134,9 +133,22 @@ export function DatabaseTables({ const nestedNavigationActive = presentation === "fleet" && Boolean(activeTable) && tableSection === "columns"; const bindingReady = database.connectionStatus === "reachable" && database.testedVersion === database.version; const currentRun = activeRun ?? database.activeSyncRun; + const clearAllCatalogTables = async () => { + if (!onClearCatalogTables || !window.confirm("Clear all catalog tables, columns, and relationships for this database? The source database will not be changed.")) return; + await onClearCatalogTables(); + }; + const leaveTable = useCallback(() => { + if (navigationBusy) return; + if (dirty && !window.confirm("Discard unsaved metadata?")) return; + setActiveTableId(null); + setTableSection("overview"); + window.setTimeout(() => originRef.current?.focus(), 0); + }, [dirty, navigationBusy]); useEffect(() => { onNavigationStateChange({ dirty, busy: navigationBusy }); }, [dirty, navigationBusy, onNavigationStateChange]); - useEffect(() => { onNestedNavigationChange?.(nestedNavigationActive); }, [nestedNavigationActive, onNestedNavigationChange]); + useEffect(() => { + onNestedNavigationChange?.(nestedNavigationActive, nestedNavigationActive ? leaveTable : undefined); + }, [leaveTable, nestedNavigationActive, onNestedNavigationChange]); useEffect(() => () => { onNestedNavigationChange?.(false); }, [onNestedNavigationChange]); useEffect(() => { if (!activeTableId || !activeTable || editorVersion === activeTable.version || busy) return; @@ -168,13 +180,6 @@ export function DatabaseTables({ setStaleBannerOpen(true); setColumnNavigation({ dirty: false, busy: false }); }; - const leaveTable = () => { - if (navigationBusy) return; - if (dirty && !window.confirm("Discard unsaved metadata?")) return; - setActiveTableId(null); - setTableSection("overview"); - window.setTimeout(() => originRef.current?.focus(), 0); - }; const navigateDatabase = (target: "databases" | "overview" | "relationships") => { if (navigationBusy) return; if (dirty && !window.confirm("Discard unsaved metadata?")) return; @@ -281,6 +286,28 @@ export function DatabaseTables({ toast.error(apiErrorMessage(error)); } finally { setBusy(null); } }; + const generateColumnDescriptions = async () => { + if (selectedIds.length === 0 || !selectedMetadataModel) return; + setBusy("generate"); + try { + const columns = (await Promise.all(selectedIds.map((tableId) => listCatalogColumns(databaseId, tableId)))).flat(); + const columnIds = columns.map((column) => column.id); + if (columnIds.length === 0) { + toast.error("The selected tables have no synchronized catalog columns."); + return; + } + const run = await startDescriptionGenerationRun( + databaseId, + selectedMetadataModel, + "selected_columns", + columnIds, + ); + onDescriptionGenerationRunStarted(run); + toast.success(`Column description generation started for ${columnIds.length} column${columnIds.length === 1 ? "" : "s"}`); + } catch (error) { + toast.error(apiErrorMessage(error)); + } finally { setBusy(null); } + }; const suggestSensitive = async () => { if (selectedIds.length === 0) return; setBusy("suggest"); @@ -298,6 +325,7 @@ export function DatabaseTables({ type FleetTableAction = | "generate-descriptions" + | "generate-column-descriptions" | "sync-tables" | "sync-columns" | "consolidate-descriptions" @@ -324,6 +352,24 @@ export function DatabaseTables({ ? "Another action is running." : undefined, }, + { + id: "generate-column-descriptions", + label: "Generate column descriptions", + group: "Descriptions", + runLabel: "Generate", + disabled: !canManage || selectedIds.length === 0 || !selectedMetadataModel || descriptionGenerationActive || busy !== null || Boolean(currentRun), + disabledReason: !canManage + ? "You do not have permission to generate descriptions." + : selectedIds.length === 0 + ? "Select at least one table." + : !selectedMetadataModel + ? NO_METADATA_GENERATION_LLM_MODEL_MESSAGE + : descriptionGenerationActive || Boolean(currentRun) + ? "Wait for the active catalog operation to finish." + : busy !== null + ? "Another action is running." + : undefined, + }, { id: "consolidate-descriptions", label: "Move generated to Description", @@ -421,6 +467,7 @@ export function DatabaseTables({ const runFleetAction = async (action: FleetTableAction) => { if (action === "generate-descriptions") await generateDescriptions(); + else if (action === "generate-column-descriptions") await generateColumnDescriptions(); else if (action === "sync-tables") await synchronize("tables"); else if (action === "consolidate-descriptions") await consolidateDescriptions(); else if (action === "sync-columns") await synchronize("columns", selectedIds); @@ -464,7 +511,7 @@ export function DatabaseTables({ const databaseHeader = (
-
+

Database management

{database.workspaceName}

@@ -472,6 +519,7 @@ export function DatabaseTables({
{currentRun ? : null} + {onClearCatalogTables ? : null}
@@ -488,18 +536,6 @@ export function DatabaseTables({ return (
-

Physical table

{activeTable.name}

@@ -513,9 +549,7 @@ export function DatabaseTables({ descriptionGenerationActive={descriptionGenerationActive} onDescriptionGenerationRunStarted={onDescriptionGenerationRunStarted} onNavigationStateChange={setColumnNavigation} - onSync={() => void synchronize("columns", [activeTable.id])} onSuggestSensitive={onSuggestSensitive} - bindingReady={bindingReady} catalogOperationActive={Boolean(currentRun)} onCatalogMetricsChanged={onCatalogMetricsChanged} presentation="fleet" @@ -626,7 +660,6 @@ export function DatabaseTables({ descriptionGenerationActive={descriptionGenerationActive} onDescriptionGenerationRunStarted={onDescriptionGenerationRunStarted} onNavigationStateChange={setColumnNavigation} - onSync={() => void synchronize("columns", [activeTable.id])} onSuggestSensitive={onSuggestSensitive} onCatalogMetricsChanged={onCatalogMetricsChanged} /> @@ -728,6 +761,7 @@ export function DatabaseTables({ void generateDescriptions()}>Generate {selectedIds.length === 1 ? "description" : "descriptions"} + void generateColumnDescriptions()}>Generate descriptions for all columns void synchronize("columns", selectedIds)}>Synchronize columns void consolidateDescriptions()}>Move generated description to Description @@ -769,13 +803,6 @@ export function DatabaseTables({ ) )} - {presentation === "fleet" ? ( - - ) : null} - {onOpenDescriptionHistory ? : null} - {onOpenSensitiveHistory ? : null}
{!bindingReady ?
Test the current database binding from Overview before synchronizing tables.
: null} {fleetQueryError ??
diff --git a/frontend/src/shell/database-management/DescriptionGenerationDrawer.test.tsx b/frontend/src/shell/database-management/DescriptionGenerationDrawer.test.tsx index 44c957d1..9df43da9 100644 --- a/frontend/src/shell/database-management/DescriptionGenerationDrawer.test.tsx +++ b/frontend/src/shell/database-management/DescriptionGenerationDrawer.test.tsx @@ -109,6 +109,9 @@ test("replays ordered SSE events, reconnects from the latest sequence, and keeps renderDrawer(); const log = await screen.findByRole("log", { name: "Description generation events" }); + const progress = screen.getByRole("table", { name: "Description generation progress" }); + expect(within(progress).getAllByRole("columnheader")).toHaveLength(5); + expect(log).toHaveClass("thot-catalog-drawer__event-log"); expect(await within(log).findByText("Run queued")).toBeVisible(); await waitFor(() => expect(FakeEventSource.instances).toHaveLength(1)); expect(FakeEventSource.instances[0].url).toBe( diff --git a/frontend/src/shell/database-management/DescriptionGenerationDrawer.tsx b/frontend/src/shell/database-management/DescriptionGenerationDrawer.tsx index 45acc36a..23ee78ff 100644 --- a/frontend/src/shell/database-management/DescriptionGenerationDrawer.tsx +++ b/frontend/src/shell/database-management/DescriptionGenerationDrawer.tsx @@ -300,6 +300,16 @@ export function DescriptionGenerationDrawer({ } const apparentlyStale = isApparentlyStale(run); const finalError = safeFinalError(run.errorSummary); + const modelDisplay = modelLabel && modelLabel !== run.modelId + ? `${modelLabel} (${run.modelId})` + : run.modelId; + const progressCounters = [ + ["total", "Total"], + ["processed", "Processed"], + ["generated", "Generated"], + ["nonGeneratable", "Not generated"], + ["failed", "Failed"], + ] as const; const footer = ["queued", "running"].includes(run.status) ? (
{apparentlyStale ? ( @@ -341,55 +351,80 @@ export function DescriptionGenerationDrawer({ open ariaLabel="Description generation" eyebrow="Description generation" - title={statusLabel(run)} - description={`${modelLabel}${modelLabel !== run.modelId ? ` (${run.modelId})` : ""} · ${run.scope.replaceAll("_", " ")}`} + title={( + + {!isTerminal(run) ? + )} + description={run.scope.replaceAll("_", " ")} onClose={onClose} closeLabel="Close description generation" busy={action !== null} - bodyClassName="thot-catalog-drawer__history-layout" + bodyClassName="thot-catalog-drawer__history-layout thot-catalog-drawer__run-layout" footer={footer} > - {runQuery.isError ? ( +
+ {runQuery.isError ? (
{apiErrorMessage(runQuery.error)}
) : null} -
+
+
+

AI usage

+
+
Model
{modelDisplay}
+
Input tokens
{(run.inputTokens ?? 0).toLocaleString("en-US")}
+
Cache tokens
{(run.cacheReadTokens ?? 0).toLocaleString("en-US")}
+
Output tokens
{(run.outputTokens ?? 0).toLocaleString("en-US")}
+
+
+
+

Timestamps

+
+
Created
{timestamp(run.createdAt)}
+
Started
{timestamp(run.startedAt)}
+
Updated
{timestamp(run.updatedAt)}
+
Finished
{timestamp(run.finishedAt)}
+
+
+
+ +

Progress

-
- {([ - ["total", "Total"], - ["processed", "Processed"], - ["generated", "Generated"], - ["nonGeneratable", "Not generated"], - ["failed", "Failed"], - ] as const).map(([name, label]) => ( -
-

{label}

-

{run[name]}

-
- ))} +
+ + + + {progressCounters.map(([name, label]) => ( + + ))} + + + + + {progressCounters.map(([name]) => ( + + ))} + + +
+ {label} +
+ {run[name]} +
-
-

Timestamps

-
-
Created
{timestamp(run.createdAt)}
-
Started
{timestamp(run.startedAt)}
-
Updated
{timestamp(run.updatedAt)}
-
Finished
{timestamp(run.finishedAt)}
-
-
- {finalError ? ( -
+
{finalError}
) : null} -
+

Events

{events.length} @@ -397,7 +432,7 @@ export function DescriptionGenerationDrawer({
{events.length === 0 ?

Waiting for events…

: events.map((event) => (

@@ -407,8 +442,9 @@ export function DescriptionGenerationDrawer({

{eventQuery.isError ?

{apiErrorMessage(eventQuery.error)}

: null}
+
-
+

Recent runs

@@ -424,7 +460,7 @@ export function DescriptionGenerationDrawer({ ) : visibleHistory.length === 0 ? (

No description generation runs yet.

) : ( -
+
{visibleHistory.map((item) => (