feat: refine metadata catalog workflows

This commit is contained in:
Codex
2026-09-02 11:38:47 +02:00
parent 076c9742c5
commit 4531746038
48 changed files with 1345 additions and 493 deletions
+3 -4
View File
@@ -108,10 +108,9 @@ and staging only: `?db-ui=legacy` is honored in Vite development or when
prototype on port `5173` also remains temporary until owner acceptance of the integrated surface, prototype on port `5173` also remains temporary until owner acceptance of the integrated surface,
after which both migration aids can be removed. after which both migration aids can be removed.
Schema refresh is one durable asynchronous engine with database-table, database-column, Schema refresh is one durable asynchronous engine with database-table, selected-table-column,
selected-table-column, relationship, and full-database actions. Database-level menus expose the relationship, and full-database actions. Database-level menus expose only the table, relationship,
table, all-column, relationship, and full scopes separately; selecting tables exposes column and full scopes; selecting tables exposes column synchronization plus manual column and relationship cleanup for that subset. Database selections
synchronization plus manual column and relationship cleanup for that subset. Database selections
also expose manual table and relationship cleanup. Cleanup selections are atomic and share the also expose manual table and relationship cleanup. Cleanup selections are atomic and share the
one-active-operation-per-database exclusion with synchronization. Runs have leases and one-active-operation-per-database exclusion with synchronization. Runs have leases and
restart recovery, atomic apply, destructive-diff confirmation with re-scan, cancellation before restart recovery, atomic apply, destructive-diff confirmation with re-scan, cancellation before
+2 -2
View File
@@ -59,7 +59,7 @@ import { DescriptionGenerationWorker } from "./catalog/description-generation-wo
import { SensitiveDataSuggester } from "./catalog/sensitive-data-suggester.js"; import { SensitiveDataSuggester } from "./catalog/sensitive-data-suggester.js";
import { SensitiveDataSuggestionRunner } from "./catalog/sensitive-data-suggestion-runner.js"; import { SensitiveDataSuggestionRunner } from "./catalog/sensitive-data-suggestion-runner.js";
import { import {
PostgresDescriptionSourceSampler, ConcreteDescriptionSourceSampler,
type DescriptionSourceSampler, type DescriptionSourceSampler,
} from "./catalog/description-source-sampler.js"; } from "./catalog/description-source-sampler.js";
import { catalogDescriptionGenerationRoutes } from "./routes/catalog-description-generation.js"; import { catalogDescriptionGenerationRoutes } from "./routes/catalog-description-generation.js";
@@ -177,7 +177,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
{ connectTimeoutMs: config.workspaceDiagnosticTimeoutMs }, { connectTimeoutMs: config.workspaceDiagnosticTimeoutMs },
); );
const descriptionSourceSampler = deps?.descriptionSourceSampler const descriptionSourceSampler = deps?.descriptionSourceSampler
?? new PostgresDescriptionSourceSampler(catalogPostgresAccess); ?? new ConcreteDescriptionSourceSampler(catalogPostgresAccess, workspaceSecretStore);
const descriptionGenerationWorker = new DescriptionGenerationWorker( const descriptionGenerationWorker = new DescriptionGenerationWorker(
catalogRepository, catalogRepository,
workspaceRegistry, workspaceRegistry,
@@ -1,7 +1,7 @@
import { z } from "zod"; import { z } from "zod";
import type { WorkspaceRegistry } from "../workspaces/registry.js"; import type { WorkspaceRegistry } from "../workspaces/registry.js";
import type { MetadataGenerationModels, ResolvedMetadataGenerationModel } from "./metadata-generation-models.js"; import type { MetadataGenerationModels, ResolvedMetadataGenerationModel } from "./metadata-generation-models.js";
import type { ModelCompleter, ModelCompletionMessage } from "./model-completer.js"; import type { ModelCompleter, ModelCompletionMessage, ModelCompletionResult } from "./model-completer.js";
import { import {
ModelCompletionCancelledError, ModelCompletionCancelledError,
ModelCompletionProviderError, ModelCompletionProviderError,
@@ -38,6 +38,7 @@ const MAX_SAMPLE_FIELDS_PER_ROW = 4;
const MAX_SAMPLE_COLUMNS = 4; const MAX_SAMPLE_COLUMNS = 4;
const MAX_REPRESENTATIVE_VALUES_PER_REQUEST = 5; const MAX_REPRESENTATIVE_VALUES_PER_REQUEST = 5;
const MAX_TARGET_SAMPLE_JSON_BYTES = 8 * 1024; const MAX_TARGET_SAMPLE_JSON_BYTES = 8 * 1024;
const MAX_COMPLETION_ATTEMPTS_PER_BATCH = 2;
const generatedOutcomeSchema = z.object({ const generatedOutcomeSchema = z.object({
targetId: z.uuid(), targetId: z.uuid(),
outcome: z.literal("generated"), outcome: z.literal("generated"),
@@ -55,7 +56,9 @@ const completionResponseSchema = z.object({
results: z.array(outcomeSchema).min(1).max(MAX_TARGETS_PER_BATCH), results: z.array(outcomeSchema).min(1).max(MAX_TARGETS_PER_BATCH),
}).strict(); }).strict();
class InvalidModelOutcomeError extends Error {} class InvalidModelJsonError extends Error {}
class InvalidModelSchemaError extends Error {}
class MissingModelTargetsError extends Error {}
class DescriptionGenerationBatchError extends Error { class DescriptionGenerationBatchError extends Error {
constructor( constructor(
readonly failure: unknown, readonly failure: unknown,
@@ -143,6 +146,9 @@ interface DescriptionGenerationCounters {
generated: number; generated: number;
nonGeneratable: number; nonGeneratable: number;
failed: number; failed: number;
inputTokens: number;
cacheReadTokens: number;
outputTokens: number;
consecutiveTechnicalFailures: number; consecutiveTechnicalFailures: number;
} }
@@ -152,6 +158,9 @@ function persistedCounters(counters: DescriptionGenerationCounters) {
generated: counters.generated, generated: counters.generated,
nonGeneratable: counters.nonGeneratable, nonGeneratable: counters.nonGeneratable,
failed: counters.failed, failed: counters.failed,
inputTokens: counters.inputTokens,
cacheReadTokens: counters.cacheReadTokens,
outputTokens: counters.outputTokens,
}; };
} }
@@ -625,36 +634,47 @@ function messagesFor(
} }
function parseOutcomes(content: string, expectedTargetIds: readonly string[]): Map<string, ParsedOutcome> { function parseOutcomes(content: string, expectedTargetIds: readonly string[]): Map<string, ParsedOutcome> {
const trimmed = content.trim();
const fenced = /^```(?:json)?[ \t]*\r?\n([\s\S]*?)\r?\n```$/iu.exec(trimmed);
let parsed: unknown;
try { try {
const trimmed = content.trim(); parsed = JSON.parse(fenced?.[1] ?? trimmed);
const fenced = /^```(?:json)?[ \t]*\r?\n([\s\S]*?)\r?\n```$/iu.exec(trimmed); } catch {
const outcomes = completionResponseSchema.parse(JSON.parse(fenced?.[1] ?? trimmed)).results; throw new InvalidModelJsonError();
const expected = new Set(expectedTargetIds);
if (outcomes.length !== expectedTargetIds.length || expected.size !== expectedTargetIds.length) {
throw new InvalidModelOutcomeError();
}
const mapped = new Map<string, ParsedOutcome>();
for (const outcome of outcomes) {
if (!expected.has(outcome.targetId) || mapped.has(outcome.targetId)) {
throw new InvalidModelOutcomeError();
}
mapped.set(outcome.targetId, outcome.outcome === "generated"
? { ...outcome, description: outcome.description.trim() }
: outcome);
}
if (mapped.size !== expected.size) throw new InvalidModelOutcomeError();
return mapped;
} catch (error) {
if (error instanceof InvalidModelOutcomeError) throw error;
throw new InvalidModelOutcomeError();
} }
const response = completionResponseSchema.safeParse(parsed);
if (!response.success) throw new InvalidModelSchemaError();
const outcomes = response.data.results;
const expected = new Set(expectedTargetIds);
if (outcomes.length !== expectedTargetIds.length || expected.size !== expectedTargetIds.length) {
throw new MissingModelTargetsError();
}
const mapped = new Map<string, ParsedOutcome>();
for (const outcome of outcomes) {
if (!expected.has(outcome.targetId) || mapped.has(outcome.targetId)) {
throw new MissingModelTargetsError();
}
mapped.set(outcome.targetId, outcome.outcome === "generated"
? { ...outcome, description: outcome.description.trim() }
: outcome);
}
if (mapped.size !== expected.size) throw new MissingModelTargetsError();
return mapped;
} }
function safeFailure(error: unknown): string { function safeFailure(error: unknown): string {
if (error instanceof DescriptionGenerationFailureStreakError) return error.message; if (error instanceof DescriptionGenerationFailureStreakError) return error.message;
const failure = error instanceof DescriptionGenerationBatchError ? error.failure : error; const failure = error instanceof DescriptionGenerationBatchError ? error.failure : error;
if (failure instanceof ModelCompletionProviderError) return "The model provider request failed."; if (failure instanceof ModelCompletionProviderError) return "The model provider request failed.";
if (failure instanceof InvalidModelOutcomeError) return "The model response was invalid."; if (failure instanceof InvalidModelJsonError) return "The model response was not valid JSON.";
if (failure instanceof InvalidModelSchemaError) {
return "The model response did not match the required schema.";
}
if (failure instanceof MissingModelTargetsError) {
return "The model response was missing one or more requested targets.";
}
return "Description generation failed."; return "Description generation failed.";
} }
@@ -907,6 +927,9 @@ export class DescriptionGenerationWorker {
generated: 0, generated: 0,
nonGeneratable: 0, nonGeneratable: 0,
failed: 0, failed: 0,
inputTokens: 0,
cacheReadTokens: 0,
outputTokens: 0,
consecutiveTechnicalFailures: 0, consecutiveTechnicalFailures: 0,
}; };
await this.processTargets(run, database, plan.columnTargets, model, counters, signal); await this.processTargets(run, database, plan.columnTargets, model, counters, signal);
@@ -973,20 +996,41 @@ export class DescriptionGenerationWorker {
); );
} }
throwIfCancelled(signal); throwIfCancelled(signal);
let outcomes: Map<string, ParsedOutcome>; const expectedTargetIds = batch.map((target) => (
try { target.kind === "column" ? target.column.id : target.table.id
const content = await this.completer.complete({ ));
model, const messages = messagesFor(database, batch, run.language, sourceSamples);
messages: messagesFor(database, batch, run.language, sourceSamples), let outcomes: Map<string, ParsedOutcome> | undefined;
signal, let terminalFailure: unknown;
}); for (let attempt = 1; attempt <= MAX_COMPLETION_ATTEMPTS_PER_BATCH; attempt += 1) {
throwIfCancelled(signal); try {
outcomes = parseOutcomes(content, batch.map((target) => ( const completion = await this.completer.complete({ model, messages, signal });
target.kind === "column" ? target.column.id : target.table.id const result: ModelCompletionResult = typeof completion === "string"
))); ? { content: completion, usage: { input: 0, cacheRead: 0, output: 0 } }
} catch (error) { : completion;
if (error instanceof ModelCompletionCancelledError) throw error; counters.inputTokens += result.usage.input;
const batchError = new DescriptionGenerationBatchError(error, batch.map(failureTarget)); counters.cacheReadTokens += result.usage.cacheRead;
counters.outputTokens += result.usage.output;
throwIfCancelled(signal);
outcomes = parseOutcomes(result.content, expectedTargetIds);
break;
} catch (error) {
if (error instanceof ModelCompletionCancelledError) throw error;
terminalFailure = error;
if (attempt < MAX_COMPLETION_ATTEMPTS_PER_BATCH) {
await this.appendEvent(
run.id,
"warning",
`${safeFailure(error)} Retrying batch (attempt ${attempt + 1} of ${MAX_COMPLETION_ATTEMPTS_PER_BATCH}).`,
);
}
}
}
if (!outcomes) {
const batchError = new DescriptionGenerationBatchError(
terminalFailure,
batch.map(failureTarget),
);
counters.processed += batch.length; counters.processed += batch.length;
counters.failed += batch.length; counters.failed += batch.length;
counters.consecutiveTechnicalFailures += 1; counters.consecutiveTechnicalFailures += 1;
@@ -1009,7 +1053,10 @@ export class DescriptionGenerationWorker {
const targetId = target.kind === "column" ? target.column.id : target.table.id; const targetId = target.kind === "column" ? target.column.id : target.table.id;
const outcome = outcomes.get(targetId); const outcome = outcomes.get(targetId);
if (!outcome) { if (!outcome) {
throw new DescriptionGenerationBatchError(new InvalidModelOutcomeError(), [failureTarget(target)]); throw new DescriptionGenerationBatchError(
new MissingModelTargetsError(),
[failureTarget(target)],
);
} }
const generatedDescription = outcome.outcome === "generated" const generatedDescription = outcome.outcome === "generated"
? outcome.description ? outcome.description
+132 -43
View File
@@ -1,5 +1,8 @@
import { readFile } from "node:fs/promises";
import type { WorkspaceSecretStore } from "../workspaces/secret-store.js";
import type { CatalogPostgresAccess } from "./postgres-access.js"; import type { CatalogPostgresAccess } from "./postgres-access.js";
import type { WorkspaceDatabase } from "./types.js"; import { CATALOG_SECRET_IDS } from "./secrets.js";
import { CatalogConnectorError, type WorkspaceDatabase } from "./types.js";
const MAX_SOURCE_ROWS = 5; const MAX_SOURCE_ROWS = 5;
const MAX_REPRESENTATIVE_VALUES = 5; const MAX_REPRESENTATIVE_VALUES = 5;
@@ -79,15 +82,82 @@ function distinctKey(value: Exclude<DescriptionSourceSampleValue, null>): string
return `${typeof value}:${String(value)}`; return `${typeof value}:${String(value)}`;
} }
/** PostgreSQL-wire sampler. REST bindings remain unsupported by CatalogPostgresAccess. */ function columnsFor(target: DescriptionSourceSamplingTarget): string[] {
export class PostgresDescriptionSourceSampler implements DescriptionSourceSampler { return [...new Set(target.columnNames)].slice(0, MAX_SOURCE_COLUMNS_PER_TARGET);
constructor(private readonly access: CatalogPostgresAccess) {} }
function normalizedSample(
target: DescriptionSourceSamplingTarget,
columnNames: readonly string[],
sourceRows: readonly Record<string, unknown>[],
): DescriptionTargetSourceSample {
const rows = sourceRows.slice(0, MAX_SOURCE_ROWS).map((row) => ({
fields: columnNames.flatMap((name) => {
const value = normalizeValue(row[name]);
return value === undefined ? [] : [{ name, value }];
}),
}));
const valuesByColumn = new Map<string, Exclude<DescriptionSourceSampleValue, null>[]>();
const seenByColumn = new Map<string, Set<string>>();
let representativeValueCount = 0;
for (const row of rows) {
for (const field of row.fields) {
if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break;
if (field.value === null) continue;
const seen = seenByColumn.get(field.name) ?? new Set<string>();
const key = distinctKey(field.value);
if (seen.has(key)) continue;
seen.add(key);
seenByColumn.set(field.name, seen);
const values = valuesByColumn.get(field.name) ?? [];
values.push(field.value);
valuesByColumn.set(field.name, values);
representativeValueCount += 1;
}
if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break;
}
return {
targetId: target.targetId,
tableName: target.tableName,
rows,
representativeValues: columnNames.flatMap((column) => {
const values = valuesByColumn.get(column);
return values && values.length > 0 ? [{ column, values }] : [];
}),
};
}
function samplingSql(
database: WorkspaceDatabase,
target: DescriptionSourceSamplingTarget,
columns: readonly string[],
): string {
const projections = columns.map((columnName) => {
const identifier = quoteIdentifier(columnName);
return `LEFT((${identifier})::text, ${MAX_SOURCE_VALUE_BYTES}) AS ${identifier}`;
});
return [
`SELECT ${projections.join(", ")}`,
`FROM ${quoteIdentifier(database.schema)}.${quoteIdentifier(target.tableName)}`,
`LIMIT ${MAX_SOURCE_ROWS}`,
].join(" ");
}
/** Bounded source sampler that follows the database's PostgreSQL-wire or REST binding. */
export class ConcreteDescriptionSourceSampler implements DescriptionSourceSampler {
constructor(
private readonly access: CatalogPostgresAccess,
private readonly secretStore?: Pick<WorkspaceSecretStore, "materialize">,
) {}
async sample( async sample(
database: WorkspaceDatabase, database: WorkspaceDatabase,
targets: readonly DescriptionSourceSamplingTarget[], targets: readonly DescriptionSourceSamplingTarget[],
signal: AbortSignal, signal: AbortSignal,
): Promise<readonly DescriptionTargetSourceSample[]> { ): Promise<readonly DescriptionTargetSourceSample[]> {
if (database.binding.transport === "rest_api") {
return await this.sampleRest(database, targets, signal);
}
const client = await this.access.connect(database, signal); const client = await this.access.connect(database, signal);
let transactionOpen = false; let transactionOpen = false;
try { try {
@@ -95,7 +165,7 @@ export class PostgresDescriptionSourceSampler implements DescriptionSourceSample
transactionOpen = true; transactionOpen = true;
const samples: DescriptionTargetSourceSample[] = []; const samples: DescriptionTargetSourceSample[] = [];
for (const target of targets) { for (const target of targets) {
const columnNames = [...new Set(target.columnNames)].slice(0, MAX_SOURCE_COLUMNS_PER_TARGET); const columnNames = columnsFor(target);
if (columnNames.length === 0) { if (columnNames.length === 0) {
samples.push({ samples.push({
targetId: target.targetId, targetId: target.targetId,
@@ -115,44 +185,7 @@ export class PostgresDescriptionSourceSampler implements DescriptionSourceSample
"LIMIT $2", "LIMIT $2",
].join(" "); ].join(" ");
const result = await client.query(sql, [MAX_SOURCE_VALUE_BYTES, MAX_SOURCE_ROWS]); const result = await client.query(sql, [MAX_SOURCE_VALUE_BYTES, MAX_SOURCE_ROWS]);
const rows = result.rows.slice(0, MAX_SOURCE_ROWS).map((row) => ({ samples.push(normalizedSample(target, columnNames, result.rows));
fields: columnNames.flatMap((name) => {
const value = normalizeValue(row[name]);
return value === undefined ? [] : [{ name, value }];
}),
}));
const valuesByColumn = new Map<
string,
Exclude<DescriptionSourceSampleValue, null>[]
>();
const seenByColumn = new Map<string, Set<string>>();
let representativeValueCount = 0;
for (const row of rows) {
for (const field of row.fields) {
if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break;
if (field.value === null) continue;
const seen = seenByColumn.get(field.name) ?? new Set<string>();
const key = distinctKey(field.value);
if (seen.has(key)) continue;
seen.add(key);
seenByColumn.set(field.name, seen);
const values = valuesByColumn.get(field.name) ?? [];
values.push(field.value);
valuesByColumn.set(field.name, values);
representativeValueCount += 1;
}
if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break;
}
const representativeValues = columnNames.flatMap((column) => {
const values = valuesByColumn.get(column);
return values && values.length > 0 ? [{ column, values }] : [];
});
samples.push({
targetId: target.targetId,
tableName: target.tableName,
rows,
representativeValues,
});
} }
return samples; return samples;
} finally { } finally {
@@ -160,4 +193,60 @@ export class PostgresDescriptionSourceSampler implements DescriptionSourceSample
await client.end().catch(() => undefined); await client.end().catch(() => undefined);
} }
} }
private async sampleRest(
database: WorkspaceDatabase,
targets: readonly DescriptionSourceSamplingTarget[],
signal: AbortSignal,
): Promise<readonly DescriptionTargetSourceSample[]> {
if (!this.secretStore) throw new CatalogConnectorError("REST source sampling is not configured");
const auth = database.binding.restAuth ?? "bearer";
const materialized = this.secretStore.materialize(
database.workspaceId,
auth === "none" ? [] : [CATALOG_SECRET_IDS.apiKey],
);
try {
const headers: Record<string, string> = { "content-type": "application/json" };
if (auth !== "none") {
const credentialFile = materialized.files.get(CATALOG_SECRET_IDS.apiKey);
if (!credentialFile) throw new CatalogConnectorError("REST API key is not configured");
const credential = (await readFile(credentialFile, "utf8")).trim();
if (auth === "bearer") headers.authorization = `Bearer ${credential}`;
else headers["x-api-key"] = credential;
}
const baseUrl = database.binding.baseUrl?.replace(/\/+$/, "");
if (!baseUrl) throw new CatalogConnectorError("Database binding is incomplete");
const samples: DescriptionTargetSourceSample[] = [];
for (const target of targets) {
const columnNames = columnsFor(target);
if (columnNames.length === 0) {
samples.push(normalizedSample(target, columnNames, []));
continue;
}
const response = await fetch(`${baseUrl}/rpc/run_query`, {
method: "POST",
headers,
body: JSON.stringify({ query_text: samplingSql(database, target, columnNames) }),
signal,
});
if (!response.ok) throw new CatalogConnectorError("REST source sampling failed");
const body: unknown = await response.json();
if (!Array.isArray(body)
|| body.some((row) => !row || typeof row !== "object" || Array.isArray(row))) {
throw new CatalogConnectorError("REST source sampling response is invalid");
}
samples.push(normalizedSample(
target,
columnNames,
body as Array<Record<string, unknown>>,
));
}
return samples;
} catch (error) {
if (error instanceof CatalogConnectorError) throw error;
throw new CatalogConnectorError("REST source sampling failed");
} finally {
materialized.release();
}
}
} }
+8 -2
View File
@@ -335,7 +335,10 @@ export class MemoryCatalogRepository implements CatalogRepository {
processed: 0, processed: 0,
generated: 0, generated: 0,
nonGeneratable: 0, nonGeneratable: 0,
failed: 0, failed: 0,
inputTokens: 0,
cacheReadTokens: 0,
outputTokens: 0,
createdAt: now, createdAt: now,
startedAt: null, startedAt: null,
updatedAt: now, updatedAt: now,
@@ -444,7 +447,10 @@ export class MemoryCatalogRepository implements CatalogRepository {
status: "running", status: "running",
total: 0, total: 0,
suggestedSensitive: 0, suggestedSensitive: 0,
suggestedNonSensitive: 0, suggestedNonSensitive: 0,
inputTokens: 0,
cacheReadTokens: 0,
outputTokens: 0,
createdAt: now, createdAt: now,
startedAt: now, startedAt: now,
updatedAt: now, updatedAt: now,
+2
View File
@@ -11,6 +11,7 @@ import * as descriptionGenerationRunsMigration from "./migrations/005_descriptio
import * as sensitiveDataFlagMigration from "./migrations/006_sensitive_data_flag.js"; import * as sensitiveDataFlagMigration from "./migrations/006_sensitive_data_flag.js";
import * as sensitiveDataSuggestionRunsMigration from "./migrations/007_sensitive_data_suggestion_runs.js"; import * as sensitiveDataSuggestionRunsMigration from "./migrations/007_sensitive_data_suggestion_runs.js";
import * as catalogLogicalRelationshipsMigration from "./migrations/008_catalog_logical_relationships.js"; import * as catalogLogicalRelationshipsMigration from "./migrations/008_catalog_logical_relationships.js";
import * as aiTokenUsageMigration from "./migrations/009_ai_token_usage.js";
const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL; const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL;
const host = process.env.THT_CATALOG_DB_HOST; const host = process.env.THT_CATALOG_DB_HOST;
@@ -44,6 +45,7 @@ const provider: MigrationProvider = {
"006_sensitive_data_flag": sensitiveDataFlagMigration, "006_sensitive_data_flag": sensitiveDataFlagMigration,
"007_sensitive_data_suggestion_runs": sensitiveDataSuggestionRunsMigration, "007_sensitive_data_suggestion_runs": sensitiveDataSuggestionRunsMigration,
"008_catalog_logical_relationships": catalogLogicalRelationshipsMigration, "008_catalog_logical_relationships": catalogLogicalRelationshipsMigration,
"009_ai_token_usage": aiTokenUsageMigration,
}; };
}, },
}; };
@@ -0,0 +1,20 @@
import type { Kysely } from "kysely";
import type { CatalogDatabase } from "../repository.js";
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
for (const table of ["description_generation_runs", "sensitive_data_suggestion_runs"] as const) {
await db.schema.alterTable(table)
.addColumn("input_tokens", "integer", (col) => col.notNull().defaultTo(0))
.addColumn("cache_read_tokens", "integer", (col) => col.notNull().defaultTo(0))
.addColumn("output_tokens", "integer", (col) => col.notNull().defaultTo(0))
.execute();
}
}
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
for (const table of ["sensitive_data_suggestion_runs", "description_generation_runs"] as const) {
await db.schema.alterTable(table)
.dropColumn("input_tokens").dropColumn("cache_read_tokens").dropColumn("output_tokens")
.execute();
}
}
+7 -5
View File
@@ -4,7 +4,7 @@ import type { ResolvedMetadataGenerationModel } from "./metadata-generation-mode
const MAX_HELPER_OUTPUT_BYTES = 64 * 1024; const MAX_HELPER_OUTPUT_BYTES = 64 * 1024;
const helperOutputSchema = z.discriminatedUnion("ok", [ const helperOutputSchema = z.discriminatedUnion("ok", [
z.object({ ok: z.literal(true), content: z.string() }).strict(), z.object({ ok: z.literal(true), content: z.string(), usage: z.object({ input: z.number().int().nonnegative(), cacheRead: z.number().int().nonnegative(), output: z.number().int().nonnegative() }).strict().optional() }).strict(),
z.object({ ok: z.literal(false), error: z.literal("provider_failure") }).strict(), z.object({ ok: z.literal(false), error: z.literal("provider_failure") }).strict(),
]); ]);
@@ -18,10 +18,12 @@ export interface ModelCompletionRequest {
messages: readonly ModelCompletionMessage[]; messages: readonly ModelCompletionMessage[];
signal: AbortSignal; signal: AbortSignal;
} }
export interface ModelCompletionUsage { input: number; cacheRead: number; output: number; }
export interface ModelCompletionResult { content: string; usage: ModelCompletionUsage; }
/** The provider boundary used by Description Generation. */ /** The provider boundary used by Description Generation. */
export interface ModelCompleter { export interface ModelCompleter {
complete(request: ModelCompletionRequest): Promise<string>; complete(request: ModelCompletionRequest): Promise<string | ModelCompletionResult>;
} }
export class ModelCompletionProviderError extends Error { export class ModelCompletionProviderError extends Error {
@@ -47,7 +49,7 @@ export class PythonModelCompleter implements ModelCompleter {
terminationGraceMs?: number; terminationGraceMs?: number;
}) {} }) {}
async complete(request: ModelCompletionRequest): Promise<string> { async complete(request: ModelCompletionRequest): Promise<ModelCompletionResult> {
if (request.signal.aborted) throw new ModelCompletionCancelledError(); if (request.signal.aborted) throw new ModelCompletionCancelledError();
const payload = { const payload = {
model: `${request.model.provider}/${request.model.model}`, model: `${request.model.provider}/${request.model.model}`,
@@ -62,7 +64,7 @@ export class PythonModelCompleter implements ModelCompleter {
...(request.model.disableThinking === true ? { disable_thinking: true } : {}), ...(request.model.disableThinking === true ? { disable_thinking: true } : {}),
}; };
return await new Promise<string>((resolve, reject) => { return await new Promise<ModelCompletionResult>((resolve, reject) => {
const child = spawn( const child = spawn(
this.options.pythonExecutable, this.options.pythonExecutable,
["-m", this.options.helperModule ?? "tht.internal.litellm_completion"], ["-m", this.options.helperModule ?? "tht.internal.litellm_completion"],
@@ -135,7 +137,7 @@ export class PythonModelCompleter implements ModelCompleter {
if (!output.ok) return fail(); if (!output.ok) return fail();
settled = true; settled = true;
cleanup(); cleanup();
resolve(output.content); resolve({ content: output.content, usage: output.usage ?? { input: 0, cacheRead: 0, output: 0 } });
} catch { } catch {
fail(); fail();
} }
+12
View File
@@ -171,6 +171,9 @@ interface DescriptionGenerationRunTable {
generated: number; generated: number;
nonGeneratable: number; nonGeneratable: number;
failed: number; failed: number;
inputTokens: number;
cacheReadTokens: number;
outputTokens: number;
createdAt: Timestamp; createdAt: Timestamp;
startedAt: Timestamp | null; startedAt: Timestamp | null;
updatedAt: Timestamp; updatedAt: Timestamp;
@@ -195,6 +198,9 @@ interface SensitiveDataSuggestionRunTable {
total: number; total: number;
suggestedSensitive: number; suggestedSensitive: number;
suggestedNonSensitive: number; suggestedNonSensitive: number;
inputTokens: number;
cacheReadTokens: number;
outputTokens: number;
createdAt: Timestamp; createdAt: Timestamp;
startedAt: Timestamp; startedAt: Timestamp;
updatedAt: Timestamp; updatedAt: Timestamp;
@@ -816,6 +822,9 @@ export class KyselyCatalogRepository implements CatalogRepository {
generated: 0, generated: 0,
nonGeneratable: 0, nonGeneratable: 0,
failed: 0, failed: 0,
inputTokens: 0,
cacheReadTokens: 0,
outputTokens: 0,
startedAt: null, startedAt: null,
finishedAt: null, finishedAt: null,
errorSummary: null, errorSummary: null,
@@ -943,6 +952,9 @@ export class KyselyCatalogRepository implements CatalogRepository {
total: 0, total: 0,
suggestedSensitive: 0, suggestedSensitive: 0,
suggestedNonSensitive: 0, suggestedNonSensitive: 0,
inputTokens: 0,
cacheReadTokens: 0,
outputTokens: 0,
finishedAt: null, finishedAt: null,
errorSummary: null, errorSummary: null,
}).returningAll().executeTakeFirstOrThrow(); }).returningAll().executeTakeFirstOrThrow();
@@ -1,6 +1,6 @@
import { z } from "zod"; import { z } from "zod";
import type { MetadataGenerationModels } from "./metadata-generation-models.js"; import type { MetadataGenerationModels } from "./metadata-generation-models.js";
import type { ModelCompleter, ModelCompletionMessage } from "./model-completer.js"; import type { ModelCompleter, ModelCompletionMessage, ModelCompletionResult, ModelCompletionUsage } from "./model-completer.js";
import type { import type {
CatalogColumn, CatalogColumn,
CatalogRepository, CatalogRepository,
@@ -201,6 +201,8 @@ export class SensitiveDataSuggester {
targetIds: readonly string[], targetIds: readonly string[],
signal: AbortSignal, signal: AbortSignal,
onPrepared?: (total: number) => void | Promise<void>, onPrepared?: (total: number) => void | Promise<void>,
onProgress?: (processed: number, suggestions: readonly SensitiveDataSuggestion[]) => void | Promise<void>,
onUsage?: (usage: ModelCompletionUsage) => void | Promise<void>,
): Promise<readonly SensitiveDataSuggestion[]> { ): Promise<readonly SensitiveDataSuggestion[]> {
const database = await this.repository.get(databaseId); const database = await this.repository.get(databaseId);
if (!database) throw new SensitiveDataSuggestionTargetNotFoundError("database"); if (!database) throw new SensitiveDataSuggestionTargetNotFoundError("database");
@@ -212,11 +214,16 @@ export class SensitiveDataSuggester {
for (const batch of batchesFor(database, columns)) { for (const batch of batchesFor(database, columns)) {
let received: Map<string, { columnId: string; sensitive: boolean }> | undefined; let received: Map<string, { columnId: string; sensitive: boolean }> | undefined;
for (let attempt = 0; attempt < 2 && !received; attempt += 1) { for (let attempt = 0; attempt < 2 && !received; attempt += 1) {
const content = await this.completer.complete({ const completion = await this.completer.complete({
model, model,
signal, signal,
messages: [systemMessage, { role: "user", content: userContent(database, batch) }], messages: [systemMessage, { role: "user", content: userContent(database, batch) }],
}); });
const result: ModelCompletionResult = typeof completion === "string"
? { content: completion, usage: { input: 0, cacheRead: 0, output: 0 } }
: completion;
await onUsage?.(result.usage);
const content = result.content;
try { try {
const parsed = responseSchema.parse(JSON.parse(content)); const parsed = responseSchema.parse(JSON.parse(content));
const expected = new Set(batch.map((column) => column.columnId)); const expected = new Set(batch.map((column) => column.columnId));
@@ -240,6 +247,7 @@ export class SensitiveDataSuggester {
currentSensitive: column.currentSensitive, currentSensitive: column.currentSensitive,
sensitive: received!.get(column.columnId)!.sensitive, sensitive: received!.get(column.columnId)!.sensitive,
}))); })));
await onProgress?.(suggestions.length, suggestions.slice(-batch.length));
} }
return suggestions; return suggestions;
} }
@@ -10,6 +10,7 @@ import type {
SensitiveDataSuggestionRun, SensitiveDataSuggestionRun,
SensitiveDataSuggestionScope, SensitiveDataSuggestionScope,
} from "./types.js"; } from "./types.js";
import type { ModelCompletionUsage } from "./model-completer.js";
const interruptedMessage = "Sensitive-field suggestion generation was interrupted by backend restart."; const interruptedMessage = "Sensitive-field suggestion generation was interrupted by backend restart.";
const failedMessage = "Sensitive-field suggestion generation failed."; const failedMessage = "Sensitive-field suggestion generation failed.";
@@ -73,6 +74,31 @@ export class SensitiveDataSuggestionRunner {
}); });
if (!prepared) throw new Error("Sensitive Data Suggestion Run disappeared"); if (!prepared) throw new Error("Sensitive Data Suggestion Run disappeared");
}, },
async (processed, batch) => {
const suggestedSensitive = batch.filter((suggestion) => suggestion.sensitive).length;
const suggestedNonSensitive = batch.length - suggestedSensitive;
const current = await this.repository.getSensitiveDataSuggestionRun(started.id);
if (!current) throw new Error("Sensitive Data Suggestion Run disappeared");
const progress = await this.repository.updateSensitiveDataSuggestionRun(started.id, {
suggestedSensitive: current.suggestedSensitive + suggestedSensitive,
suggestedNonSensitive: current.suggestedNonSensitive + suggestedNonSensitive,
});
if (!progress) throw new Error("Sensitive Data Suggestion Run disappeared");
await this.repository.appendSensitiveDataSuggestionEvent(
started.id,
"info",
`Classified ${processed} of ${progress.total} columns.`,
);
},
async (usage: ModelCompletionUsage) => {
const current = await this.repository.getSensitiveDataSuggestionRun(started.id);
if (!current) throw new Error("Sensitive Data Suggestion Run disappeared");
await this.repository.updateSensitiveDataSuggestionRun(started.id, {
inputTokens: current.inputTokens + usage.input,
cacheReadTokens: current.cacheReadTokens + usage.cacheRead,
outputTokens: current.outputTokens + usage.output,
});
},
); );
const suggestedSensitive = suggestions.filter((suggestion) => suggestion.sensitive).length; const suggestedSensitive = suggestions.filter((suggestion) => suggestion.sensitive).length;
const suggestedNonSensitive = suggestions.length - suggestedSensitive; const suggestedNonSensitive = suggestions.length - suggestedSensitive;
+12
View File
@@ -234,6 +234,9 @@ export interface DescriptionGenerationRun {
generated: number; generated: number;
nonGeneratable: number; nonGeneratable: number;
failed: number; failed: number;
inputTokens: number;
cacheReadTokens: number;
outputTokens: number;
createdAt: string; createdAt: string;
startedAt: string | null; startedAt: string | null;
updatedAt: string; updatedAt: string;
@@ -250,6 +253,9 @@ export interface DescriptionGenerationRunUpdate {
startedAt?: string | null; startedAt?: string | null;
finishedAt?: string | null; finishedAt?: string | null;
errorSummary?: string | null; errorSummary?: string | null;
inputTokens?: number;
cacheReadTokens?: number;
outputTokens?: number;
} }
export interface DescriptionGenerationEvent { export interface DescriptionGenerationEvent {
@@ -272,6 +278,9 @@ export interface SensitiveDataSuggestionRun {
total: number; total: number;
suggestedSensitive: number; suggestedSensitive: number;
suggestedNonSensitive: number; suggestedNonSensitive: number;
inputTokens: number;
cacheReadTokens: number;
outputTokens: number;
createdAt: string; createdAt: string;
startedAt: string; startedAt: string;
updatedAt: string; updatedAt: string;
@@ -286,6 +295,9 @@ export interface SensitiveDataSuggestionRunUpdate {
suggestedNonSensitive?: number; suggestedNonSensitive?: number;
finishedAt?: string | null; finishedAt?: string | null;
errorSummary?: string | null; errorSummary?: string | null;
inputTokens?: number;
cacheReadTokens?: number;
outputTokens?: number;
} }
export interface SensitiveDataSuggestionEvent { export interface SensitiveDataSuggestionEvent {
@@ -101,6 +101,9 @@ function publicRun(run: DescriptionGenerationRun) {
generated: run.generated, generated: run.generated,
nonGeneratable: run.nonGeneratable, nonGeneratable: run.nonGeneratable,
failed: run.failed, failed: run.failed,
inputTokens: run.inputTokens,
cacheReadTokens: run.cacheReadTokens,
outputTokens: run.outputTokens,
createdAt: run.createdAt, createdAt: run.createdAt,
startedAt: run.startedAt, startedAt: run.startedAt,
updatedAt: run.updatedAt, updatedAt: run.updatedAt,
@@ -129,6 +132,9 @@ function publicSensitiveDataSuggestionRun(run: SensitiveDataSuggestionRun) {
total: run.total, total: run.total,
suggestedSensitive: run.suggestedSensitive, suggestedSensitive: run.suggestedSensitive,
suggestedNonSensitive: run.suggestedNonSensitive, suggestedNonSensitive: run.suggestedNonSensitive,
inputTokens: run.inputTokens,
cacheReadTokens: run.cacheReadTokens,
outputTokens: run.outputTokens,
createdAt: run.createdAt, createdAt: run.createdAt,
startedAt: run.startedAt, startedAt: run.startedAt,
updatedAt: run.updatedAt, updatedAt: run.updatedAt,
@@ -218,6 +218,12 @@ test("suggests sensitive flags from structural metadata without persisting them"
runId: responseBody.run.id, runId: responseBody.run.id,
sequence: 2, sequence: 2,
level: "info", level: "info",
message: "Classified 1 of 1 columns.",
},
{
runId: responseBody.run.id,
sequence: 3,
level: "info",
message: "Sensitive-field suggestion generation completed for 1 column.", message: "Sensitive-field suggestion generation completed for 1 column.",
}, },
]); ]);
@@ -671,6 +677,7 @@ test("generates one selected Catalog Column from a single JSON code fence", asyn
errorSummary: null, errorSummary: null,
}); });
expect(Object.keys(start.json()).sort()).toEqual([ expect(Object.keys(start.json()).sort()).toEqual([
"cacheReadTokens",
"createdAt", "createdAt",
"databaseId", "databaseId",
"errorSummary", "errorSummary",
@@ -678,9 +685,11 @@ test("generates one selected Catalog Column from a single JSON code fence", asyn
"finishedAt", "finishedAt",
"generated", "generated",
"id", "id",
"inputTokens",
"language", "language",
"modelId", "modelId",
"nonGeneratable", "nonGeneratable",
"outputTokens",
"processed", "processed",
"scope", "scope",
"startedAt", "startedAt",
@@ -1272,7 +1281,7 @@ test("Stop aborts source sampling before any model request", async () => {
test("an isolated exhausted technical batch failure allows completion with errors", async () => { test("an isolated exhausted technical batch failure allows completion with errors", async () => {
const modelCompleter: ModelCompleter = { const modelCompleter: ModelCompleter = {
complete: vi.fn(async (request) => { complete: vi.fn(async (request) => {
if (vi.mocked(modelCompleter.complete).mock.calls.length === 1) { if (vi.mocked(modelCompleter.complete).mock.calls.length <= 2) {
throw new ModelCompletionProviderError(); throw new ModelCompletionProviderError();
} }
const context = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n")); const context = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n"));
@@ -1320,7 +1329,7 @@ test("an isolated exhausted technical batch failure allows completion with error
failed: 10, failed: 10,
errorSummary: "Description generation completed with errors.", errorSummary: "Description generation completed with errors.",
}); });
expect(modelCompleter.complete).toHaveBeenCalledTimes(2); expect(modelCompleter.complete).toHaveBeenCalledTimes(3);
const updated = new Map( const updated = new Map(
(await repository.listColumns(database.id, table.id)).map((column) => [column.id, column]), (await repository.listColumns(database.id, table.id)).map((column) => [column.id, column]),
); );
@@ -1346,10 +1355,10 @@ test("success resets the technical-failure streak and the third later failure st
const modelCompleter: ModelCompleter = { const modelCompleter: ModelCompleter = {
complete: vi.fn(async (request) => { complete: vi.fn(async (request) => {
const call = vi.mocked(modelCompleter.complete).mock.calls.length; const call = vi.mocked(modelCompleter.complete).mock.calls.length;
if ([1, 2, 4, 5, 6].includes(call)) { if ([1, 2, 4, 5, 6, 7, 8, 9].includes(call)) {
throw Object.assign(new ModelCompletionProviderError(), { message: sensitiveDiagnostic }); throw Object.assign(new ModelCompletionProviderError(), { message: sensitiveDiagnostic });
} }
if (call > 6) throw new Error("a later batch must not start"); if (call > 9) throw new Error("a later batch must not start");
const context = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n")); const context = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n"));
return JSON.stringify({ return JSON.stringify({
results: context.targets.map((target: { targetId: string }) => ({ results: context.targets.map((target: { targetId: string }) => ({
@@ -1389,25 +1398,25 @@ test("success resets the technical-failure streak and the third later failure st
expect(run).toMatchObject({ expect(run).toMatchObject({
status: "failed", status: "failed",
total: 61, total: 61,
processed: 60, processed: 50,
generated: 10, generated: 10,
nonGeneratable: 0, nonGeneratable: 0,
failed: 50, failed: 40,
errorSummary: "Description generation stopped after three consecutive technical batch failures.", errorSummary: "Description generation stopped after three consecutive technical batch failures.",
}); });
expect(modelCompleter.complete).toHaveBeenCalledTimes(6); expect(modelCompleter.complete).toHaveBeenCalledTimes(9);
const requests = vi.mocked(modelCompleter.complete).mock.calls.map(([request]) => request); const requests = vi.mocked(modelCompleter.complete).mock.calls.map(([request]) => request);
expect(new Set(requests.map((request) => request.signal)).size).toBe(1); expect(new Set(requests.map((request) => request.signal)).size).toBe(1);
expect(new Set(requests.map((request) => request.model.id))).toEqual(new Set([configuredModel.id])); expect(new Set(requests.map((request) => request.model.id))).toEqual(new Set([configuredModel.id]));
const updated = new Map( const updated = new Map(
(await repository.listColumns(database.id, table.id)).map((column) => [column.id, column]), (await repository.listColumns(database.id, table.id)).map((column) => [column.id, column]),
); );
targetIds.slice(20, 30).forEach((targetId) => { targetIds.slice(10, 20).forEach((targetId) => {
expect(updated.get(targetId)?.generatedDescription).toBe("Successful reset batch."); expect(updated.get(targetId)?.generatedDescription).toBe("Successful reset batch.");
}); });
expect(updated.get(targetIds[60]!)?.generatedDescription).toBeNull(); expect(updated.get(targetIds[50]!)?.generatedDescription).toBeNull();
const events = await repository.listDescriptionGenerationEvents(run.id); const events = await repository.listDescriptionGenerationEvents(run.id);
expect(events.filter((event) => event.message.includes("model provider request failed"))).toHaveLength(5); expect(events.filter((event) => event.message.includes("model provider request failed"))).toHaveLength(8);
expect(events.at(-1)).toEqual(expect.objectContaining({ expect(events.at(-1)).toEqual(expect.objectContaining({
level: "error", level: "error",
message: "Description generation stopped after three consecutive technical batch failures.", message: "Description generation stopped after three consecutive technical batch failures.",
@@ -1660,6 +1669,7 @@ test("Description Generation history is newest-first, bounded, and exposes only
expect(response.statusCode).toBe(200); expect(response.statusCode).toBe(200);
expect(response.json().map((run: { id: string }) => run.id)).toEqual(ids.slice(1).reverse()); expect(response.json().map((run: { id: string }) => run.id)).toEqual(ids.slice(1).reverse());
expect(Object.keys(response.json()[0]).sort()).toEqual([ expect(Object.keys(response.json()[0]).sort()).toEqual([
"cacheReadTokens",
"createdAt", "createdAt",
"databaseId", "databaseId",
"errorSummary", "errorSummary",
@@ -1667,9 +1677,11 @@ test("Description Generation history is newest-first, bounded, and exposes only
"finishedAt", "finishedAt",
"generated", "generated",
"id", "id",
"inputTokens",
"language", "language",
"modelId", "modelId",
"nonGeneratable", "nonGeneratable",
"outputTokens",
"processed", "processed",
"scope", "scope",
"startedAt", "startedAt",
@@ -1945,7 +1957,7 @@ test("retains completed batch writes when a later batch response is malformed",
}); });
const { run } = await waitForTerminalRun(app, start.json().id); const { run } = await waitForTerminalRun(app, start.json().id);
expect(modelCompleter.complete).toHaveBeenCalledTimes(2); expect(modelCompleter.complete).toHaveBeenCalledTimes(3);
expect(run).toMatchObject({ expect(run).toMatchObject({
status: "completed_with_errors", status: "completed_with_errors",
total: 11, total: 11,
@@ -1972,9 +1984,14 @@ test("retains completed batch writes when a later batch response is malformed",
expect(events.slice(2, 12).map((event) => event.message)).toEqual( expect(events.slice(2, 12).map((event) => event.message)).toEqual(
orderedIds.slice(0, 10).map((targetId) => `Generated description for Catalog Column ${targetId}.`), orderedIds.slice(0, 10).map((targetId) => `Generated description for Catalog Column ${targetId}.`),
); );
expect(events.find((event) => event.level === "warning" && event.message.includes("Retrying batch"))).toEqual(
expect.objectContaining({
message: "The model response did not match the required schema. Retrying batch (attempt 2 of 2).",
}),
);
expect(events.find((event) => event.level === "error")).toEqual(expect.objectContaining({ expect(events.find((event) => event.level === "error")).toEqual(expect.objectContaining({
level: "error", level: "error",
message: `The model response was invalid. Affected Catalog Column target: ${orderedIds[10]}.`, message: `The model response did not match the required schema. Affected Catalog Column target: ${orderedIds[10]}.`,
})); }));
} finally { } finally {
await app.close(); await app.close();
@@ -2186,7 +2203,7 @@ test("Generate Missing skips prior partial results and includes null, empty, and
const metadata = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n")); const metadata = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n"));
if (mode === "partial") { if (mode === "partial") {
partialCall += 1; partialCall += 1;
if (partialCall === 2) throw new ModelCompletionProviderError(); if (partialCall === 2 || partialCall === 3) throw new ModelCompletionProviderError();
return JSON.stringify({ return JSON.stringify({
results: metadata.targets.map((target: { targetId: string }) => ({ results: metadata.targets.map((target: { targetId: string }) => ({
targetId: target.targetId, targetId: target.targetId,
@@ -2512,6 +2529,58 @@ test("localizes valid non-generatable Catalog Column results in English", async
} }
}); });
test("retries invalid JSON once and completes the batch when the second response is valid", async () => {
const modelCompleter: ModelCompleter = {
complete: vi.fn(async (request) => {
if (vi.mocked(modelCompleter.complete).mock.calls.length === 1) {
return { content: "not-json", usage: { input: 11, cacheRead: 3, output: 2 } };
}
const context = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n"));
return {
content: JSON.stringify({
results: context.targets.map((target: { targetId: string }) => ({
targetId: target.targetId,
outcome: "generated",
description: "Generated after the application retry.",
})),
}),
usage: { input: 7, cacheRead: 1, output: 5 },
};
}),
};
const { app, repository, database, table, column } = await setup(modelCompleter);
try {
const start = await app.inject({
method: "POST",
url: `/catalog/databases/${database.id}/description-generation-runs`,
payload: { modelId: configuredModel.id, scope: "selected_columns", targetIds: [column.id] },
});
const { run } = await waitForTerminalRun(app, start.json().id);
expect(run).toMatchObject({
status: "completed",
processed: 1,
generated: 1,
failed: 0,
inputTokens: 18,
cacheReadTokens: 4,
outputTokens: 7,
});
expect(modelCompleter.complete).toHaveBeenCalledTimes(2);
expect(await repository.getColumn(database.id, table.id, column.id)).toMatchObject({
generatedDescription: "Generated after the application retry.",
});
expect(await repository.listDescriptionGenerationEvents(run.id)).toEqual(expect.arrayContaining([
expect.objectContaining({
level: "warning",
message: "The model response was not valid JSON. Retrying batch (attempt 2 of 2).",
}),
]));
} finally {
await app.close();
}
});
test("fails safely when the provider fails and redacts provider diagnostics", async () => { test("fails safely when the provider fails and redacts provider diagnostics", async () => {
const sensitiveDiagnostic = "test-provider-secret private prompt raw provider payload"; const sensitiveDiagnostic = "test-provider-secret private prompt raw provider payload";
const modelCompleter: ModelCompleter = { const modelCompleter: ModelCompleter = {
@@ -2528,6 +2597,7 @@ test("fails safely when the provider fails and redacts provider diagnostics", as
}); });
const { run } = await waitForTerminalRun(app, start.json().id); const { run } = await waitForTerminalRun(app, start.json().id);
expect(modelCompleter.complete).toHaveBeenCalledTimes(2);
expect(run).toMatchObject({ expect(run).toMatchObject({
status: "completed_with_errors", status: "completed_with_errors",
processed: 1, processed: 1,
@@ -2565,7 +2635,7 @@ test.each([
["duplicate mappings", (targetIds: readonly string[]) => ({ results: [ ["duplicate mappings", (targetIds: readonly string[]) => ({ results: [
{ targetId: targetIds[0], outcome: "generated", description: "First valid value" }, { targetId: targetIds[0], outcome: "generated", description: "First valid value" },
{ targetId: targetIds[0], outcome: "non_generatable" }, { targetId: targetIds[0], outcome: "non_generatable" },
] })], ] }), "The model response was missing one or more requested targets."],
["unknown mappings", (targetIds: readonly string[]) => ({ results: [ ["unknown mappings", (targetIds: readonly string[]) => ({ results: [
{ targetId: targetIds[0], outcome: "non_generatable" }, { targetId: targetIds[0], outcome: "non_generatable" },
{ {
@@ -2573,15 +2643,15 @@ test.each([
outcome: "generated", outcome: "generated",
description: "Unknown target value", description: "Unknown target value",
}, },
] })], ] }), "The model response was missing one or more requested targets."],
["missing mappings", (targetIds: readonly string[]) => ({ results: [ ["missing mappings", (targetIds: readonly string[]) => ({ results: [
{ targetId: targetIds[0], outcome: "generated", description: "Only one result" }, { targetId: targetIds[0], outcome: "generated", description: "Only one result" },
] })], ] }), "The model response was missing one or more requested targets."],
["malformed mappings", (targetIds: readonly string[]) => ({ results: [ ["malformed mappings", (targetIds: readonly string[]) => ({ results: [
{ targetId: targetIds[0], outcome: "generated", description: "First valid value" }, { targetId: targetIds[0], outcome: "generated", description: "First valid value" },
{ targetId: targetIds[1], outcome: "generated", description: " " }, { targetId: targetIds[1], outcome: "generated", description: " " },
] })], ] }), "The model response did not match the required schema."],
] as const)("rejects %s without applying any result from the batch", async (_name, responseFor) => { ] as const)("rejects %s without applying any result from the batch", async (_name, responseFor, failureMessage) => {
let selectedColumnIds: string[] = []; let selectedColumnIds: string[] = [];
const modelCompleter: ModelCompleter = { const modelCompleter: ModelCompleter = {
complete: vi.fn(async () => JSON.stringify(responseFor(selectedColumnIds))), complete: vi.fn(async () => JSON.stringify(responseFor(selectedColumnIds))),
@@ -2625,6 +2695,7 @@ test.each([
}); });
const { run } = await waitForTerminalRun(app, start.json().id); const { run } = await waitForTerminalRun(app, start.json().id);
expect(modelCompleter.complete).toHaveBeenCalledTimes(2);
expect(run).toMatchObject({ expect(run).toMatchObject({
status: "completed_with_errors", status: "completed_with_errors",
processed: 2, processed: 2,
@@ -2644,7 +2715,7 @@ test.each([
expect((await repository.listDescriptionGenerationEvents(run.id)).find((event) => event.level === "error")).toEqual( expect((await repository.listDescriptionGenerationEvents(run.id)).find((event) => event.level === "error")).toEqual(
expect.objectContaining({ expect.objectContaining({
level: "error", level: "error",
message: `The model response was invalid. Affected Catalog Column targets: ${selectedColumnIds.join(", ")}.`, message: `${failureMessage} Affected Catalog Column targets: ${selectedColumnIds.join(", ")}.`,
}), }),
); );
} finally { } finally {
@@ -13,6 +13,7 @@ import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema
import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js"; import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js";
import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js"; import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js";
import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js"; import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js";
import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js";
import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js"; import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js";
import { loadConfig } from "../src/config.js"; import { loadConfig } from "../src/config.js";
import type { WorkspaceRegistry } from "../src/workspaces/registry.js"; import type { WorkspaceRegistry } from "../src/workspaces/registry.js";
@@ -48,6 +49,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
await upSensitiveDataFlag(db); await upSensitiveDataFlag(db);
await upDescriptionGeneration(db); await upDescriptionGeneration(db);
await upSensitiveSuggestionRuns(db); await upSensitiveSuggestionRuns(db);
await upAiTokenUsage(db);
const repository = new KyselyCatalogRepository(db); const repository = new KyselyCatalogRepository(db);
const database = await repository.create({ const database = await repository.create({
workspaceId: "psd-clinical", workspaceId: "psd-clinical",
@@ -124,7 +126,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
description: "Elenco dei pazienti e dei loro dati clinici.", description: "Elenco dei pazienti e dei loro dati clinici.",
}] }); }] });
} }
if (call === 4) { if (call === 5) {
return JSON.stringify({ results: [ return JSON.stringify({ results: [
{ {
targetId: birthDate.id, targetId: birthDate.id,
@@ -138,14 +140,14 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
}, },
] }); ] });
} }
if (call === 5) { if (call === 6) {
return JSON.stringify({ results: [{ return JSON.stringify({ results: [{
targetId: table.id, targetId: table.id,
outcome: "generated", outcome: "generated",
description: "Descrizione rigenerata della tabella pazienti.", description: "Descrizione rigenerata della tabella pazienti.",
}] }); }] });
} }
if (call === 6) { if (call === 7) {
return JSON.stringify({ results: [{ return JSON.stringify({ results: [{
targetId: birthDate.id, targetId: birthDate.id,
outcome: "generated", outcome: "generated",
@@ -340,7 +342,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
expect(await repository.getColumn(database.id, table.id, birthDate.id)).toMatchObject({ expect(await repository.getColumn(database.id, table.id, birthDate.id)).toMatchObject({
generatedDescription: "Descrizione recuperata della data di nascita.", generatedDescription: "Descrizione recuperata della data di nascita.",
}); });
expect(modelCompleter.complete).toHaveBeenCalledTimes(6); expect(modelCompleter.complete).toHaveBeenCalledTimes(7);
expect(JSON.stringify(vi.mocked(modelCompleter.complete).mock.calls)).toContain(persistedSampleSecret); expect(JSON.stringify(vi.mocked(modelCompleter.complete).mock.calls)).toContain(persistedSampleSecret);
const runIds = [ const runIds = [
@@ -1,6 +1,9 @@
import { expect, test, vi } from "vitest"; import { expect, test, vi } from "vitest";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { import {
PostgresDescriptionSourceSampler, ConcreteDescriptionSourceSampler,
type DescriptionSourceSamplingTarget, type DescriptionSourceSamplingTarget,
} from "../src/catalog/description-source-sampler.js"; } from "../src/catalog/description-source-sampler.js";
import type { import type {
@@ -8,6 +11,8 @@ import type {
CatalogPostgresAccess, CatalogPostgresAccess,
} from "../src/catalog/postgres-access.js"; } from "../src/catalog/postgres-access.js";
import type { WorkspaceDatabase } from "../src/catalog/types.js"; import type { WorkspaceDatabase } from "../src/catalog/types.js";
import type { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
import { CATALOG_SECRET_IDS } from "../src/catalog/secrets.js";
const database: WorkspaceDatabase = { const database: WorkspaceDatabase = {
id: "11111111-1111-4111-8111-111111111111", id: "11111111-1111-4111-8111-111111111111",
@@ -51,7 +56,7 @@ test("samples at most five source rows and five distinct non-null examples in a
const access: CatalogPostgresAccess = { const access: CatalogPostgresAccess = {
connect: vi.fn(async () => ({ query, end }) as CatalogDatabaseClient), connect: vi.fn(async () => ({ query, end }) as CatalogDatabaseClient),
}; };
const sampler = new PostgresDescriptionSourceSampler(access); const sampler = new ConcreteDescriptionSourceSampler(access);
const controller = new AbortController(); const controller = new AbortController();
const samples = await sampler.sample(database, [target], controller.signal); const samples = await sampler.sample(database, [target], controller.signal);
@@ -92,13 +97,71 @@ test("samples at most five source rows and five distinct non-null examples in a
expect(end).toHaveBeenCalledOnce(); expect(end).toHaveBeenCalledOnce();
}); });
test("samples source rows through the configured REST run_query binding", async () => {
const root = mkdtempSync(join(tmpdir(), "tht-source-rest-"));
const credentialFile = join(root, "api-key");
writeFileSync(credentialFile, "test-api-key\n", { mode: 0o600 });
const release = vi.fn();
const secretStore = {
materialize: vi.fn(() => ({
files: new Map([[CATALOG_SECRET_IDS.apiKey, credentialFile]]),
release,
})),
} as unknown as WorkspaceSecretStore;
const fetchMock = vi.fn(async () => new Response(JSON.stringify([
{ 'status"code': "active", ward: null },
{ 'status"code': "pending", ward: "A" },
]), { status: 200, headers: { "content-type": "application/json" } }));
vi.stubGlobal("fetch", fetchMock);
const access: CatalogPostgresAccess = {
connect: vi.fn(async () => { throw new Error("PostgreSQL access must not be used"); }),
};
const sampler = new ConcreteDescriptionSourceSampler(access, secretStore);
const restDatabase: WorkspaceDatabase = {
...database,
binding: {
transport: "rest_api",
baseUrl: "https://dwh.example.test/root/",
restPath: "/health",
restAuth: "x-api-key",
},
};
try {
await expect(sampler.sample(restDatabase, [target], new AbortController().signal)).resolves.toEqual([{
targetId: target.targetId,
tableName: target.tableName,
rows: [
{ fields: [{ name: 'status"code', value: "active" }, { name: "ward", value: null }] },
{ fields: [{ name: 'status"code', value: "pending" }, { name: "ward", value: "A" }] },
],
representativeValues: [
{ column: 'status"code', values: ["active", "pending"] },
{ column: "ward", values: ["A"] },
],
}]);
expect(access.connect).not.toHaveBeenCalled();
expect(fetchMock).toHaveBeenCalledWith("https://dwh.example.test/root/rpc/run_query", expect.objectContaining({
method: "POST",
headers: { "content-type": "application/json", "x-api-key": "test-api-key" },
body: JSON.stringify({
query_text: 'SELECT LEFT(("status""code")::text, 256) AS "status""code", LEFT(("ward")::text, 256) AS "ward" FROM "clinical""data"."patient""facts" LIMIT 5',
}),
}));
expect(release).toHaveBeenCalledOnce();
} finally {
vi.unstubAllGlobals();
rmSync(root, { recursive: true, force: true });
}
});
test("does not issue a SELECT when a protected target has no source columns", async () => { test("does not issue a SELECT when a protected target has no source columns", async () => {
const query = vi.fn(async () => ({ rows: [] })); const query = vi.fn(async () => ({ rows: [] }));
const end = vi.fn(async () => undefined); const end = vi.fn(async () => undefined);
const access: CatalogPostgresAccess = { const access: CatalogPostgresAccess = {
connect: vi.fn(async () => ({ query, end }) as CatalogDatabaseClient), connect: vi.fn(async () => ({ query, end }) as CatalogDatabaseClient),
}; };
const sampler = new PostgresDescriptionSourceSampler(access); const sampler = new ConcreteDescriptionSourceSampler(access);
const samples = await sampler.sample(database, [{ const samples = await sampler.sample(database, [{
targetId: target.targetId, targetId: target.targetId,
@@ -129,7 +192,7 @@ test("rolls back and closes the source connection when sampling fails", async ()
const access: CatalogPostgresAccess = { const access: CatalogPostgresAccess = {
connect: vi.fn(async () => ({ query, end }) as CatalogDatabaseClient), connect: vi.fn(async () => ({ query, end }) as CatalogDatabaseClient),
}; };
const sampler = new PostgresDescriptionSourceSampler(access); const sampler = new ConcreteDescriptionSourceSampler(access);
const controller = new AbortController(); const controller = new AbortController();
await expect(sampler.sample(database, [target], controller.signal)).rejects.toThrow(); await expect(sampler.sample(database, [target], controller.signal)).rejects.toThrow();
@@ -13,6 +13,7 @@ import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_des
import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js"; import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js";
import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js"; import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js";
import { up as upLogicalRelationships } from "../src/catalog/migrations/008_catalog_logical_relationships.js"; import { up as upLogicalRelationships } from "../src/catalog/migrations/008_catalog_logical_relationships.js";
import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js";
const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0; const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
@@ -28,6 +29,9 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
await upSchemaSync(db); await upSchemaSync(db);
await upSensitiveDataFlag(db); await upSensitiveDataFlag(db);
await upLogicalRelationships(db); await upLogicalRelationships(db);
await upDescriptionGeneration(db);
await upSensitiveSuggestionRuns(db);
await upAiTokenUsage(db);
await sql`CREATE ROLE thothii_catalog_runtime`.execute(db); await sql`CREATE ROLE thothii_catalog_runtime`.execute(db);
await upRuntimeSequencePrivileges(db); await upRuntimeSequencePrivileges(db);
const sequencePrivilege = await sql<{ allowed: boolean }>` const sequencePrivilege = await sql<{ allowed: boolean }>`
@@ -386,6 +390,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
await upLogicalRelationships(db); await upLogicalRelationships(db);
await upDescriptionGeneration(db); await upDescriptionGeneration(db);
await upSensitiveSuggestionRuns(db); await upSensitiveSuggestionRuns(db);
await upAiTokenUsage(db);
const repository = new KyselyCatalogRepository(db); const repository = new KyselyCatalogRepository(db);
const firstDatabase = await repository.create({ const firstDatabase = await repository.create({
workspaceId: "generation-one", workspaceId: "generation-one",
@@ -127,7 +127,7 @@ test("resolves only a configured selection for the later generation boundary", (
expect(() => models.resolve("unknown-model")).toThrow(MetadataGenerationModelUnavailableError); expect(() => models.resolve("unknown-model")).toThrow(MetadataGenerationModelUnavailableError);
}); });
test("loads DeepSeek, GLM, and an explicit keyless Qwen endpoint from installation setup", () => { test("loads DeepSeek models, GLM, and an explicit keyless Qwen endpoint from installation setup", () => {
const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration: const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration:
default: glm-53 default: glm-53
models: models:
@@ -135,6 +135,10 @@ test("loads DeepSeek, GLM, and an explicit keyless Qwen endpoint from installati
label: DeepSeek V4 Pro label: DeepSeek V4 Pro
litellm: {provider: deepseek, model: deepseek-v4-pro} litellm: {provider: deepseek, model: deepseek-v4-pro}
apiKeyEnv: DEEPSEEK_API_KEY apiKeyEnv: DEEPSEEK_API_KEY
- id: deepseek-v4-flash
label: DeepSeek V4 Flash
litellm: {provider: deepseek, model: deepseek-v4-flash}
apiKeyEnv: DEEPSEEK_API_KEY
- id: glm-53 - id: glm-53
label: GLM 5.3 label: GLM 5.3
litellm: litellm:
@@ -156,6 +160,7 @@ test("loads DeepSeek, GLM, and an explicit keyless Qwen endpoint from installati
expect(models.catalog()).toEqual({ expect(models.catalog()).toEqual({
models: [ models: [
{ id: "deepseek-v4-pro", label: "DeepSeek V4 Pro" }, { id: "deepseek-v4-pro", label: "DeepSeek V4 Pro" },
{ id: "deepseek-v4-flash", label: "DeepSeek V4 Flash" },
{ id: "glm-53", label: "GLM 5.3" }, { id: "glm-53", label: "GLM 5.3" },
{ id: "qwen-36", label: "Qwen 3.6" }, { id: "qwen-36", label: "Qwen 3.6" },
], ],
+2 -2
View File
@@ -64,7 +64,7 @@ sys.stdout.write(json.dumps({"ok": True, "content": "Descrizione italiana"}))
signal: new AbortController().signal, signal: new AbortController().signal,
}); });
expect(content).toBe("Descrizione italiana"); expect(content).toEqual({ content: "Descrizione italiana", usage: { input: 0, cacheRead: 0, output: 0 } });
const captured = JSON.parse(readFileSync(join(roots[0]!, "request.json"), "utf8")); const captured = JSON.parse(readFileSync(join(roots[0]!, "request.json"), "utf8"));
expect(captured.request).toEqual({ expect(captured.request).toEqual({
model: "openai/gpt-4.1-mini", model: "openai/gpt-4.1-mini",
@@ -100,7 +100,7 @@ sys.stdout.write(json.dumps({"ok": True, "content": "Descrizione Qwen"}))
}, },
messages: [{ role: "user", content: "Describe invented metadata." }], messages: [{ role: "user", content: "Describe invented metadata." }],
signal: new AbortController().signal, signal: new AbortController().signal,
})).resolves.toBe("Descrizione Qwen"); })).resolves.toEqual({ content: "Descrizione Qwen", usage: { input: 0, cacheRead: 0, output: 0 } });
expect(JSON.parse(readFileSync(join(roots[0]!, "request.json"), "utf8"))).toEqual({ expect(JSON.parse(readFileSync(join(roots[0]!, "request.json"), "utf8"))).toEqual({
model: "openai/qwen3.6-35b-a3b", model: "openai/qwen3.6-35b-a3b",
@@ -17,6 +17,12 @@ metadataGeneration:
provider: deepseek provider: deepseek
model: deepseek-v4-pro model: deepseek-v4-pro
apiKeyEnv: DEEPSEEK_API_KEY apiKeyEnv: DEEPSEEK_API_KEY
- id: deepseek-v4-flash
label: DeepSeek V4 Flash
litellm:
provider: deepseek
model: deepseek-v4-flash
apiKeyEnv: DEEPSEEK_API_KEY
- id: glm-53 - id: glm-53
label: GLM 5.3 label: GLM 5.3
litellm: litellm:
+1
View File
@@ -108,6 +108,7 @@ not supported by this installation until the declarative contract is extended ge
| Model | Level | Why | Visibility | | Model | Level | Why | Visibility |
|---|---|---|---| |---|---|---|---|
| `deepseek/deepseek-v4-pro` | Built-in Pi | Known public API, already in the build | All projects | | `deepseek/deepseek-v4-pro` | Built-in Pi | Known public API, already in the build | All projects |
| `deepseek/deepseek-v4-flash` | Built-in Pi | Known public API, already in the build | All projects |
| `zai/glm-5.3` | `deploy/pi/models.json` | Custom OpenAI-compatible endpoint | ThothII installation | | `zai/glm-5.3` | `deploy/pi/models.json` | Custom OpenAI-compatible endpoint | ThothII installation |
| `local-qwen/qwen3.6-35b-a3b` | `deploy/pi/models.json` | Locally configured OpenAI-compatible endpoint | ThothII installation | | `local-qwen/qwen3.6-35b-a3b` | `deploy/pi/models.json` | Locally configured OpenAI-compatible endpoint | ThothII installation |
@@ -179,12 +179,12 @@ canonico; i percorsi e i comportamenti descrivono il sorgente disponibile il 202
e Schema Sync. Il renderer e il runtime delle sessioni NL→SQL restano fuori scope e continuano a e Schema Sync. Il renderer e il runtime delle sessioni NL→SQL restano fuori scope e continuano a
rifiutarla finché non verrà deciso il relativo cutover. rifiutarla finché non verrà deciso il relativo cutover.
55. I menu di azione a livello Workspace Database espongono separatamente `Synchronize tables`, 55. I menu di azione a livello Workspace Database espongono separatamente `Synchronize tables`,
`Synchronize all columns`, `Synchronize relationships` e `Synchronize all`. Su una selezione di `Synchronize relationships` e `Synchronize all`. Su una selezione di
database lo scope scelto viene avviato per ogni database idoneo; non viene sostituito database lo scope scelto viene avviato per ogni database idoneo; non viene sostituito
implicitamente con una sincronizzazione completa. implicitamente con una sincronizzazione completa.
56. Per lo scope Columns, `tableIds` vuoto significa tutte le Catalog Table correnti del Workspace 56. Lo scope Columns è disponibile dalla grid Tables e limita la riconciliazione alle tabelle
Database; `tableIds` valorizzato limita invece la riconciliazione alle tabelle indicate. La grid selezionate; la pagina Columns non espone azioni di sincronizzazione. La grid Tables espone
Tables espone `Synchronize columns` sulle tabelle selezionate. `Synchronize columns` sulle tabelle selezionate.
## Correzione del modello mentale corrente ## Correzione del modello mentale corrente
@@ -570,9 +570,9 @@ Le griglie che dispongono di azioni massive usano checkbox e una toolbar contest
menu `Actions` e cancellazione della selezione. La selezione identifica ID espliciti, può essere menu `Actions` e cancellazione della selezione. La selezione identifica ID espliciti, può essere
accumulata attraverso i filtri e viene azzerata dopo successo, nuova sincronizzazione o uscita accumulata attraverso i filtri e viene azzerata dopo successo, nuova sincronizzazione o uscita
dalla pagina; un'azione è all-or-nothing se un elemento non è idoneo. I menu a livello database dalla pagina; un'azione è all-or-nothing se un elemento non è idoneo. I menu a livello database
espongono gli scope fisici come azioni distinte: `Synchronize tables`, `Synchronize all columns`, espongono gli scope fisici come azioni distinte: `Synchronize tables`, `Synchronize relationships`
`Synchronize relationships` e `Synchronize all`. La grid Tables espone invece `Synchronize e `Synchronize all`. La grid Tables espone invece `Synchronize columns` per le tabelle selezionate;
columns` per le tabelle selezionate. Le selezioni database aggiungono `Delete all tables` e la pagina Columns non espone sincronizzazione. Le selezioni database aggiungono `Delete all tables` e
`Delete all relationships`; le selezioni tabelle aggiungono `Delete all columns` e `Delete all `Delete all relationships`; le selezioni tabelle aggiungono `Delete all columns` e `Delete all
relationships`. Queste operazioni sono atomiche, richiedono conferma e non modificano database relationships`. Queste operazioni sono atomiche, richiedono conferma e non modificano database
esterno, binding, configurazione o segreti. Test connection resta un'azione distinta; griglie senza esterno, binding, configurazione o segreti. Test connection resta un'azione distinta; griglie senza
@@ -17,6 +17,7 @@ The installation-level model catalog contained these entries, with `glm-53` as t
| Selection ID | LiteLLM route | Authentication | | Selection ID | LiteLLM route | Authentication |
| --- | --- | --- | | --- | --- | --- |
| `deepseek-v4-pro` | `deepseek/deepseek-v4-pro` | Protected `DEEPSEEK_API_KEY` value already used by core | | `deepseek-v4-pro` | `deepseek/deepseek-v4-pro` | Protected `DEEPSEEK_API_KEY` value already used by core |
| `deepseek-v4-flash` | `deepseek/deepseek-v4-flash` | Protected `DEEPSEEK_API_KEY` value already used by core |
| `glm-53` | `openai/glm-5.3` on the Z.AI coding endpoint | Protected `ZAI_API_KEY` value already used by core | | `glm-53` | `openai/glm-5.3` on the Z.AI coding endpoint | Protected `ZAI_API_KEY` value already used by core |
| `qwen-36` | `openai/qwen3.6-35b-a3b` on the VPN-only AritmoLab endpoint | No operator API key | | `qwen-36` | `openai/qwen3.6-35b-a3b` on the VPN-only AritmoLab endpoint | No operator API key |
@@ -72,6 +72,28 @@ const table = {
updatedAt: "2026-08-27T09:00:00Z", updatedAt: "2026-08-27T09:00:00Z",
}; };
const descriptionRun = {
id: "77777777-7777-4777-8777-777777777777",
databaseId,
scope: "selected_columns",
modelId: "glm-53",
language: "it",
status: "completed",
total: 5,
processed: 5,
generated: 4,
nonGeneratable: 1,
failed: 0,
inputTokens: 1200,
cacheReadTokens: 80,
outputTokens: 420,
createdAt: "2026-08-31T09:00:00Z",
startedAt: "2026-08-31T09:00:01Z",
updatedAt: "2026-08-31T09:00:04Z",
finishedAt: "2026-08-31T09:00:04Z",
errorSummary: null,
};
function metrics(requestedDatabaseId: string | null) { function metrics(requestedDatabaseId: string | null) {
return { return {
scope: requestedDatabaseId ? "database" : "global", scope: requestedDatabaseId ? "database" : "global",
@@ -133,7 +155,8 @@ async function expectContextPanelGeometry(page: Page, accessibleName: string) {
const managerCenter = managerBox.x + managerBox.width / 2; const managerCenter = managerBox.x + managerBox.width / 2;
const panelCenter = panelBox.x + panelBox.width / 2; const panelCenter = panelBox.x + panelBox.width / 2;
expect(Math.abs(panelCenter - managerCenter)).toBeLessThanOrEqual(1); expect(Math.abs(panelCenter - managerCenter)).toBeLessThanOrEqual(1);
expect(Math.abs(panelBox.width / managerBox.width - 0.6)).toBeLessThanOrEqual(0.005); const expectedPanelWidth = Math.min(1200, managerBox.width - (managerBox.width <= 768 ? 24 : 32));
expect(Math.abs(panelBox.width - expectedPanelWidth)).toBeLessThanOrEqual(1);
expect(panelBox.x).toBeGreaterThanOrEqual(managerBox.x - 1); expect(panelBox.x).toBeGreaterThanOrEqual(managerBox.x - 1);
expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(managerBox.x + managerBox.width + 1); expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(managerBox.x + managerBox.width + 1);
expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(sessionRailBox.x + 1); expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(sessionRailBox.x + 1);
@@ -297,7 +320,8 @@ async function expectWorkAreaPanelGeometry(page: Page, accessibleName: string) {
expect(Math.abs( expect(Math.abs(
panelBox.x + panelBox.width / 2 - (workAreaBox.x + workAreaBox.width / 2), panelBox.x + panelBox.width / 2 - (workAreaBox.x + workAreaBox.width / 2),
)).toBeLessThanOrEqual(1); )).toBeLessThanOrEqual(1);
expect(Math.abs(panelBox.width / workAreaBox.width - 0.6)).toBeLessThanOrEqual(0.005); const expectedPanelWidth = Math.min(1200, workAreaBox.width - (workAreaBox.width <= 768 ? 24 : 32));
expect(Math.abs(panelBox.width - expectedPanelWidth)).toBeLessThanOrEqual(1);
expect(panelBox.x).toBeGreaterThanOrEqual(workAreaBox.x - 1); expect(panelBox.x).toBeGreaterThanOrEqual(workAreaBox.x - 1);
expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(workAreaBox.x + workAreaBox.width + 1); expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(workAreaBox.x + workAreaBox.width + 1);
expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(sessionRailBox.x + 1); expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(sessionRailBox.x + 1);
@@ -344,12 +368,20 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb
"GET /api/catalog/metadata-generation/models": { "GET /api/catalog/metadata-generation/models": {
models: [ models: [
{ id: "deepseek-v4-pro", label: "DeepSeek V4 Pro" }, { id: "deepseek-v4-pro", label: "DeepSeek V4 Pro" },
{ id: "deepseek-v4-flash", label: "DeepSeek V4 Flash" },
{ id: "glm-53", label: "GLM 5.3" }, { id: "glm-53", label: "GLM 5.3" },
{ id: "qwen-36", label: "AritmoLab Qwen 3.6 35B A3B" }, { id: "qwen-36", label: "AritmoLab Qwen 3.6 35B A3B" },
], ],
default: "glm-53", default: "glm-53",
}, },
"GET /api/catalog/description-generation-runs?limit=50": [], "GET /api/catalog/description-generation-runs?limit=50": [descriptionRun],
[`GET /api/catalog/description-generation-runs/${descriptionRun.id}`]: descriptionRun,
[`GET /api/catalog/description-generation-runs/${descriptionRun.id}/events-list?after=0`]: [{
sequence: 1,
level: "info",
message: "Description generation completed.",
createdAt: "2026-08-31T09:00:04Z",
}],
"GET /api/catalog/sensitive-data-suggestion-runs?limit=50": [], "GET /api/catalog/sensitive-data-suggestion-runs?limit=50": [],
[`GET /api/catalog/databases/${databaseId}/tables`]: [table], [`GET /api/catalog/databases/${databaseId}/tables`]: [table],
}; };
@@ -387,6 +419,7 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb
const adminNavigationRail = page.getByRole("complementary", { name: "Session navigation" }); const adminNavigationRail = page.getByRole("complementary", { name: "Session navigation" });
const administration = adminNavigationRail.getByRole("button", { name: "Administration", exact: true }); const administration = adminNavigationRail.getByRole("button", { name: "Administration", exact: true });
await expect(administration).toHaveCSS("font-family", /Manrope/);
await expect(administration).toHaveAttribute("aria-expanded", "false"); await expect(administration).toHaveAttribute("aria-expanded", "false");
await expect(adminNavigationRail.getByRole("region", { name: "Administration" })).toHaveCount(0); await expect(adminNavigationRail.getByRole("region", { name: "Administration" })).toHaveCount(0);
await administration.click(); await administration.click();
@@ -435,6 +468,7 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb
await expect(metadataModelSelector).toHaveValue("glm-53"); await expect(metadataModelSelector).toHaveValue("glm-53");
await expect(metadataModelSelector.locator("option")).toHaveText([ await expect(metadataModelSelector.locator("option")).toHaveText([
"DeepSeek V4 Pro", "DeepSeek V4 Pro",
"DeepSeek V4 Flash",
"GLM 5.3", "GLM 5.3",
"AritmoLab Qwen 3.6 35B A3B", "AritmoLab Qwen 3.6 35B A3B",
]); ]);
@@ -488,10 +522,55 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb
.click(); .click();
await expectContextPanelGeometry(page, "Description generation"); await expectContextPanelGeometry(page, "Description generation");
await expectContextPanelHeaderUsesPrimary(page, "Description generation"); await expectContextPanelHeaderUsesPrimary(page, "Description generation");
await page const descriptionPanel = page.getByRole("dialog", { name: "Description generation" });
.getByRole("dialog", { name: "Description generation" }) const descriptionBody = descriptionPanel.locator(
.getByRole("button", { name: /close/i }) ':scope > [data-catalog-panel-region="body"]',
.click(); );
const progressTable = descriptionPanel.getByRole("table", {
name: "Description generation progress",
});
const eventLog = descriptionPanel.getByRole("log", {
name: "Description generation events",
});
const recentRuns = descriptionPanel.getByRole("region", {
name: "Description generation history",
});
await expect(progressTable.getByRole("columnheader")).toHaveCount(5);
await expect(eventLog).toBeVisible();
await expect(recentRuns).toBeVisible();
const [bodyBox, eventLogBox, recentRunsBox] = await Promise.all([
descriptionBody.boundingBox(),
eventLog.boundingBox(),
recentRuns.boundingBox(),
]);
expect(bodyBox).not.toBeNull();
expect(eventLogBox).not.toBeNull();
expect(recentRunsBox).not.toBeNull();
if (bodyBox && eventLogBox && recentRunsBox) {
const eventBottomGutter = bodyBox.y + bodyBox.height - eventLogBox.y - eventLogBox.height;
const recentBottomGutter = bodyBox.y + bodyBox.height - recentRunsBox.y - recentRunsBox.height;
expect(eventBottomGutter).toBeGreaterThanOrEqual(10);
expect(recentBottomGutter).toBeGreaterThanOrEqual(10);
expect(eventLogBox.x + eventLogBox.width).toBeLessThan(recentRunsBox.x);
}
await page.setViewportSize({ width: 720, height: 800 });
const [compactManagerBox, compactDescriptionPanelBox] = await Promise.all([
page.getByTestId("conversation-column").getByRole("main", {
name: "Database management",
}).boundingBox(),
descriptionPanel.boundingBox(),
]);
expect(compactManagerBox).not.toBeNull();
expect(compactDescriptionPanelBox).not.toBeNull();
if (compactManagerBox && compactDescriptionPanelBox) {
expect(compactManagerBox.width - compactDescriptionPanelBox.width).toBeGreaterThanOrEqual(20);
}
expect(await descriptionBody.evaluate((element) => getComputedStyle(element).display)).not.toBe("grid");
await page.setViewportSize({ width: 1910, height: 911 });
await descriptionPanel.getByRole("button", { name: /close/i }).click();
const applicationBar = page.locator( const applicationBar = page.locator(
'main[aria-label="Database management"] .thot-fleet-ledger__header', 'main[aria-label="Database management"] .thot-fleet-ledger__header',
@@ -734,7 +813,7 @@ test("Workspace and Pi management share the centered work-area panel without cov
expect(compactWorkAreaBox).not.toBeNull(); expect(compactWorkAreaBox).not.toBeNull();
expect(compactPanelBox).not.toBeNull(); expect(compactPanelBox).not.toBeNull();
if (compactWorkAreaBox && compactPanelBox) { if (compactWorkAreaBox && compactPanelBox) {
expect(Math.abs(compactPanelBox.width - compactWorkAreaBox.width + 16)).toBeLessThanOrEqual(1); expect(Math.abs(compactPanelBox.width - compactWorkAreaBox.width + 24)).toBeLessThanOrEqual(1);
expect(Math.abs( expect(Math.abs(
compactPanelBox.x + compactPanelBox.width / 2 compactPanelBox.x + compactPanelBox.width / 2
- (compactWorkAreaBox.x + compactWorkAreaBox.width / 2), - (compactWorkAreaBox.x + compactWorkAreaBox.width / 2),
@@ -269,7 +269,6 @@ export function FleetSelectionActions({ props, direction = "row" }: { props: Var
if (state.inventoryLevel === "databases") actions = [ if (state.inventoryLevel === "databases") actions = [
{ id: "databases-test", group: "Connection", label: "Test connections", icon: <TestTube2 size={14} />, disabled, onClick: run(`Testing ${count} selected connection${count === 1 ? "" : "s"}`) }, { id: "databases-test", group: "Connection", label: "Test connections", icon: <TestTube2 size={14} />, disabled, onClick: run(`Testing ${count} selected connection${count === 1 ? "" : "s"}`) },
{ id: "databases-sync-tables", group: "Synchronization", label: "Synchronize tables", icon: <RefreshCw size={14} />, disabled, onClick: run("Table synchronization started", "running") }, { id: "databases-sync-tables", group: "Synchronization", label: "Synchronize tables", icon: <RefreshCw size={14} />, disabled, onClick: run("Table synchronization started", "running") },
{ id: "databases-sync-columns", group: "Synchronization", label: "Synchronize all columns", icon: <Columns3 size={14} />, disabled, onClick: run("Column synchronization started", "running") },
{ id: "databases-sync-relationships", group: "Synchronization", label: "Synchronize relationships", icon: <Activity size={14} />, disabled, onClick: run("Relationship synchronization started", "running") }, { id: "databases-sync-relationships", group: "Synchronization", label: "Synchronize relationships", icon: <Activity size={14} />, disabled, onClick: run("Relationship synchronization started", "running") },
{ id: "databases-sync-all", group: "Synchronization", label: "Synchronize all", icon: <RefreshCw size={14} />, disabled, onClick: run("Full schema synchronization started", "running") }, { id: "databases-sync-all", group: "Synchronization", label: "Synchronize all", icon: <RefreshCw size={14} />, disabled, onClick: run("Full schema synchronization started", "running") },
{ id: "databases-generate-all", group: "Descriptions", label: "Generate all descriptions", icon: <Sparkles size={14} />, disabled, onClick: run("Generate all confirmation opened") }, { id: "databases-generate-all", group: "Descriptions", label: "Generate all descriptions", icon: <Sparkles size={14} />, disabled, onClick: run("Generate all confirmation opened") },
+6
View File
@@ -157,6 +157,9 @@ export interface SensitiveDataSuggestionRun {
total: number; total: number;
suggestedSensitive: number; suggestedSensitive: number;
suggestedNonSensitive: number; suggestedNonSensitive: number;
inputTokens?: number;
cacheReadTokens?: number;
outputTokens?: number;
createdAt: string; createdAt: string;
startedAt: string | null; startedAt: string | null;
updatedAt: string; updatedAt: string;
@@ -297,6 +300,9 @@ export interface DescriptionGenerationRun {
generated: number; generated: number;
nonGeneratable: number; nonGeneratable: number;
failed: number; failed: number;
inputTokens?: number;
cacheReadTokens?: number;
outputTokens?: number;
createdAt: string; createdAt: string;
startedAt: string | null; startedAt: string | null;
updatedAt: string; updatedAt: string;
+1 -1
View File
@@ -4,7 +4,7 @@ import { cva, type VariantProps } from "class-variance-authority"
import { cn } from "@/lib/utils" import { cn } from "@/lib/utils"
const buttonVariants = cva( const buttonVariants = cva(
"group/button inline-flex shrink-0 items-center justify-center rounded-md border border-transparent bg-clip-padding text-sm font-semibold tracking-[0.005em] whitespace-nowrap transition-all outline-none select-none focus-visible:ring-3 focus-visible:ring-ring/25 disabled:pointer-events-none disabled:opacity-50 aria-invalid:border-destructive aria-invalid:ring-3 aria-invalid:ring-destructive/20 dark:aria-invalid:border-destructive/50 dark:aria-invalid:ring-destructive/40 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4", "group/button inline-flex shrink-0 items-center justify-center rounded-md border border-transparent bg-clip-padding font-sans text-sm font-semibold tracking-[0.005em] whitespace-nowrap transition-all outline-none select-none focus-visible:ring-3 focus-visible:ring-ring/25 disabled:pointer-events-none disabled:opacity-50 aria-invalid:border-destructive aria-invalid:ring-3 aria-invalid:ring-destructive/20 dark:aria-invalid:border-destructive/50 dark:aria-invalid:ring-destructive/40 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
{ {
variants: { variants: {
variant: { variant: {
@@ -67,6 +67,7 @@ describe("authenticated shell permissions", () => {
const sessionNavigation = screen.getByRole("complementary", { name: "Session navigation" }); const sessionNavigation = screen.getByRole("complementary", { name: "Session navigation" });
const trigger = await within(sessionNavigation).findByRole("button", { name: "Administration" }); const trigger = await within(sessionNavigation).findByRole("button", { name: "Administration" });
expect(trigger).toHaveClass("font-sans");
expect(trigger).toHaveAttribute("aria-expanded", "false"); expect(trigger).toHaveAttribute("aria-expanded", "false");
expect(within(sessionNavigation).queryByRole("region", { name: "Administration" })).not.toBeInTheDocument(); expect(within(sessionNavigation).queryByRole("region", { name: "Administration" })).not.toBeInTheDocument();
+24 -2
View File
@@ -45,6 +45,23 @@ interface AppShellProps {
type ActiveSurface = "core" | "database-management"; type ActiveSurface = "core" | "database-management";
type ActiveManagementPanel = "workspace" | "pi" | null; type ActiveManagementPanel = "workspace" | "pi" | null;
const SESSION_SCOPES: readonly SessionScope[] = ["mine", "all"]; const SESSION_SCOPES: readonly SessionScope[] = ["mine", "all"];
const APP_NAVIGATION_STORAGE_KEY = "thothii:app-navigation";
function readPersistedNavigation(): { surface: ActiveSurface; panel: ActiveManagementPanel } {
if (import.meta.env.MODE === "test") return { surface: "core", panel: null };
try {
const parsed = JSON.parse(window.localStorage.getItem(APP_NAVIGATION_STORAGE_KEY) ?? "null") as {
surface?: unknown;
panel?: unknown;
} | null;
return {
surface: parsed?.surface === "database-management" ? "database-management" : "core",
panel: parsed?.panel === "workspace" || parsed?.panel === "pi" ? parsed.panel : null,
};
} catch {
return { surface: "core", panel: null };
}
}
function sessionScopeTabClass(selected: boolean): string { function sessionScopeTabClass(selected: boolean): string {
const base = "relative -mb-px flex h-8 w-full cursor-pointer items-center justify-center rounded-t-md border border-b-2 px-2 text-xs font-semibold tracking-[0.005em] outline-none focus-visible:z-10 focus-visible:ring-3 focus-visible:ring-[oklch(var(--nav-active-border)/0.28)]"; const base = "relative -mb-px flex h-8 w-full cursor-pointer items-center justify-center rounded-t-md border border-b-2 px-2 text-xs font-semibold tracking-[0.005em] outline-none focus-visible:z-10 focus-visible:ring-3 focus-visible:ring-[oklch(var(--nav-active-border)/0.28)]";
@@ -166,7 +183,8 @@ export function AppShell({ canLogout }: AppShellProps) {
const queryClient = useQueryClient(); const queryClient = useQueryClient();
const [showActivity, setShowActivity] = useState(false); const [showActivity, setShowActivity] = useState(false);
const [activeSurface, setActiveSurface] = useState<ActiveSurface>("core"); const persistedNavigation = readPersistedNavigation();
const [activeSurface, setActiveSurface] = useState<ActiveSurface>(persistedNavigation.surface);
const databaseManagementPresentation = resolveDatabaseManagementPresentation({ const databaseManagementPresentation = resolveDatabaseManagementPresentation({
isDevelopment: import.meta.env.DEV, isDevelopment: import.meta.env.DEV,
mode: import.meta.env.MODE, mode: import.meta.env.MODE,
@@ -177,7 +195,7 @@ export function AppShell({ canLogout }: AppShellProps) {
const updateDatabaseNavigationState = useCallback((state: { dirty: boolean; busy: boolean }) => { const updateDatabaseNavigationState = useCallback((state: { dirty: boolean; busy: boolean }) => {
databaseNavigationRef.current = state; databaseNavigationRef.current = state;
}, []); }, []);
const [activeManagementPanel, setActiveManagementPanel] = useState<ActiveManagementPanel>(null); const [activeManagementPanel, setActiveManagementPanel] = useState<ActiveManagementPanel>(persistedNavigation.panel);
const [adminNavigationValue, setAdminNavigationValue] = useState<string[]>([]); const [adminNavigationValue, setAdminNavigationValue] = useState<string[]>([]);
const [activeOpen, setActiveOpen] = useState(true); const [activeOpen, setActiveOpen] = useState(true);
const [archiveOpen, setArchiveOpen] = useState(false); const [archiveOpen, setArchiveOpen] = useState(false);
@@ -188,6 +206,10 @@ export function AppShell({ canLogout }: AppShellProps) {
const [collapsedGroups, setCollapsedGroups] = useState<Record<string, boolean>>({}); const [collapsedGroups, setCollapsedGroups] = useState<Record<string, boolean>>({});
const [renameGroupTarget, setRenameGroupTarget] = useState<string | null>(null); const [renameGroupTarget, setRenameGroupTarget] = useState<string | null>(null);
const operationEpochRef = useRef(0); const operationEpochRef = useRef(0);
useEffect(() => {
if (import.meta.env.MODE === "test") return;
window.localStorage.setItem(APP_NAVIGATION_STORAGE_KEY, JSON.stringify({ surface: activeSurface, panel: activeManagementPanel }));
}, [activeManagementPanel, activeSurface]);
useEffect(() => () => { operationEpochRef.current += 1; }, []); useEffect(() => () => { operationEpochRef.current += 1; }, []);
const groups = useMemo( const groups = useMemo(
@@ -197,7 +197,6 @@ function registerCompletedSyncRun(run: CatalogSyncRun) {
const synchronizationScopes = [ const synchronizationScopes = [
{ scope: "tables", label: "Synchronize tables" }, { scope: "tables", label: "Synchronize tables" },
{ scope: "columns", label: "Synchronize all columns" },
{ scope: "relationships", label: "Synchronize relationships" }, { scope: "relationships", label: "Synchronize relationships" },
{ scope: "all", label: "Synchronize all" }, { scope: "all", label: "Synchronize all" },
] as const; ] as const;
@@ -234,9 +233,7 @@ test("renders Fleet Ledger with real metrics, conceptual row tooltips, and persi
expect(screen.queryByText("Inspect physical metadata, curate descriptions and control catalog operations.")).not.toBeInTheDocument(); expect(screen.queryByText("Inspect physical metadata, curate descriptions and control catalog operations.")).not.toBeInTheDocument();
expect(screen.queryByRole("button", { name: "Back to workspace" })).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Back to workspace" })).not.toBeInTheDocument();
expect(screen.queryByRole("button", { name: /Add database/i })).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: /Add database/i })).not.toBeInTheDocument();
expect(screen.getByRole("columnheader", { name: /Revision \/ Evidence/ })).toBeVisible(); expect(screen.getByRole("columnheader", { name: /Catalog status/ })).toBeVisible();
expect(screen.getByRole("columnheader", { name: /NL→SQL runtime/ })).toBeVisible();
expect(screen.getByRole("columnheader", { name: /Metadata Catalog/ })).toBeVisible();
const summary = screen.getByRole("region", { name: "Fleet summary" }); const summary = screen.getByRole("region", { name: "Fleet summary" });
expect(await within(summary).findByText("2,275")).toBeVisible(); expect(await within(summary).findByText("2,275")).toBeVisible();
expect(within(summary).getByText("75%")).toBeVisible(); expect(within(summary).getByText("75%")).toBeVisible();
@@ -261,13 +258,14 @@ test("renders Fleet Ledger with real metrics, conceptual row tooltips, and persi
await user.click(within(restoredDatabaseRow).getByRole("checkbox", { name: /toggle row selection/i })); await user.click(within(restoredDatabaseRow).getByRole("checkbox", { name: /toggle row selection/i }));
const actionPicker = screen.getByRole("combobox", { name: "Batch action" }); const actionPicker = screen.getByRole("combobox", { name: "Batch action" });
expect(within(actionPicker).getByRole("option", { name: /^Synchronize all(?:,|$)/ })).toBeVisible(); expect(within(actionPicker).getByRole("option", { name: /^Synchronize all(?:,|$)/ })).toBeVisible();
expect(within(actionPicker).queryByRole("option", { name: "Synchronize all columns" })).not.toBeInTheDocument();
await user.selectOptions(actionPicker, "generate-missing"); await user.selectOptions(actionPicker, "generate-missing");
expect(screen.getAllByRole("combobox", { name: "Metadata-generation LLM model" })).toHaveLength(1); expect(screen.getAllByRole("combobox", { name: "Metadata-generation LLM model" })).toHaveLength(1);
expect(modelSelector).toHaveValue("local-qwen"); expect(modelSelector).toHaveValue("local-qwen");
}); });
test("shows repository, NL→SQL runtime, and Metadata Catalog states independently", async () => { test("shows synchronization, description, and sensitive-data states independently", async () => {
const configured = makeDatabase({ connectionStatus: "reachable", testedVersion: 3 }); const configured = makeDatabase({ connectionStatus: "reachable", testedVersion: 3 });
const needsRuntimeConfiguration = makeDatabase({ const needsRuntimeConfiguration = makeDatabase({
...unconfigured, ...unconfigured,
@@ -281,21 +279,58 @@ test("shows repository, NL→SQL runtime, and Metadata Catalog states independen
renderPage({ rows: [configured, needsRuntimeConfiguration, orphan], presentation: "fleet" }); renderPage({ rows: [configured, needsRuntimeConfiguration, orphan], presentation: "fleet" });
const configuredRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); const configuredRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
expect(within(configuredRow).getByText("Active revision aaaaaaa")).toBeVisible(); expect(within(configuredRow).getByText("Not synchronized")).toBeVisible();
expect(within(configuredRow).getByText("Evidence materialized · filesystem")).toBeVisible(); expect(within(configuredRow).getByText(/Descriptions ·/)).toBeVisible();
expect(within(configuredRow).getByText("Ready")).toBeVisible(); expect(within(configuredRow).getByText(/Sensitive ·/)).toBeVisible();
expect(within(configuredRow).getByText("Configured")).toBeVisible();
expect(within(configuredRow).getByText("Connection reachable")).toBeVisible();
const needsConfigurationRow = screen.getByRole("row", { name: /Research laboratory/ }); const needsConfigurationRow = screen.getByRole("row", { name: /Research laboratory/ });
expect(within(needsConfigurationRow).getByText("Evidence credentials required · http")).toBeVisible(); expect(within(needsConfigurationRow).getByText("Not synchronized")).toBeVisible();
expect(within(needsConfigurationRow).getByText("Configuration required")).toBeVisible(); expect(within(needsConfigurationRow).getByText(/Descriptions ·/)).toBeVisible();
expect(within(needsConfigurationRow).getByText("Not configured")).toBeVisible();
const orphanRow = screen.getByRole("row", { name: /Retired workspace/ }); const orphanRow = screen.getByRole("row", { name: /Retired workspace/ });
expect(within(orphanRow).getByText("Workspace missing")).toBeVisible(); expect(within(orphanRow).getByText("Not synchronized")).toBeVisible();
expect(within(orphanRow).getByText("Unavailable")).toBeVisible(); });
expect(within(orphanRow).getByText("Orphaned configuration")).toBeVisible();
test("shows the configured access type and endpoint in catalog status", async () => {
const direct = makeDatabase();
const rest = makeDatabase({
id: "33333333-3333-4333-8333-333333333333",
workspaceId: "rest-lab",
workspaceName: "REST laboratory",
binding: {
transport: "rest_api",
baseUrl: "https://203.0.113.18/dwh/",
restPath: "/health",
restAuth: "x-api-key",
},
});
const ssh = makeDatabase({
id: "44444444-4444-4444-8444-444444444444",
workspaceId: "ssh-lab",
workspaceName: "SSH laboratory",
binding: {
transport: "ssh_tunnel",
sshHost: "bastion.example.test",
sshPort: 2222,
sshUsername: "operator",
sshTargetHost: "postgres.internal",
sshTargetPort: 5432,
username: "reader",
},
});
renderPage({ rows: [direct, rest, ssh], presentation: "fleet" });
const directRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
expect(within(directRow).getByText("Direct PostgreSQL")).toBeVisible();
expect(within(directRow).getByText("db.internal:5432")).toBeVisible();
const restRow = screen.getByRole("row", { name: /REST laboratory/ });
expect(within(restRow).getByText("REST")).toBeVisible();
expect(within(restRow).getByText("203.0.113.18")).toBeVisible();
const sshRow = screen.getByRole("row", { name: /SSH laboratory/ });
expect(within(sshRow).getByText("SSH")).toBeVisible();
expect(within(sshRow).getByText("bastion.example.test:2222")).toBeVisible();
}); });
test("keeps Fleet Ledger visible when a catalog response omits the Evidence projection", async () => { test("keeps Fleet Ledger visible when a catalog response omits the Evidence projection", async () => {
@@ -306,7 +341,7 @@ test("keeps Fleet Ledger visible when a catalog response omits the Evidence proj
expect(await screen.findByRole("heading", { name: "Database management" })).toBeVisible(); expect(await screen.findByRole("heading", { name: "Database management" })).toBeVisible();
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
expect(within(databaseRow).getByText("Evidence state unavailable")).toBeVisible(); expect(within(databaseRow).getByText("Not synchronized")).toBeVisible();
}); });
test("opens the relationship map directly from a Fleet database and restores focus on return", async () => { test("opens the relationship map directly from a Fleet database and restores focus on return", async () => {
@@ -359,7 +394,7 @@ test("offers catalog configuration directly on an unconfigured Fleet workspace",
expect(within(drawer).getByLabelText("Workspace")).toHaveValue("Research laboratory"); expect(within(drawer).getByLabelText("Workspace")).toHaveValue("Research laboratory");
}); });
test("keeps only the local back action while browsing Fleet Ledger columns", async () => { test("keeps the table back action in the Fleet Ledger breadcrumb while browsing columns", async () => {
const user = userEvent.setup(); const user = userEvent.setup();
server.use( server.use(
http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([patientsTable])), http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([patientsTable])),
@@ -372,9 +407,17 @@ test("keeps only the local back action while browsing Fleet Ledger columns", asy
name: "View tables for Policlinico San Donato", name: "View tables for Policlinico San Donato",
})); }));
expect(await screen.findByRole("button", { name: "Back to databases" })).toBeVisible(); expect(await screen.findByRole("button", { name: "Back to databases" })).toBeVisible();
expect(screen.getByRole("combobox", { name: "Table action" })).toBeDisabled();
expect(screen.queryByRole("button", { name: "Sync history" })).not.toBeInTheDocument();
expect(screen.queryByRole("button", { name: "Description history" })).not.toBeInTheDocument();
expect(screen.queryByRole("button", { name: "Sensitive history" })).not.toBeInTheDocument();
await user.click(await screen.findByRole("button", { name: "View columns for patients" })); await user.click(await screen.findByRole("button", { name: "View columns for patients" }));
expect(await screen.findByRole("button", { name: "Back to tables" })).toBeVisible(); const tableBackAction = await screen.findByRole("button", { name: "Back to tables" });
expect(tableBackAction).toBeVisible();
expect(screen.getByRole("combobox", { name: "Column action" })).toBeDisabled();
expect(screen.getByRole("navigation", { name: "Database hierarchy" }).parentElement)
.toContainElement(tableBackAction);
await waitFor(() => { await waitFor(() => {
expect(screen.queryByRole("button", { name: "Back to databases" })).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Back to databases" })).not.toBeInTheDocument();
}); });
@@ -427,7 +470,7 @@ test("reopens database synchronization history when no run is active", async ()
renderPage({ rows: [makeDatabase()], presentation: "fleet" }); renderPage({ rows: [makeDatabase()], presentation: "fleet" });
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
await user.click(within(databaseRow).getByRole("button", { name: "View synchronization history for Policlinico San Donato" })); await user.click(within(databaseRow).getByText("Policlinico San Donato"));
const drawer = await screen.findByRole("dialog", { name: "Schema synchronization" }); const drawer = await screen.findByRole("dialog", { name: "Schema synchronization" });
expect(drawer).toBeVisible(); expect(drawer).toBeVisible();
@@ -628,6 +671,7 @@ test.each(synchronizationScopes)(
await user.click(screen.getByRole("menuitem", { name: label })); await user.click(screen.getByRole("menuitem", { name: label }));
await waitFor(() => expect(startBody).toEqual({ version: 3, scope, tableIds: [] })); await waitFor(() => expect(startBody).toEqual({ version: 3, scope, tableIds: [] }));
expect(screen.getByText("1 selected")).toBeVisible();
}, },
); );
@@ -1001,15 +1045,16 @@ test("selected database Actions confirms and deletes catalog tables for the full
], ],
target: "tables", target: "tables",
})); }));
await waitFor(() => expect(screen.queryByText("2 selected")).not.toBeInTheDocument()); expect(screen.getByText("2 selected")).toBeVisible();
await waitFor(() => expect(screen.getByRole("textbox", { name: "Search databases" })).toHaveFocus()); await user.click(screen.getByRole("button", { name: "Clear" }));
expect(screen.queryByText("2 selected")).not.toBeInTheDocument();
}); });
test("presents completed synchronization steps as success and skips unneeded confirmation", async () => { test("presents completed database synchronization steps as success and skips unneeded confirmation", async () => {
const user = userEvent.setup(); const user = userEvent.setup();
const run = { const run = {
...makeSyncRun("columns"), ...makeSyncRun("all"),
counts: { tables: 163, columns: 2_275 }, counts: { tables: 163, columns: 2_275, relationships: 18 },
}; };
registerCompletedSyncRun(run); registerCompletedSyncRun(run);
server.use(http.post("/api/catalog/databases/:databaseId/sync-runs", () => ( server.use(http.post("/api/catalog/databases/:databaseId/sync-runs", () => (
@@ -1022,7 +1067,7 @@ test("presents completed synchronization steps as success and skips unneeded con
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
await user.click(within(databaseRow).getByRole("checkbox", { name: /toggle row selection/i })); await user.click(within(databaseRow).getByRole("checkbox", { name: /toggle row selection/i }));
await user.click(screen.getByRole("button", { name: "Actions" })); await user.click(screen.getByRole("button", { name: "Actions" }));
await user.click(await screen.findByRole("menuitem", { name: "Synchronize all columns" })); await user.click(await screen.findByRole("menuitem", { name: "Synchronize all" }));
expect(await screen.findByRole("heading", { name: "Succeeded" })).toBeVisible(); expect(await screen.findByRole("heading", { name: "Succeeded" })).toBeVisible();
const steps = screen.getByRole("list", { name: "Synchronization steps" }); const steps = screen.getByRole("list", { name: "Synchronization steps" });
@@ -1039,7 +1084,7 @@ test("presents completed synchronization steps as success and skips unneeded con
expect(steps.querySelectorAll(".text-primary")).toHaveLength(0); expect(steps.querySelectorAll(".text-primary")).toHaveLength(0);
}); });
test("database Overview exposes every synchronization scope", async () => { test("database Overview exposes the database synchronization scopes", async () => {
const user = userEvent.setup(); const user = userEvent.setup();
let startBody: unknown; let startBody: unknown;
const run = makeSyncRun("relationships"); const run = makeSyncRun("relationships");
@@ -1520,6 +1565,37 @@ test("starts description generation for multiple selected tables", async () => {
expect(screen.getByText("2 selected")).toBeVisible(); expect(screen.getByText("2 selected")).toBeVisible();
}); });
test("generates descriptions for every column in selected tables", async () => {
const user = userEvent.setup();
const queuedRun = makeDescriptionGenerationRun({ scope: "selected_columns", total: 1 });
let startBody: unknown;
server.use(
http.get("/api/catalog/metadata-generation/models", () => HttpResponse.json({ models: [{ id: "local-qwen", label: "Local Qwen" }], default: "local-qwen" })),
http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([patientsTable])),
http.get(`/api/catalog/databases/:databaseId/tables/${patientsTable.id}/columns`, () => HttpResponse.json([patientIdColumn])),
http.post("/api/catalog/databases/:databaseId/description-generation-runs", async ({ request }) => {
startBody = await request.json();
return HttpResponse.json(queuedRun, { status: 202 });
}),
http.get("/api/catalog/description-generation-runs/:runId", () => HttpResponse.json(queuedRun)),
http.get("/api/catalog/description-generation-runs/:runId/events-list", () => HttpResponse.json([])),
);
renderPage({ rows: [makeDatabase({ connectionStatus: "reachable", testedVersion: 3 })] });
await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" }));
await user.click(screen.getByRole("tab", { name: "Tables" }));
await user.click(within(await screen.findByRole("row", { name: /patients/ })).getByRole("checkbox"));
await user.click(screen.getByRole("button", { name: "Actions" }));
await user.click(await screen.findByRole("menuitem", { name: "Generate descriptions for all columns" }));
await waitFor(() => expect(startBody).toEqual({
modelId: "local-qwen",
scope: "selected_columns",
targetIds: [patientIdColumn.id],
}));
expect(await screen.findByText("Column description generation started for 1 column")).toBeVisible();
});
test("observes an active run from another browser and reopens a terminal run from history", async () => { test("observes an active run from another browser and reopens a terminal run from history", async () => {
const user = userEvent.setup(); const user = userEvent.setup();
const activeRun = makeDescriptionGenerationRun({ const activeRun = makeDescriptionGenerationRun({
@@ -1913,6 +1989,12 @@ test("starts one selected column with the configured default model", async () =>
await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" })); await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" }));
await user.click(screen.getByRole("tab", { name: "Tables" })); await user.click(screen.getByRole("tab", { name: "Tables" }));
await user.click(await screen.findByRole("button", { name: "View columns for patients" })); await user.click(await screen.findByRole("button", { name: "View columns for patients" }));
expect(screen.queryByRole("button", { name: "Sync columns" })).not.toBeInTheDocument();
expect(screen.getByRole("columnheader", { name: /Select/i })).toBeVisible();
expect(await screen.findByRole("checkbox", { name: "Sensitive data for id" })).toHaveAttribute(
"title",
"Mark this column as sensitive; this does not select it for actions.",
);
const columnRow = (await screen.findAllByRole("row", { name: /Patient identifier/ })) const columnRow = (await screen.findAllByRole("row", { name: /Patient identifier/ }))
.find((row) => within(row).queryByRole("checkbox", { name: /toggle row selection/i })); .find((row) => within(row).queryByRole("checkbox", { name: /toggle row selection/i }));
expect(columnRow).toBeDefined(); expect(columnRow).toBeDefined();
+92 -8
View File
@@ -80,6 +80,20 @@ const SYNC_STARTED_MESSAGES: Record<CatalogSyncScope, string> = {
relationships: "Relationship synchronization started", relationships: "Relationship synchronization started",
all: "Full schema synchronization started", all: "Full schema synchronization started",
}; };
const DATABASE_SCREEN_STORAGE_KEY = "thothii:database-management-screen";
function readPersistedDatabaseScreen(): DatabaseScreen {
if (import.meta.env.MODE === "test") return { kind: "list" };
try {
const parsed = JSON.parse(window.localStorage.getItem(DATABASE_SCREEN_STORAGE_KEY) ?? "null") as { kind?: unknown; workspaceId?: unknown } | null;
if ((parsed?.kind === "tables" || parsed?.kind === "relationships") && typeof parsed.workspaceId === "string") {
return { kind: parsed.kind, workspaceId: parsed.workspaceId };
}
} catch {
// Ignore malformed persisted navigation and return to the safe list view.
}
return { kind: "list" };
}
interface Props { interface Props {
canManage: boolean; canManage: boolean;
@@ -156,12 +170,13 @@ export function DatabaseManagementPage({
(run) => run.status === "running", (run) => run.status === "running",
); );
const [screen, setScreen] = useState<DatabaseScreen>({ kind: "list" }); const [screen, setScreen] = useState<DatabaseScreen>(readPersistedDatabaseScreen);
const [selectedMetadataModel, setSelectedMetadataModel] = useState(""); const [selectedMetadataModel, setSelectedMetadataModel] = useState("");
const [draft, setDraft] = useState<DatabaseFormDraft | null>(null); const [draft, setDraft] = useState<DatabaseFormDraft | null>(null);
const [baseline, setBaseline] = useState(""); const [baseline, setBaseline] = useState("");
const [formSource, setFormSource] = useState<FormSource | null>(null); const [formSource, setFormSource] = useState<FormSource | null>(null);
const [search, setSearch] = useState(""); const [search, setSearch] = useState("");
const [selectedDatabaseId, setSelectedDatabaseId] = useState<string | null>(null);
const [busyAction, setBusyAction] = useState<DatabaseBusyAction>(null); const [busyAction, setBusyAction] = useState<DatabaseBusyAction>(null);
const [stale, setStale] = useState(false); const [stale, setStale] = useState(false);
const [staleBannerOpen, setStaleBannerOpen] = useState(true); const [staleBannerOpen, setStaleBannerOpen] = useState(true);
@@ -171,6 +186,11 @@ export function DatabaseManagementPage({
busy: false, busy: false,
}); });
const [tableNestedNavigationActive, setTableNestedNavigationActive] = useState(false); const [tableNestedNavigationActive, setTableNestedNavigationActive] = useState(false);
const [tableNestedNavigationBack, setTableNestedNavigationBack] = useState<(() => void) | null>(null);
const handleTableNestedNavigationChange = useCallback((active: boolean, onBack?: () => void) => {
setTableNestedNavigationActive(active);
setTableNestedNavigationBack(() => (active ? onBack ?? null : null));
}, []);
const [activeSyncRun, setActiveSyncRun] = useState<CatalogSyncRun | null>(null); const [activeSyncRun, setActiveSyncRun] = useState<CatalogSyncRun | null>(null);
const [syncHistoryDatabaseId, setSyncHistoryDatabaseId] = useState<string | null>(null); const [syncHistoryDatabaseId, setSyncHistoryDatabaseId] = useState<string | null>(null);
const [syncDrawerOpen, setSyncDrawerOpen] = useState(false); const [syncDrawerOpen, setSyncDrawerOpen] = useState(false);
@@ -184,6 +204,12 @@ export function DatabaseManagementPage({
suggestions: SensitiveDataSuggestion[]; suggestions: SensitiveDataSuggestion[];
} | null>(null); } | null>(null);
useEffect(() => {
if (import.meta.env.MODE === "test") return;
const persisted = screen.kind === "tables" || screen.kind === "relationships" ? screen : { kind: "list" };
window.localStorage.setItem(DATABASE_SCREEN_STORAGE_KEY, JSON.stringify(persisted));
}, [screen]);
const originRef = useRef<HTMLElement | null>(null); const originRef = useRef<HTMLElement | null>(null);
const searchInputRef = useRef<HTMLInputElement>(null); const searchInputRef = useRef<HTMLInputElement>(null);
const formHeadingRef = useRef<HTMLHeadingElement>(null); const formHeadingRef = useRef<HTMLHeadingElement>(null);
@@ -284,6 +310,7 @@ export function DatabaseManagementPage({
setPartialSecretFailure(null); setPartialSecretFailure(null);
setTablesNavigationState({ dirty: false, busy: false }); setTablesNavigationState({ dirty: false, busy: false });
setTableNestedNavigationActive(false); setTableNestedNavigationActive(false);
setTableNestedNavigationBack(null);
restoreListFocus(); restoreListFocus();
}, [restoreListFocus]); }, [restoreListFocus]);
@@ -346,6 +373,7 @@ export function DatabaseManagementPage({
setPartialSecretFailure(null); setPartialSecretFailure(null);
setTablesNavigationState({ dirty: false, busy: false }); setTablesNavigationState({ dirty: false, busy: false });
setTableNestedNavigationActive(false); setTableNestedNavigationActive(false);
setTableNestedNavigationBack(null);
setScreen({ kind: "tables", workspaceId: row.workspaceId }); setScreen({ kind: "tables", workspaceId: row.workspaceId });
}, []); }, []);
@@ -358,6 +386,8 @@ export function DatabaseManagementPage({
setStale(false); setStale(false);
setPartialSecretFailure(null); setPartialSecretFailure(null);
setTablesNavigationState({ dirty: false, busy: false }); setTablesNavigationState({ dirty: false, busy: false });
setTableNestedNavigationActive(false);
setTableNestedNavigationBack(null);
setScreen({ kind: "relationships", workspaceId: row.workspaceId }); setScreen({ kind: "relationships", workspaceId: row.workspaceId });
}, []); }, []);
@@ -856,6 +886,11 @@ export function DatabaseManagementPage({
} }
}, [invalidateCatalogMetrics, queryClient]); }, [invalidateCatalogMetrics, queryClient]);
const clearActiveCatalogTables = useCallback(async () => {
if (!activeRow?.id) return;
await deleteSelectedMetadata([activeRow], "tables");
}, [activeRow, deleteSelectedMetadata]);
const syncDatabase = useCallback(async (scope: CatalogSyncScope) => { const syncDatabase = useCallback(async (scope: CatalogSyncScope) => {
if (!activeRow?.id || !activeRow.configured) return; if (!activeRow?.id || !activeRow.configured) return;
try { try {
@@ -1038,13 +1073,12 @@ export function DatabaseManagementPage({
onOpenOverview={() => openOverview(activeRow)} onOpenOverview={() => openOverview(activeRow)}
onOpenRelationships={() => openRelationships(activeRow)} onOpenRelationships={() => openRelationships(activeRow)}
onNavigationStateChange={setTablesNavigationState} onNavigationStateChange={setTablesNavigationState}
onNestedNavigationChange={setTableNestedNavigationActive} onNestedNavigationChange={handleTableNestedNavigationChange}
onRunStarted={rememberSyncRun} onRunStarted={rememberSyncRun}
onOpenSync={() => openSync(activeRow)} onOpenSync={() => openSync(activeRow)}
onOpenDescriptionHistory={openDescriptionGenerationHistory}
onOpenSensitiveHistory={openSensitiveDataSuggestionHistory}
onDescriptionGenerationRunStarted={rememberDescriptionGenerationRun} onDescriptionGenerationRunStarted={rememberDescriptionGenerationRun}
onSuggestSensitive={suggestActiveDatabaseSensitiveFields} onSuggestSensitive={suggestActiveDatabaseSensitiveFields}
onClearCatalogTables={clearActiveCatalogTables}
onCatalogMetricsChanged={invalidateCatalogMetrics} onCatalogMetricsChanged={invalidateCatalogMetrics}
/> />
) : screen.kind === "relationships" && relationshipsVisible ? ( ) : screen.kind === "relationships" && relationshipsVisible ? (
@@ -1088,12 +1122,14 @@ export function DatabaseManagementPage({
onEdit={editRow} onEdit={editRow}
onDelete={deleteRow} onDelete={deleteRow}
onOpenSync={openSync} onOpenSync={openSync}
onSelectionChange={(selected) => setSelectedDatabaseId(selected.length === 1 ? selected[0].id ?? null : null)}
onTestSelected={testSelected} onTestSelected={testSelected}
onSyncSelected={syncSelected} onSyncSelected={syncSelected}
selectedMetadataModel={selectedMetadataModelAvailable ? selectedMetadataModel : null} selectedMetadataModel={selectedMetadataModelAvailable ? selectedMetadataModel : null}
descriptionGenerationActive={descriptionGenerationActive} descriptionGenerationActive={descriptionGenerationActive}
onGenerateDescriptions={generateDatabaseDescriptions} onGenerateDescriptions={generateDatabaseDescriptions}
onSuggestSensitive={suggestDatabaseSensitiveFields} onSuggestSensitive={suggestDatabaseSensitiveFields}
sensitiveDataSuggestionRuns={sensitiveDataSuggestionRuns}
onDeleteMetadataSelected={deleteSelectedMetadata} onDeleteMetadataSelected={deleteSelectedMetadata}
onRefresh={refreshList} onRefresh={refreshList}
/> />
@@ -1125,6 +1161,45 @@ export function DatabaseManagementPage({
actions={( actions={(
<> <>
{fleetMetadataModelControl} {fleetMetadataModelControl}
<Button
type="button"
variant="outline"
size="sm"
className="whitespace-nowrap"
aria-label="View schema synchronization logs"
title="View schema synchronization progress and history"
disabled={!canManage || (!selectedDatabaseId && !activeSyncRun?.databaseId)}
onClick={() => openSync(selectedDatabaseId ? rows.find((row) => row.id === selectedDatabaseId) : undefined)}
>
<RefreshCw aria-hidden="true" />
{currentActiveRun ? "Schema sync active" : "Schema sync logs"}
</Button>
<Button
type="button"
variant="outline"
size="sm"
className="whitespace-nowrap"
aria-label="View AI description generation logs"
title="View AI description generation progress and history"
disabled={!canManage}
onClick={openDescriptionGenerationHistory}
>
<History aria-hidden="true" />
{descriptionGenerationActive ? "AI descriptions active" : "AI description logs"}
</Button>
<Button
type="button"
variant="outline"
size="sm"
className="whitespace-nowrap"
aria-label="View sensitive analysis progress"
title="View sensitive analysis progress and history"
disabled={!canManage}
onClick={openSensitiveDataSuggestionHistory}
>
<History aria-hidden="true" />
{observedActiveSensitiveDataSuggestionRun ? "Sensitive analysis active" : "Sensitive analysis history"}
</Button>
</> </>
)} )}
/> />
@@ -1144,9 +1219,17 @@ export function DatabaseManagementPage({
navigation={( navigation={(
<FleetLedgerBreadcrumb <FleetLedgerBreadcrumb
items={fleetBreadcrumb} items={fleetBreadcrumb}
back={screen.kind === "relationships" || (screen.kind === "tables" && !tableNestedNavigationActive) back={screen.kind === "relationships"
? { label: "Back to databases", onBack: showList, disabled: navigationBusy } ? { label: "Back to databases", onBack: showList, disabled: navigationBusy }
: undefined} : screen.kind === "tables" && tableNestedNavigationActive
? {
label: "Back to tables",
onBack: () => tableNestedNavigationBack?.(),
disabled: navigationBusy || !tableNestedNavigationBack,
}
: screen.kind === "tables"
? { label: "Back to databases", onBack: showList, disabled: navigationBusy }
: undefined}
/> />
)} )}
children={<FleetLedgerContent busy={isLoading || isFetching}>{fleetContent}</FleetLedgerContent>} children={<FleetLedgerContent busy={isLoading || isFetching}>{fleetContent}</FleetLedgerContent>}
@@ -1246,10 +1329,12 @@ export function DatabaseManagementPage({
onEdit={editRow} onEdit={editRow}
onDelete={deleteRow} onDelete={deleteRow}
onOpenSync={openSync} onOpenSync={openSync}
onSelectionChange={(selected) => setSelectedDatabaseId(selected.length === 1 ? selected[0].id ?? null : null)}
onTestSelected={testSelected} onTestSelected={testSelected}
onSyncSelected={syncSelected} onSyncSelected={syncSelected}
selectedMetadataModel={selectedMetadataModelAvailable ? selectedMetadataModel : null} selectedMetadataModel={selectedMetadataModelAvailable ? selectedMetadataModel : null}
descriptionGenerationActive={descriptionGenerationActive} descriptionGenerationActive={descriptionGenerationActive}
sensitiveDataSuggestionRuns={sensitiveDataSuggestionRuns}
onGenerateDescriptions={generateDatabaseDescriptions} onGenerateDescriptions={generateDatabaseDescriptions}
onSuggestSensitive={suggestDatabaseSensitiveFields} onSuggestSensitive={suggestDatabaseSensitiveFields}
onDeleteMetadataSelected={deleteSelectedMetadata} onDeleteMetadataSelected={deleteSelectedMetadata}
@@ -1304,10 +1389,9 @@ export function DatabaseManagementPage({
onNavigationStateChange={setTablesNavigationState} onNavigationStateChange={setTablesNavigationState}
onRunStarted={rememberSyncRun} onRunStarted={rememberSyncRun}
onOpenSync={() => openSync(activeRow)} onOpenSync={() => openSync(activeRow)}
onOpenDescriptionHistory={openDescriptionGenerationHistory}
onOpenSensitiveHistory={openSensitiveDataSuggestionHistory}
onDescriptionGenerationRunStarted={rememberDescriptionGenerationRun} onDescriptionGenerationRunStarted={rememberDescriptionGenerationRun}
onSuggestSensitive={suggestActiveDatabaseSensitiveFields} onSuggestSensitive={suggestActiveDatabaseSensitiveFields}
onClearCatalogTables={clearActiveCatalogTables}
onCatalogMetricsChanged={invalidateCatalogMetrics} onCatalogMetricsChanged={invalidateCatalogMetrics}
/> />
) : null} ) : null}
@@ -285,7 +285,12 @@ export function CatalogSyncDrawer({
open open
ariaLabel="Schema synchronization" ariaLabel="Schema synchronization"
eyebrow="Schema synchronization" eyebrow="Schema synchronization"
title={run ? `${stateLabel(run)[0].toUpperCase()}${stateLabel(run).slice(1)}` : "Synchronization history"} title={run ? (
<span className="inline-flex items-center gap-2">
{!terminal(run) ? <LoaderCircle className="size-5 animate-spin" aria-hidden="true" /> : null}
{stateLabel(run)[0].toUpperCase()}{stateLabel(run).slice(1)}
</span>
) : "Synchronization history"}
description={run ? `${run.scope} · ${elapsed(run, now)}` : undefined} description={run ? `${run.scope} · ${elapsed(run, now)}` : undefined}
onClose={onClose} onClose={onClose}
closeLabel="Close synchronization drawer" closeLabel="Close synchronization drawer"
@@ -30,9 +30,7 @@ interface Props {
descriptionGenerationActive: boolean; descriptionGenerationActive: boolean;
onDescriptionGenerationRunStarted: (run: DescriptionGenerationRun) => void; onDescriptionGenerationRunStarted: (run: DescriptionGenerationRun) => void;
onNavigationStateChange: (state: DatabaseNavigationState) => void; onNavigationStateChange: (state: DatabaseNavigationState) => void;
onSync: () => void;
onSuggestSensitive: (selection: SensitiveDataSuggestionRequest, scopeLabel: string) => Promise<void>; onSuggestSensitive: (selection: SensitiveDataSuggestionRequest, scopeLabel: string) => Promise<void>;
bindingReady?: boolean;
catalogOperationActive?: boolean; catalogOperationActive?: boolean;
onCatalogMetricsChanged?: () => void | Promise<void>; onCatalogMetricsChanged?: () => void | Promise<void>;
presentation?: "legacy" | "fleet"; presentation?: "legacy" | "fleet";
@@ -79,6 +77,7 @@ function SensitiveCell({ data, context }: ICellRendererParams<CatalogColumn, unk
checked={data.sensitive} checked={data.sensitive}
disabled={!context.canManage || context.busy} disabled={!context.canManage || context.busy}
aria-label={`Sensitive data for ${data.name}`} aria-label={`Sensitive data for ${data.name}`}
title="Mark this column as sensitive; this does not select it for actions."
onChange={(event) => context.onSensitiveChange(data, event.target.checked)} onChange={(event) => context.onSensitiveChange(data, event.target.checked)}
onClick={(event) => event.stopPropagation()} onClick={(event) => event.stopPropagation()}
/> />
@@ -94,9 +93,7 @@ export function DatabaseColumns({
descriptionGenerationActive, descriptionGenerationActive,
onDescriptionGenerationRunStarted, onDescriptionGenerationRunStarted,
onNavigationStateChange, onNavigationStateChange,
onSync,
onSuggestSensitive, onSuggestSensitive,
bindingReady = true,
catalogOperationActive = false, catalogOperationActive = false,
onCatalogMetricsChanged, onCatalogMetricsChanged,
presentation = "legacy", presentation = "legacy",
@@ -291,7 +288,6 @@ export function DatabaseColumns({
| "generate-descriptions" | "generate-descriptions"
| "consolidate-descriptions" | "consolidate-descriptions"
| "suggest-sensitive" | "suggest-sensitive"
| "sync-columns"
| "save-sensitive"; | "save-sensitive";
const fleetActions: readonly FleetActionOption<FleetColumnAction>[] = [ const fleetActions: readonly FleetActionOption<FleetColumnAction>[] = [
@@ -347,23 +343,6 @@ export function DatabaseColumns({
? "Another action is running." ? "Another action is running."
: undefined, : undefined,
}, },
{
id: "sync-columns",
label: "Synchronize columns",
group: "Synchronization",
scopeLabel: "All columns in this table",
runLabel: "Synchronize",
disabled: !canManage || !bindingReady || catalogOperationActive || busy,
disabledReason: !canManage
? "You do not have permission to synchronize the catalog."
: !bindingReady
? "Test the current database binding first."
: catalogOperationActive
? "Wait for the active catalog operation to finish."
: busy
? "Another action is running."
: undefined,
},
{ {
id: "save-sensitive", id: "save-sensitive",
label: "Save sensitive-field changes", label: "Save sensitive-field changes",
@@ -386,7 +365,6 @@ export function DatabaseColumns({
if (action === "generate-descriptions") await generateDescriptions(); if (action === "generate-descriptions") await generateDescriptions();
else if (action === "consolidate-descriptions") await consolidateDescriptions(); else if (action === "consolidate-descriptions") await consolidateDescriptions();
else if (action === "suggest-sensitive") await suggestSensitive(); else if (action === "suggest-sensitive") await suggestSensitive();
else if (action === "sync-columns") onSync();
else await saveSensitive(); else await saveSensitive();
}; };
@@ -397,7 +375,7 @@ export function DatabaseColumns({
const columns = useMemo<ColDef<CatalogColumn>[]>(() => [ const columns = useMemo<ColDef<CatalogColumn>[]>(() => [
{ field: "ordinalPosition", headerName: "#", width: 64, maxWidth: 64, filter: "agNumberColumnFilter" }, { field: "ordinalPosition", headerName: "#", width: 64, maxWidth: 64, filter: "agNumberColumnFilter" },
{ field: "sensitive", headerName: "Sensitive", minWidth: 110, width: 110, sortable: false, filter: false, resizable: false, cellRenderer: SensitiveCell }, { field: "sensitive", headerName: "Sensitive", headerTooltip: "Mark this column as sensitive; this does not select it for actions.", minWidth: 110, width: 110, sortable: false, filter: false, resizable: false, cellRenderer: SensitiveCell },
{ field: "name", headerName: "Name", minWidth: 190, flex: 1, cellClass: "font-mono text-xs" }, { field: "name", headerName: "Name", minWidth: 190, flex: 1, cellClass: "font-mono text-xs" },
{ field: "dataType", headerName: "Type", minWidth: 150, flex: 0.8, cellClass: "font-mono text-xs" }, { field: "dataType", headerName: "Type", minWidth: 150, flex: 0.8, cellClass: "font-mono text-xs" },
{ headerName: "Keys", minWidth: 125, width: 125, sortable: false, filter: false, cellRenderer: KeyCell }, { headerName: "Keys", minWidth: 125, width: 125, sortable: false, filter: false, cellRenderer: KeyCell },
@@ -512,7 +490,6 @@ export function DatabaseColumns({
<span className="text-xs tabular-nums text-muted-foreground">{data.length}</span> <span className="text-xs tabular-nums text-muted-foreground">{data.length}</span>
{changedSensitiveColumns.length > 0 ? <Button type="button" disabled={!canManage || busy} onClick={() => void saveSensitive()}><Save />{sensitiveAction === "save" ? "Saving…" : "Save sensitive fields"}</Button> : null} {changedSensitiveColumns.length > 0 ? <Button type="button" disabled={!canManage || busy} onClick={() => void saveSensitive()}><Save />{sensitiveAction === "save" ? "Saving…" : "Save sensitive fields"}</Button> : null}
<Button type="button" variant="outline" disabled={isFetching || busy} onClick={() => void refetch()}><RefreshCw className={isFetching ? "animate-spin" : ""} />Refresh</Button> <Button type="button" variant="outline" disabled={isFetching || busy} onClick={() => void refetch()}><RefreshCw className={isFetching ? "animate-spin" : ""} />Refresh</Button>
<Button type="button" disabled={!canManage || busy} onClick={onSync}><RefreshCw />Sync columns</Button>
</> </>
) )
)} )}
@@ -529,6 +506,17 @@ export function DatabaseColumns({
defaultColDef={{ sortable: true, filter: true, resizable: true }} defaultColDef={{ sortable: true, filter: true, resizable: true }}
getRowId={({ data: row }) => row.id} getRowId={({ data: row }) => row.id}
rowSelection={{ mode: "multiRow", selectAll: "filtered", enableClickSelection: false }} rowSelection={{ mode: "multiRow", selectAll: "filtered", enableClickSelection: false }}
selectionColumnDef={{
headerName: "Select",
headerTooltip: "Select columns for actions",
width: 72,
minWidth: 72,
maxWidth: 72,
pinned: "left",
sortable: false,
resizable: false,
suppressMovable: true,
}}
onSelectionChanged={({ api }) => setSelectedIds(api.getSelectedRows().map((column) => column.id))} onSelectionChanged={({ api }) => setSelectedIds(api.getSelectedRows().map((column) => column.id))}
rowHeight={44} rowHeight={44}
headerHeight={38} headerHeight={38}
@@ -115,7 +115,6 @@ test("Fleet columns distinguish a failed query from an empty catalog and retry i
descriptionGenerationActive={false} descriptionGenerationActive={false}
onDescriptionGenerationRunStarted={noop} onDescriptionGenerationRunStarted={noop}
onNavigationStateChange={noop} onNavigationStateChange={noop}
onSync={noop}
onSuggestSensitive={noopAsync} onSuggestSensitive={noopAsync}
/>, />,
); );
@@ -12,7 +12,6 @@ import { Button } from "../../components/ui/button";
import type { import type {
CatalogDatabase, CatalogDatabase,
CatalogSecretName, CatalogSecretName,
CatalogSyncScope,
CatalogSyncRun, CatalogSyncRun,
DatabaseBinding, DatabaseBinding,
DatabaseTransport, DatabaseTransport,
@@ -23,7 +22,7 @@ import type {
DatabaseFormMode, DatabaseFormMode,
} from "./model"; } from "./model";
import { statusLabel } from "./model"; import { statusLabel } from "./model";
import { DatabaseSyncMenu } from "./DatabaseSyncMenu"; import { DatabaseSyncMenu, type DatabaseSyncScope } from "./DatabaseSyncMenu";
const inputClass = "h-9 w-full rounded-md border border-input bg-card px-3 text-sm outline-none transition focus:border-primary/60 focus:ring-3 focus:ring-ring/15 read-only:bg-muted/40 read-only:text-muted-foreground disabled:bg-muted disabled:text-muted-foreground"; const inputClass = "h-9 w-full rounded-md border border-input bg-card px-3 text-sm outline-none transition focus:border-primary/60 focus:ring-3 focus:ring-ring/15 read-only:bg-muted/40 read-only:text-muted-foreground disabled:bg-muted disabled:text-muted-foreground";
const labelClass = "grid min-w-0 gap-1.5 text-xs font-medium text-foreground"; const labelClass = "grid min-w-0 gap-1.5 text-xs font-medium text-foreground";
@@ -135,7 +134,7 @@ interface DatabaseFormProps {
onRetrySecrets: () => void; onRetrySecrets: () => void;
onOpenTables: (origin: HTMLButtonElement) => void; onOpenTables: (origin: HTMLButtonElement) => void;
onOpenRelationships: () => void; onOpenRelationships: () => void;
onSync: (scope: CatalogSyncScope) => void; onSync: (scope: DatabaseSyncScope) => void;
onOpenSync: () => void; onOpenSync: () => void;
onOpenDescriptionHistory?: () => void; onOpenDescriptionHistory?: () => void;
onOpenSensitiveHistory?: () => void; onOpenSensitiveHistory?: () => void;
@@ -1,4 +1,5 @@
import { useEffect, useMemo, useRef, useState, type RefObject } from "react"; import { useEffect, useMemo, useRef, useState, type RefObject } from "react";
import { useQueries } from "@tanstack/react-query";
import { Menu } from "@base-ui/react/menu"; import { Menu } from "@base-ui/react/menu";
import { AgGridReact } from "ag-grid-react"; import { AgGridReact } from "ag-grid-react";
import { import {
@@ -11,13 +12,15 @@ import "ag-grid-community/styles/ag-grid.css";
import "ag-grid-community/styles/ag-theme-alpine.css"; import "ag-grid-community/styles/ag-theme-alpine.css";
import { ChevronDown, Eye, History, Info, Link2, Pencil, RefreshCw, Rows3, Sparkles, Trash2, X } from "lucide-react"; import { ChevronDown, Eye, History, Info, Link2, Pencil, RefreshCw, Rows3, Sparkles, Trash2, X } from "lucide-react";
import { Button } from "../../components/ui/button"; import { Button } from "../../components/ui/button";
import { getCatalogMetrics } from "../../api/catalog-databases";
import type { import type {
CatalogDatabase, CatalogDatabase,
CatalogMetrics,
SensitiveDataSuggestionRun,
CatalogDatabaseMetadataDeleteTarget, CatalogDatabaseMetadataDeleteTarget,
CatalogSyncScope,
DescriptionGenerationScope, DescriptionGenerationScope,
} from "../../api/catalog-databases"; } from "../../api/catalog-databases";
import { databaseSyncItemClass, databaseSyncScopes } from "./DatabaseSyncMenu"; import { databaseSyncItemClass, databaseSyncScopes, type DatabaseSyncScope } from "./DatabaseSyncMenu";
import { FleetActionSelector, type FleetActionOption } from "./FleetActionSelector"; import { FleetActionSelector, type FleetActionOption } from "./FleetActionSelector";
import { NO_METADATA_GENERATION_LLM_MODEL_MESSAGE } from "./MetadataGenerationModelSelector"; import { NO_METADATA_GENERATION_LLM_MODEL_MESSAGE } from "./MetadataGenerationModelSelector";
@@ -37,10 +40,12 @@ interface DatabaseGridProps {
onEdit: (row: CatalogDatabase, origin: HTMLButtonElement) => void; onEdit: (row: CatalogDatabase, origin: HTMLButtonElement) => void;
onDelete: (row: CatalogDatabase, origin: HTMLButtonElement) => void; onDelete: (row: CatalogDatabase, origin: HTMLButtonElement) => void;
onOpenSync: (row: CatalogDatabase) => void; onOpenSync: (row: CatalogDatabase) => void;
onSelectionChange?: (rows: CatalogDatabase[]) => void;
onTestSelected: (rows: CatalogDatabase[]) => Promise<void>; onTestSelected: (rows: CatalogDatabase[]) => Promise<void>;
onSyncSelected: (rows: CatalogDatabase[], scope: CatalogSyncScope) => Promise<void>; onSyncSelected: (rows: CatalogDatabase[], scope: DatabaseSyncScope) => Promise<void>;
selectedMetadataModel: string | null; selectedMetadataModel: string | null;
descriptionGenerationActive: boolean; descriptionGenerationActive: boolean;
sensitiveDataSuggestionRuns?: SensitiveDataSuggestionRun[];
onGenerateDescriptions: ( onGenerateDescriptions: (
rows: CatalogDatabase[], rows: CatalogDatabase[],
scope: Extract<DescriptionGenerationScope, "all" | "missing">, scope: Extract<DescriptionGenerationScope, "all" | "missing">,
@@ -68,7 +73,6 @@ interface DatabaseGridContext {
type DatabaseFleetAction = type DatabaseFleetAction =
| "test" | "test"
| "sync-tables" | "sync-tables"
| "sync-columns"
| "sync-relationships" | "sync-relationships"
| "sync-all" | "sync-all"
| "generate-all" | "generate-all"
@@ -124,87 +128,63 @@ function StateCell({
); );
} }
function evidenceStatus(row: CatalogDatabase): { label: string; tone: StateTone } { function endpoint(host: string | undefined, port: number | undefined): string {
if (!row.workspaceEvidence) { if (!host) return "Endpoint not configured";
return { label: "Evidence state unavailable", tone: "warning" }; return port ? `${host}:${port}` : host;
}
switch (row.workspaceEvidence.state) {
case "materialized_current_revision":
return { label: "Evidence materialized", tone: "success" };
case "configuration_required":
return { label: "Evidence credentials required", tone: "warning" };
case "configured_unverified":
return { label: "Evidence configured, unverified", tone: "info" };
case "workspace_unavailable":
return { label: "Evidence unavailable", tone: "danger" };
default:
return { label: "No Evidence declared", tone: "neutral" };
}
} }
function RevisionEvidenceCell({ row }: { row: CatalogDatabase }) { function accessSummary(row: CatalogDatabase): { label: string; detail: string } {
if (!row.workspaceRevision) { const transport = row.configured
return <StateCell label="Workspace missing" detail="Revision and Evidence unavailable" tone="danger" />; ? row.binding.transport
: row.runtimeBinding?.transport ?? row.binding.transport;
if (transport === "rest_api") {
if (!row.binding.baseUrl) return { label: "REST", detail: "Endpoint not configured" };
try {
return { label: "REST", detail: new URL(row.binding.baseUrl).host };
} catch {
return { label: "REST", detail: row.binding.baseUrl };
}
} }
const evidence = evidenceStatus(row); if (transport === "ssh_tunnel") {
const source = row.workspaceEvidence?.sourceType ? ` · ${row.workspaceEvidence.sourceType}` : ""; return { label: "SSH", detail: endpoint(row.binding.sshHost, row.binding.sshPort) };
}
return {
label: "Direct PostgreSQL",
detail: endpoint(row.binding.host, row.binding.port),
};
}
function coverageStatus(total: number, complete: number): { label: string; detail: string; tone: StateTone } {
if (total === 0) return { label: "Not started", detail: "No catalog items", tone: "danger" };
if (complete >= total) return { label: "Complete", detail: `${complete}/${total}`, tone: "success" };
if (complete > 0) return { label: "Partial", detail: `${complete}/${total}`, tone: "warning" };
return { label: "Not started", detail: `0/${total}`, tone: "danger" };
}
function CatalogStatusCells({ row, metrics, sensitiveRun }: { row: CatalogDatabase; metrics?: CatalogMetrics; sensitiveRun?: SensitiveDataSuggestionRun }) {
const access = accessSummary(row);
const synchronized = row.schemaSyncedVersion === row.version;
const syncStatus = !row.schemaSyncedVersion
? { label: "Not synchronized", detail: "Schema not synchronized", tone: "danger" as const }
: synchronized
? { label: "Synchronized", detail: "Current schema", tone: "success" as const }
: { label: "Partial", detail: "Schema revision changed", tone: "warning" as const };
const descriptions = coverageStatus(metrics?.descriptionTargets ?? 0, metrics?.describedTargets ?? 0);
const sensitiveProcessed = sensitiveRun ? sensitiveRun.suggestedSensitive + sensitiveRun.suggestedNonSensitive : 0;
const sensitive = !sensitiveRun
? { label: "Not started", detail: "No analysis run", tone: "danger" as const }
: sensitiveRun.status === "completed" && sensitiveProcessed >= sensitiveRun.total
? { label: "Complete", detail: `${sensitiveProcessed}/${sensitiveRun.total}`, tone: "success" as const }
: ["failed", "interrupted"].includes(sensitiveRun.status)
? { label: "Incomplete", detail: `${sensitiveProcessed}/${sensitiveRun.total}`, tone: "danger" as const }
: { label: "Partial", detail: `${sensitiveProcessed}/${sensitiveRun.total}`, tone: "warning" as const };
return ( return (
<StateCell <div className="grid h-full min-w-[540px] grid-cols-4 gap-3 py-1">
label={`Active revision ${row.workspaceRevision.commit.slice(0, 7)}`} <StateCell label={access.label} detail={access.detail} tone="info" />
detail={`${evidence.label}${source}`} <StateCell label={syncStatus.label} detail={syncStatus.detail} tone={syncStatus.tone} />
tone={evidence.tone} <StateCell label={descriptions.label} detail={`Descriptions · ${descriptions.detail}`} tone={descriptions.tone} />
/> <StateCell label={sensitive.label} detail={`Sensitive · ${sensitive.detail}`} tone={sensitive.tone} />
); </div>
}
function transportLabel(row: CatalogDatabase): string {
const transport = row.runtimeBinding?.transport;
if (transport === "postgres_direct") return "Direct PostgreSQL";
if (transport === "rest_api") return "REST API";
if (transport === "ssh_tunnel") return "SSH tunnel";
return "No runtime binding";
}
function RuntimeBindingCell({ row }: { row: CatalogDatabase }) {
if (!row.runtimeBinding) {
return <StateCell label="Unavailable" detail="Workspace is not in the active revision" tone="danger" />;
}
if (!row.runtimeBinding.sessionTransportSupported) {
return <StateCell label="Unsupported for NL→SQL" detail={transportLabel(row)} tone="danger" />;
}
if (row.runtimeBinding.configurationState === "configuration_required") {
return <StateCell label="Configuration required" detail={transportLabel(row)} tone="warning" />;
}
return <StateCell label="Ready" detail={transportLabel(row)} tone="success" />;
}
function catalogConnectionDetail(row: CatalogDatabase): { label: string; className?: string } {
if (row.activeSyncRun) return { label: "Metadata sync in progress", className: "text-primary" };
if (row.testedVersion !== undefined && row.testedVersion !== row.version) {
return { label: "Connection retest required", className: "text-amber-700 dark:text-amber-300" };
}
if (row.connectionStatus === "reachable") return { label: "Connection reachable" };
if (row.connectionStatus === "failed") {
return { label: "Connection test failed", className: "text-destructive" };
}
return { label: "Connection not tested" };
}
function MetadataCatalogCell({ row }: { row: CatalogDatabase }) {
if (!row.workspaceAvailable) {
return <StateCell label="Orphaned configuration" detail="Workspace missing from repository" tone="danger" />;
}
if (!row.configured) {
return <StateCell label="Not configured" detail="Configure this workspace" tone="neutral" />;
}
const connection = catalogConnectionDetail(row);
return (
<StateCell
label="Configured"
detail={connection.label}
detailClassName={connection.className}
tone="success"
/>
); );
} }
@@ -259,11 +239,6 @@ function DatabaseActionsCell({
return ( return (
<div className="flex h-full items-center justify-end gap-0.5" onClick={(event) => event.stopPropagation()}> <div className="flex h-full items-center justify-end gap-0.5" onClick={(event) => event.stopPropagation()}>
{data.activeSyncRun || (context.presentation === "fleet" && data.configured && data.id) ? (
<Button type="button" variant="ghost" size="icon-lg" className={context.presentation === "fleet" ? "thot-fleet-icon-action" : undefined} data-tooltip={context.presentation === "fleet" ? "Synchronization history" : undefined} title={context.presentation === "legacy" ? `View synchronization for ${data.workspaceName}` : undefined} aria-label={`View synchronization history for ${data.workspaceName}`} onClick={() => context.onOpenSync(data)}>
{data.activeSyncRun ? <RefreshCw className="animate-spin" aria-hidden="true" /> : <History aria-hidden="true" />}
</Button>
) : null}
{context.presentation === "fleet" && context.onOpenTables ? ( {context.presentation === "fleet" && context.onOpenTables ? (
<span className="thot-fleet-icon-action inline-flex" data-tooltip="Tables"> <span className="thot-fleet-icon-action inline-flex" data-tooltip="Tables">
<Button <Button
@@ -357,16 +332,37 @@ export function DatabaseGrid({
onEdit, onEdit,
onDelete, onDelete,
onOpenSync, onOpenSync,
onSelectionChange,
onTestSelected, onTestSelected,
onSyncSelected, onSyncSelected,
selectedMetadataModel, selectedMetadataModel,
descriptionGenerationActive, descriptionGenerationActive,
sensitiveDataSuggestionRuns = [],
onGenerateDescriptions, onGenerateDescriptions,
onSuggestSensitive, onSuggestSensitive,
onDeleteMetadataSelected, onDeleteMetadataSelected,
onRefresh, onRefresh,
presentation = "legacy", presentation = "legacy",
}: DatabaseGridProps) { }: DatabaseGridProps) {
const metricQueries = useQueries({
queries: rows.filter((row) => row.id).map((row) => ({
queryKey: ["catalog-metrics", row.id],
queryFn: () => getCatalogMetrics(row.id!),
enabled: canManage,
staleTime: 10_000,
})),
});
const metricsByDatabase = useMemo(() => {
const result: Record<string, CatalogMetrics> = {};
rows.filter((row) => row.id).forEach((row, index) => {
const metrics = metricQueries[index]?.data;
if (metrics && row.id) result[row.id] = metrics;
});
return result;
}, [metricQueries, rows]);
const sensitiveRunsByDatabase = useMemo(() => new Map(
rows.filter((row) => row.id).map((row) => [row.id!, sensitiveDataSuggestionRuns.find((run) => run.databaseId === row.id)]),
), [rows, sensitiveDataSuggestionRuns]);
const compact = useCompactViewport(); const compact = useCompactViewport();
const gridRef = useRef<AgGridReact<CatalogDatabase>>(null); const gridRef = useRef<AgGridReact<CatalogDatabase>>(null);
const actionsTriggerRef = useRef<HTMLButtonElement>(null); const actionsTriggerRef = useRef<HTMLButtonElement>(null);
@@ -390,68 +386,16 @@ export function DatabaseGrid({
flex: compact ? undefined : 1.35, flex: compact ? undefined : 1.35,
width: compact ? 112 : undefined, width: compact ? 112 : undefined,
}; };
const revisionEvidence: ColDef<CatalogDatabase> = { const status: ColDef<CatalogDatabase> = {
headerName: "Revision / Evidence", headerName: "Catalog status",
minWidth: 215, minWidth: 580,
flex: 1.2, flex: 2.5,
valueGetter: ({ data }) => { cellRenderer: ({ data }: ICellRendererParams<CatalogDatabase>) => {
if (!data) return ""; const databaseId = data?.id;
const revision = data.workspaceRevision?.commit.slice(0, 7) ?? "workspace missing"; return data
return `${revision} ${evidenceStatus(data).label}`; ? <CatalogStatusCells row={data} metrics={databaseId ? metricsByDatabase[databaseId] : undefined} sensitiveRun={databaseId ? sensitiveRunsByDatabase.get(databaseId) : undefined} />
: null;
}, },
cellRenderer: ({ data }: ICellRendererParams<CatalogDatabase>) => (
data ? <RevisionEvidenceCell row={data} /> : null
),
};
const runtimeBinding: ColDef<CatalogDatabase> = {
headerName: "NL→SQL runtime",
minWidth: 175,
flex: 1,
valueGetter: ({ data }) => {
if (!data?.runtimeBinding) return "Unavailable";
if (!data.runtimeBinding.sessionTransportSupported) return "Unsupported for NL→SQL";
return data.runtimeBinding.configurationState === "ready" ? "Ready" : "Configuration required";
},
cellRenderer: ({ data }: ICellRendererParams<CatalogDatabase>) => (
data ? <RuntimeBindingCell row={data} /> : null
),
};
const database: ColDef<CatalogDatabase> = {
field: "databaseName", headerName: "Database", minWidth: 145, flex: 1,
};
const schema: ColDef<CatalogDatabase> = {
field: "schema", headerName: "Schema", minWidth: 140, flex: 0.9,
};
const endpoint: ColDef<CatalogDatabase> = {
headerName: "Endpoint",
minWidth: 190,
flex: 1.2,
valueGetter: ({ data }) => {
if (!data) return "";
if (data.binding.transport === "rest_api") return data.binding.baseUrl ?? "";
if (data.binding.transport === "ssh_tunnel") return data.binding.sshHost ?? "";
return data.binding.host ? `${data.binding.host}:${data.binding.port ?? 5432}` : "";
},
};
const metadataCatalog: ColDef<CatalogDatabase> = {
headerName: "Metadata Catalog",
minWidth: 185,
flex: 1.05,
valueGetter: ({ data }) => {
if (!data) return "";
if (!data.workspaceAvailable) return "Orphaned configuration";
return data.configured ? `Configured ${catalogConnectionDetail(data).label}` : "Not configured";
},
cellRenderer: ({ data }: ICellRendererParams<CatalogDatabase>) => (
data ? <MetadataCatalogCell row={data} /> : null
),
};
const updated: ColDef<CatalogDatabase> = {
field: "updatedAt",
headerName: "Updated",
minWidth: 170,
flex: 0.9,
valueFormatter: ({ value }) => value ? new Date(String(value)).toLocaleString() : "",
}; };
const actionsWidth = presentation === "fleet" ? 232 : 210; const actionsWidth = presentation === "fleet" ? 232 : 210;
const actions: ColDef<CatalogDatabase> = { const actions: ColDef<CatalogDatabase> = {
@@ -471,10 +415,8 @@ export function DatabaseGrid({
cellRenderer: DatabaseActionsCell, cellRenderer: DatabaseActionsCell,
}; };
return compact return [workspace, status, actions];
? [workspace, revisionEvidence, runtimeBinding, metadataCatalog, database, schema, endpoint, updated, actions] }, [compact, metricsByDatabase, presentation, sensitiveRunsByDatabase]);
: [workspace, revisionEvidence, runtimeBinding, database, schema, endpoint, metadataCatalog, updated, actions];
}, [compact, presentation]);
const configuredCount = rows.filter((row) => row.configured).length; const configuredCount = rows.filter((row) => row.configured).length;
const hiddenSelected = selectedRows.filter((row) => { const hiddenSelected = selectedRows.filter((row) => {
@@ -533,7 +475,6 @@ export function DatabaseGrid({
return [ return [
{ id: "test", label: "Test connections", group: "Connection", disabled: !canTestSelection, disabledReason: testReason }, { id: "test", label: "Test connections", group: "Connection", disabled: !canTestSelection, disabledReason: testReason },
{ id: "sync-tables", label: "Synchronize tables", group: "Synchronization", disabled: !canSyncSelection, disabledReason: syncReason }, { id: "sync-tables", label: "Synchronize tables", group: "Synchronization", disabled: !canSyncSelection, disabledReason: syncReason },
{ id: "sync-columns", label: "Synchronize all columns", group: "Synchronization", disabled: !canSyncSelection, disabledReason: syncReason },
{ id: "sync-relationships", label: "Synchronize relationships", group: "Synchronization", disabled: !canSyncSelection, disabledReason: syncReason }, { id: "sync-relationships", label: "Synchronize relationships", group: "Synchronization", disabled: !canSyncSelection, disabledReason: syncReason },
{ id: "sync-all", label: "Synchronize all", group: "Synchronization", disabled: !canSyncSelection, disabledReason: syncReason }, { id: "sync-all", label: "Synchronize all", group: "Synchronization", disabled: !canSyncSelection, disabledReason: syncReason },
{ id: "generate-missing", label: "Generate missing descriptions", group: "Descriptions", disabled: !canGenerateDescriptions, disabledReason: generationReason }, { id: "generate-missing", label: "Generate missing descriptions", group: "Descriptions", disabled: !canGenerateDescriptions, disabledReason: generationReason },
@@ -580,15 +521,10 @@ export function DatabaseGrid({
const perform = async ( const perform = async (
kind: "test" | "sync" | "generate" | "suggest" | "delete", kind: "test" | "sync" | "generate" | "suggest" | "delete",
operation: () => Promise<void>, operation: () => Promise<void>,
clearSelection = true,
) => { ) => {
setAction(kind); setAction(kind);
try { try {
await operation(); await operation();
if (clearSelection) {
gridRef.current?.api.deselectAll();
setSelectedRows([]);
}
setPendingDelete(null); setPendingDelete(null);
setPendingGenerationScope(null); setPendingGenerationScope(null);
if (kind === "delete") window.setTimeout(() => searchInputRef.current?.focus(), 0); if (kind === "delete") window.setTimeout(() => searchInputRef.current?.focus(), 0);
@@ -612,11 +548,11 @@ export function DatabaseGrid({
return; return;
} }
if (selectedAction.startsWith("sync-")) { if (selectedAction.startsWith("sync-")) {
const scope = selectedAction.slice("sync-".length) as CatalogSyncScope; const scope = selectedAction.slice("sync-".length) as DatabaseSyncScope;
await perform("sync", () => onSyncSelected(selectedRows, scope)); await perform("sync", () => onSyncSelected(selectedRows, scope));
return; return;
} }
await perform("suggest", () => onSuggestSensitive(selectedRows), false); await perform("suggest", () => onSuggestSensitive(selectedRows));
}; };
return ( return (
@@ -785,7 +721,7 @@ export function DatabaseGrid({
variant="outline" variant="outline"
disabled={!canManage || action !== null || descriptionGenerationActive} disabled={!canManage || action !== null || descriptionGenerationActive}
title={descriptionGenerationActive ? "Wait for the active description generation to finish" : undefined} title={descriptionGenerationActive ? "Wait for the active description generation to finish" : undefined}
onClick={() => void perform("suggest", () => onSuggestSensitive(selectedRows), false)} onClick={() => void perform("suggest", () => onSuggestSensitive(selectedRows))}
> >
<Sparkles />{action === "suggest" ? "Suggesting…" : "Suggest sensitive fields"} <Sparkles />{action === "suggest" ? "Suggesting…" : "Suggest sensitive fields"}
</Button> </Button>
@@ -833,7 +769,9 @@ export function DatabaseGrid({
onSelectionChanged={({ api }) => { onSelectionChanged={({ api }) => {
if (pendingDelete && action === null) setPendingDelete(null); if (pendingDelete && action === null) setPendingDelete(null);
if (pendingGenerationScope && action === null) setPendingGenerationScope(null); if (pendingGenerationScope && action === null) setPendingGenerationScope(null);
setSelectedRows(api.getSelectedRows()); const rows = api.getSelectedRows();
setSelectedRows(rows);
onSelectionChange?.(rows);
}} }}
rowHeight={52} rowHeight={52}
headerHeight={38} headerHeight={38}
@@ -3,14 +3,15 @@ import { ChevronDown, RefreshCw } from "lucide-react";
import { buttonVariants } from "../../components/ui/button"; import { buttonVariants } from "../../components/ui/button";
import type { CatalogSyncScope } from "../../api/catalog-databases"; import type { CatalogSyncScope } from "../../api/catalog-databases";
export type DatabaseSyncScope = Exclude<CatalogSyncScope, "columns">;
export const databaseSyncItemClass = [ export const databaseSyncItemClass = [
"flex cursor-default select-none items-center rounded-md px-3 py-2 text-sm outline-none", "flex cursor-default select-none items-center rounded-md px-3 py-2 text-sm outline-none",
"data-[highlighted]:bg-muted data-[disabled]:opacity-45", "data-[highlighted]:bg-muted data-[disabled]:opacity-45",
].join(" "); ].join(" ");
export const databaseSyncScopes: ReadonlyArray<{ scope: CatalogSyncScope; label: string }> = [ export const databaseSyncScopes: ReadonlyArray<{ scope: DatabaseSyncScope; label: string }> = [
{ scope: "tables", label: "Synchronize tables" }, { scope: "tables", label: "Synchronize tables" },
{ scope: "columns", label: "Synchronize all columns" },
{ scope: "relationships", label: "Synchronize relationships" }, { scope: "relationships", label: "Synchronize relationships" },
{ scope: "all", label: "Synchronize all" }, { scope: "all", label: "Synchronize all" },
]; ];
@@ -18,7 +19,7 @@ export const databaseSyncScopes: ReadonlyArray<{ scope: CatalogSyncScope; label:
interface DatabaseSyncMenuProps { interface DatabaseSyncMenuProps {
disabled: boolean; disabled: boolean;
disabledReason?: string; disabledReason?: string;
onSelect: (scope: CatalogSyncScope) => void; onSelect: (scope: DatabaseSyncScope) => void;
} }
export function DatabaseSyncMenu({ disabled, disabledReason, onSelect }: DatabaseSyncMenuProps) { export function DatabaseSyncMenu({ disabled, disabledReason, onSelect }: DatabaseSyncMenuProps) {
@@ -1,15 +1,16 @@
import { useEffect, useMemo, useRef, useState } from "react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { Menu } from "@base-ui/react/menu"; import { Menu } from "@base-ui/react/menu";
import { useQuery, useQueryClient } from "@tanstack/react-query"; import { useQuery, useQueryClient } from "@tanstack/react-query";
import { AgGridReact } from "ag-grid-react"; import { AgGridReact } from "ag-grid-react";
import type { ColDef, ICellRendererParams } from "ag-grid-community"; import type { ColDef, ICellRendererParams } from "ag-grid-community";
import { ArrowLeft, ChevronDown, Columns3, History, Pencil, RefreshCw, Save, Sparkles, Trash2, X } from "lucide-react"; import { ArrowLeft, ChevronDown, Columns3, Pencil, RefreshCw, Save, Sparkles, Trash2, X } from "lucide-react";
import { toast } from "sonner"; import { toast } from "sonner";
import { Button } from "../../components/ui/button"; import { Button } from "../../components/ui/button";
import { ApiError, apiErrorMessage } from "../../api/client"; import { ApiError, apiErrorMessage } from "../../api/client";
import { import {
consolidateCatalogDescriptions, consolidateCatalogDescriptions,
deleteCatalogTableMetadata, deleteCatalogTableMetadata,
listCatalogColumns,
listCatalogTables, listCatalogTables,
startCatalogSync, startCatalogSync,
startDescriptionGenerationRun, startDescriptionGenerationRun,
@@ -37,11 +38,10 @@ interface Props {
onOpenOverview: () => void; onOpenOverview: () => void;
onOpenRelationships: () => void; onOpenRelationships: () => void;
onNavigationStateChange: (state: DatabaseNavigationState) => void; onNavigationStateChange: (state: DatabaseNavigationState) => void;
onNestedNavigationChange?: (active: boolean) => void; onNestedNavigationChange?: (active: boolean, onBack?: () => void) => void;
onRunStarted: (run: CatalogSyncRun) => void; onRunStarted: (run: CatalogSyncRun) => void;
onOpenSync: () => void; onOpenSync: () => void;
onOpenDescriptionHistory?: () => void; onClearCatalogTables?: () => Promise<void>;
onOpenSensitiveHistory?: () => void;
onDescriptionGenerationRunStarted: (run: DescriptionGenerationRun) => void; onDescriptionGenerationRunStarted: (run: DescriptionGenerationRun) => void;
onSuggestSensitive: (selection: SensitiveDataSuggestionRequest, scopeLabel: string) => Promise<void>; onSuggestSensitive: (selection: SensitiveDataSuggestionRequest, scopeLabel: string) => Promise<void>;
onCatalogMetricsChanged?: () => void | Promise<void>; onCatalogMetricsChanged?: () => void | Promise<void>;
@@ -95,8 +95,7 @@ export function DatabaseTables({
onNestedNavigationChange, onNestedNavigationChange,
onRunStarted, onRunStarted,
onOpenSync, onOpenSync,
onOpenDescriptionHistory, onClearCatalogTables,
onOpenSensitiveHistory,
onDescriptionGenerationRunStarted, onDescriptionGenerationRunStarted,
onSuggestSensitive, onSuggestSensitive,
onCatalogMetricsChanged, onCatalogMetricsChanged,
@@ -134,9 +133,22 @@ export function DatabaseTables({
const nestedNavigationActive = presentation === "fleet" && Boolean(activeTable) && tableSection === "columns"; const nestedNavigationActive = presentation === "fleet" && Boolean(activeTable) && tableSection === "columns";
const bindingReady = database.connectionStatus === "reachable" && database.testedVersion === database.version; const bindingReady = database.connectionStatus === "reachable" && database.testedVersion === database.version;
const currentRun = activeRun ?? database.activeSyncRun; const currentRun = activeRun ?? database.activeSyncRun;
const clearAllCatalogTables = async () => {
if (!onClearCatalogTables || !window.confirm("Clear all catalog tables, columns, and relationships for this database? The source database will not be changed.")) return;
await onClearCatalogTables();
};
const leaveTable = useCallback(() => {
if (navigationBusy) return;
if (dirty && !window.confirm("Discard unsaved metadata?")) return;
setActiveTableId(null);
setTableSection("overview");
window.setTimeout(() => originRef.current?.focus(), 0);
}, [dirty, navigationBusy]);
useEffect(() => { onNavigationStateChange({ dirty, busy: navigationBusy }); }, [dirty, navigationBusy, onNavigationStateChange]); useEffect(() => { onNavigationStateChange({ dirty, busy: navigationBusy }); }, [dirty, navigationBusy, onNavigationStateChange]);
useEffect(() => { onNestedNavigationChange?.(nestedNavigationActive); }, [nestedNavigationActive, onNestedNavigationChange]); useEffect(() => {
onNestedNavigationChange?.(nestedNavigationActive, nestedNavigationActive ? leaveTable : undefined);
}, [leaveTable, nestedNavigationActive, onNestedNavigationChange]);
useEffect(() => () => { onNestedNavigationChange?.(false); }, [onNestedNavigationChange]); useEffect(() => () => { onNestedNavigationChange?.(false); }, [onNestedNavigationChange]);
useEffect(() => { useEffect(() => {
if (!activeTableId || !activeTable || editorVersion === activeTable.version || busy) return; if (!activeTableId || !activeTable || editorVersion === activeTable.version || busy) return;
@@ -168,13 +180,6 @@ export function DatabaseTables({
setStaleBannerOpen(true); setStaleBannerOpen(true);
setColumnNavigation({ dirty: false, busy: false }); setColumnNavigation({ dirty: false, busy: false });
}; };
const leaveTable = () => {
if (navigationBusy) return;
if (dirty && !window.confirm("Discard unsaved metadata?")) return;
setActiveTableId(null);
setTableSection("overview");
window.setTimeout(() => originRef.current?.focus(), 0);
};
const navigateDatabase = (target: "databases" | "overview" | "relationships") => { const navigateDatabase = (target: "databases" | "overview" | "relationships") => {
if (navigationBusy) return; if (navigationBusy) return;
if (dirty && !window.confirm("Discard unsaved metadata?")) return; if (dirty && !window.confirm("Discard unsaved metadata?")) return;
@@ -281,6 +286,28 @@ export function DatabaseTables({
toast.error(apiErrorMessage(error)); toast.error(apiErrorMessage(error));
} finally { setBusy(null); } } finally { setBusy(null); }
}; };
const generateColumnDescriptions = async () => {
if (selectedIds.length === 0 || !selectedMetadataModel) return;
setBusy("generate");
try {
const columns = (await Promise.all(selectedIds.map((tableId) => listCatalogColumns(databaseId, tableId)))).flat();
const columnIds = columns.map((column) => column.id);
if (columnIds.length === 0) {
toast.error("The selected tables have no synchronized catalog columns.");
return;
}
const run = await startDescriptionGenerationRun(
databaseId,
selectedMetadataModel,
"selected_columns",
columnIds,
);
onDescriptionGenerationRunStarted(run);
toast.success(`Column description generation started for ${columnIds.length} column${columnIds.length === 1 ? "" : "s"}`);
} catch (error) {
toast.error(apiErrorMessage(error));
} finally { setBusy(null); }
};
const suggestSensitive = async () => { const suggestSensitive = async () => {
if (selectedIds.length === 0) return; if (selectedIds.length === 0) return;
setBusy("suggest"); setBusy("suggest");
@@ -298,6 +325,7 @@ export function DatabaseTables({
type FleetTableAction = type FleetTableAction =
| "generate-descriptions" | "generate-descriptions"
| "generate-column-descriptions"
| "sync-tables" | "sync-tables"
| "sync-columns" | "sync-columns"
| "consolidate-descriptions" | "consolidate-descriptions"
@@ -324,6 +352,24 @@ export function DatabaseTables({
? "Another action is running." ? "Another action is running."
: undefined, : undefined,
}, },
{
id: "generate-column-descriptions",
label: "Generate column descriptions",
group: "Descriptions",
runLabel: "Generate",
disabled: !canManage || selectedIds.length === 0 || !selectedMetadataModel || descriptionGenerationActive || busy !== null || Boolean(currentRun),
disabledReason: !canManage
? "You do not have permission to generate descriptions."
: selectedIds.length === 0
? "Select at least one table."
: !selectedMetadataModel
? NO_METADATA_GENERATION_LLM_MODEL_MESSAGE
: descriptionGenerationActive || Boolean(currentRun)
? "Wait for the active catalog operation to finish."
: busy !== null
? "Another action is running."
: undefined,
},
{ {
id: "consolidate-descriptions", id: "consolidate-descriptions",
label: "Move generated to Description", label: "Move generated to Description",
@@ -421,6 +467,7 @@ export function DatabaseTables({
const runFleetAction = async (action: FleetTableAction) => { const runFleetAction = async (action: FleetTableAction) => {
if (action === "generate-descriptions") await generateDescriptions(); if (action === "generate-descriptions") await generateDescriptions();
else if (action === "generate-column-descriptions") await generateColumnDescriptions();
else if (action === "sync-tables") await synchronize("tables"); else if (action === "sync-tables") await synchronize("tables");
else if (action === "consolidate-descriptions") await consolidateDescriptions(); else if (action === "consolidate-descriptions") await consolidateDescriptions();
else if (action === "sync-columns") await synchronize("columns", selectedIds); else if (action === "sync-columns") await synchronize("columns", selectedIds);
@@ -464,7 +511,7 @@ export function DatabaseTables({
const databaseHeader = ( const databaseHeader = (
<div className="px-4 pt-5 sm:px-5"> <div className="px-4 pt-5 sm:px-5">
<Button type="button" variant="ghost" className="-ml-2 mb-4" disabled={navigationBusy} onClick={() => navigateDatabase("databases")}><ArrowLeft />Back to databases</Button> <Button type="button" variant="ghost" className="-ml-2 mb-4" disabled={navigationBusy} onClick={() => navigateDatabase("databases")}><ArrowLeft />Back to databases</Button>
<div className="flex flex-wrap items-start justify-between gap-4 border-b border-border pb-5"> <div className="flex flex-wrap items-start justify-between gap-4 border-b border-border pb-5">
<div> <div>
<p className="thot-label mb-1">Database management</p> <p className="thot-label mb-1">Database management</p>
<h2 ref={headingRef} tabIndex={-1} className="font-heading text-2xl font-semibold tracking-tight outline-none">{database.workspaceName}</h2> <h2 ref={headingRef} tabIndex={-1} className="font-heading text-2xl font-semibold tracking-tight outline-none">{database.workspaceName}</h2>
@@ -472,6 +519,7 @@ export function DatabaseTables({
</div> </div>
<div className="flex gap-2"> <div className="flex gap-2">
{currentRun ? <Button type="button" variant="outline" onClick={onOpenSync}><RefreshCw className="animate-spin" />View sync</Button> : null} {currentRun ? <Button type="button" variant="outline" onClick={onOpenSync}><RefreshCw className="animate-spin" />View sync</Button> : null}
{onClearCatalogTables ? <Button type="button" variant="outline" className="text-destructive" disabled={!canManage || busy !== null || Boolean(currentRun)} onClick={() => void clearAllCatalogTables()}>Clear catalog tables</Button> : null}
<Button type="button" disabled={!canManage || !bindingReady || busy !== null || Boolean(currentRun)} onClick={() => void synchronize("all")}><RefreshCw />Sync all</Button> <Button type="button" disabled={!canManage || !bindingReady || busy !== null || Boolean(currentRun)} onClick={() => void synchronize("all")}><RefreshCw />Sync all</Button>
</div> </div>
</div> </div>
@@ -488,18 +536,6 @@ export function DatabaseTables({
return ( return (
<section aria-label={`Columns for ${activeTable.name}`} className="flex min-h-0 flex-1 flex-col overflow-hidden bg-background"> <section aria-label={`Columns for ${activeTable.name}`} className="flex min-h-0 flex-1 flex-col overflow-hidden bg-background">
<div className="flex min-h-12 flex-wrap items-center gap-3 border-b border-border bg-muted/25 px-3 py-2"> <div className="flex min-h-12 flex-wrap items-center gap-3 border-b border-border bg-muted/25 px-3 py-2">
<Button
type="button"
variant="ghost"
size="sm"
className="thot-fleet-icon-action thot-fleet-back-action"
disabled={navigationBusy}
data-tooltip="Tables"
aria-label="Back to tables"
onClick={leaveTable}
>
<ArrowLeft aria-hidden="true" /> Back to tables
</Button>
<div className="min-w-0"> <div className="min-w-0">
<p className="thot-label">Physical table</p> <p className="thot-label">Physical table</p>
<h3 className="truncate font-heading text-lg font-semibold">{activeTable.name}</h3> <h3 className="truncate font-heading text-lg font-semibold">{activeTable.name}</h3>
@@ -513,9 +549,7 @@ export function DatabaseTables({
descriptionGenerationActive={descriptionGenerationActive} descriptionGenerationActive={descriptionGenerationActive}
onDescriptionGenerationRunStarted={onDescriptionGenerationRunStarted} onDescriptionGenerationRunStarted={onDescriptionGenerationRunStarted}
onNavigationStateChange={setColumnNavigation} onNavigationStateChange={setColumnNavigation}
onSync={() => void synchronize("columns", [activeTable.id])}
onSuggestSensitive={onSuggestSensitive} onSuggestSensitive={onSuggestSensitive}
bindingReady={bindingReady}
catalogOperationActive={Boolean(currentRun)} catalogOperationActive={Boolean(currentRun)}
onCatalogMetricsChanged={onCatalogMetricsChanged} onCatalogMetricsChanged={onCatalogMetricsChanged}
presentation="fleet" presentation="fleet"
@@ -626,7 +660,6 @@ export function DatabaseTables({
descriptionGenerationActive={descriptionGenerationActive} descriptionGenerationActive={descriptionGenerationActive}
onDescriptionGenerationRunStarted={onDescriptionGenerationRunStarted} onDescriptionGenerationRunStarted={onDescriptionGenerationRunStarted}
onNavigationStateChange={setColumnNavigation} onNavigationStateChange={setColumnNavigation}
onSync={() => void synchronize("columns", [activeTable.id])}
onSuggestSensitive={onSuggestSensitive} onSuggestSensitive={onSuggestSensitive}
onCatalogMetricsChanged={onCatalogMetricsChanged} onCatalogMetricsChanged={onCatalogMetricsChanged}
/> />
@@ -728,6 +761,7 @@ export function DatabaseTables({
<Menu.Positioner side="bottom" align="start" sideOffset={4}> <Menu.Positioner side="bottom" align="start" sideOffset={4}>
<Menu.Popup className="z-50 min-w-64 rounded-lg bg-popover p-1 text-popover-foreground shadow-md ring-1 ring-foreground/10 outline-none"> <Menu.Popup className="z-50 min-w-64 rounded-lg bg-popover p-1 text-popover-foreground shadow-md ring-1 ring-foreground/10 outline-none">
<Menu.Item className="rounded-md px-3 py-2 text-sm outline-none data-[highlighted]:bg-muted data-[disabled]:opacity-45" disabled={!canManage || !selectedMetadataModel || descriptionGenerationActive || busy !== null || Boolean(currentRun)} onClick={() => void generateDescriptions()}>Generate {selectedIds.length === 1 ? "description" : "descriptions"}</Menu.Item> <Menu.Item className="rounded-md px-3 py-2 text-sm outline-none data-[highlighted]:bg-muted data-[disabled]:opacity-45" disabled={!canManage || !selectedMetadataModel || descriptionGenerationActive || busy !== null || Boolean(currentRun)} onClick={() => void generateDescriptions()}>Generate {selectedIds.length === 1 ? "description" : "descriptions"}</Menu.Item>
<Menu.Item className="rounded-md px-3 py-2 text-sm outline-none data-[highlighted]:bg-muted data-[disabled]:opacity-45" disabled={!canManage || !selectedMetadataModel || descriptionGenerationActive || busy !== null || Boolean(currentRun)} onClick={() => void generateColumnDescriptions()}>Generate descriptions for all columns</Menu.Item>
<Menu.Item className="rounded-md px-3 py-2 text-sm outline-none data-[highlighted]:bg-muted data-[disabled]:opacity-45" disabled={!canManage || !bindingReady || busy !== null || Boolean(currentRun)} onClick={() => void synchronize("columns", selectedIds)}>Synchronize columns</Menu.Item> <Menu.Item className="rounded-md px-3 py-2 text-sm outline-none data-[highlighted]:bg-muted data-[disabled]:opacity-45" disabled={!canManage || !bindingReady || busy !== null || Boolean(currentRun)} onClick={() => void synchronize("columns", selectedIds)}>Synchronize columns</Menu.Item>
<Menu.Item className="rounded-md px-3 py-2 text-sm outline-none data-[highlighted]:bg-muted data-[disabled]:opacity-45" disabled={!canManage || busy !== null || Boolean(currentRun)} onClick={() => void consolidateDescriptions()}>Move generated description to Description</Menu.Item> <Menu.Item className="rounded-md px-3 py-2 text-sm outline-none data-[highlighted]:bg-muted data-[disabled]:opacity-45" disabled={!canManage || busy !== null || Boolean(currentRun)} onClick={() => void consolidateDescriptions()}>Move generated description to Description</Menu.Item>
<Menu.Separator className="my-1 h-px bg-border" /> <Menu.Separator className="my-1 h-px bg-border" />
@@ -769,13 +803,6 @@ export function DatabaseTables({
</> </>
) )
)} )}
{presentation === "fleet" ? (
<Button type="button" size="sm" variant="outline" className="thot-fleet-nav-action thot-fleet-nav-action--below" data-tooltip="Synchronization history" onClick={onOpenSync}>
{currentRun ? <RefreshCw className="animate-spin" aria-hidden="true" /> : <History aria-hidden="true" />} Sync history
</Button>
) : null}
{onOpenDescriptionHistory ? <Button type="button" size="sm" variant="outline" onClick={onOpenDescriptionHistory}><History aria-hidden="true" /> Description history</Button> : null}
{onOpenSensitiveHistory ? <Button type="button" size="sm" variant="outline" onClick={onOpenSensitiveHistory}><History aria-hidden="true" /> Sensitive history</Button> : null}
</div> </div>
{!bindingReady ? <div className="border-b border-border bg-muted/35 px-4 py-3 text-sm text-muted-foreground">Test the current database binding from Overview before synchronizing tables.</div> : null} {!bindingReady ? <div className="border-b border-border bg-muted/35 px-4 py-3 text-sm text-muted-foreground">Test the current database binding from Overview before synchronizing tables.</div> : null}
{fleetQueryError ?? <div className="relative min-h-[280px] flex-1"> {fleetQueryError ?? <div className="relative min-h-[280px] flex-1">
@@ -109,6 +109,9 @@ test("replays ordered SSE events, reconnects from the latest sequence, and keeps
renderDrawer(); renderDrawer();
const log = await screen.findByRole("log", { name: "Description generation events" }); const log = await screen.findByRole("log", { name: "Description generation events" });
const progress = screen.getByRole("table", { name: "Description generation progress" });
expect(within(progress).getAllByRole("columnheader")).toHaveLength(5);
expect(log).toHaveClass("thot-catalog-drawer__event-log");
expect(await within(log).findByText("Run queued")).toBeVisible(); expect(await within(log).findByText("Run queued")).toBeVisible();
await waitFor(() => expect(FakeEventSource.instances).toHaveLength(1)); await waitFor(() => expect(FakeEventSource.instances).toHaveLength(1));
expect(FakeEventSource.instances[0].url).toBe( expect(FakeEventSource.instances[0].url).toBe(
@@ -300,6 +300,16 @@ export function DescriptionGenerationDrawer({
} }
const apparentlyStale = isApparentlyStale(run); const apparentlyStale = isApparentlyStale(run);
const finalError = safeFinalError(run.errorSummary); const finalError = safeFinalError(run.errorSummary);
const modelDisplay = modelLabel && modelLabel !== run.modelId
? `${modelLabel} (${run.modelId})`
: run.modelId;
const progressCounters = [
["total", "Total"],
["processed", "Processed"],
["generated", "Generated"],
["nonGeneratable", "Not generated"],
["failed", "Failed"],
] as const;
const footer = ["queued", "running"].includes(run.status) ? ( const footer = ["queued", "running"].includes(run.status) ? (
<div className="w-full"> <div className="w-full">
{apparentlyStale ? ( {apparentlyStale ? (
@@ -341,55 +351,80 @@ export function DescriptionGenerationDrawer({
open open
ariaLabel="Description generation" ariaLabel="Description generation"
eyebrow="Description generation" eyebrow="Description generation"
title={statusLabel(run)} title={(
description={`${modelLabel}${modelLabel !== run.modelId ? ` (${run.modelId})` : ""} · ${run.scope.replaceAll("_", " ")}`} <span className="inline-flex items-center gap-2">
{!isTerminal(run) ? <LoaderCircle className="size-5 animate-spin" aria-hidden="true" /> : null}
{statusLabel(run)}
</span>
)}
description={run.scope.replaceAll("_", " ")}
onClose={onClose} onClose={onClose}
closeLabel="Close description generation" closeLabel="Close description generation"
busy={action !== null} busy={action !== null}
bodyClassName="thot-catalog-drawer__history-layout" bodyClassName="thot-catalog-drawer__history-layout thot-catalog-drawer__run-layout"
footer={footer} footer={footer}
> >
{runQuery.isError ? ( <div className="thot-catalog-drawer__run-main">
{runQuery.isError ? (
<div role="alert" className="rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive"> <div role="alert" className="rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive">
{apiErrorMessage(runQuery.error)} {apiErrorMessage(runQuery.error)}
</div> </div>
) : null} ) : null}
<section aria-label="Description generation counters"> <div className={`grid gap-4 sm:grid-cols-2${runQuery.isError ? " mt-4" : ""}`}>
<section aria-label="AI usage">
<h3 className="thot-label mb-2">AI usage</h3>
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1 text-xs">
<dt>Model</dt><dd className="truncate text-right" title={modelDisplay}>{modelDisplay}</dd>
<dt>Input tokens</dt><dd className="text-right tabular-nums">{(run.inputTokens ?? 0).toLocaleString("en-US")}</dd>
<dt>Cache tokens</dt><dd className="text-right tabular-nums">{(run.cacheReadTokens ?? 0).toLocaleString("en-US")}</dd>
<dt>Output tokens</dt><dd className="text-right tabular-nums">{(run.outputTokens ?? 0).toLocaleString("en-US")}</dd>
</dl>
</section>
<section aria-label="Description generation timestamps">
<h3 className="thot-label mb-2">Timestamps</h3>
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1 text-xs">
<dt className="text-muted-foreground">Created</dt><dd className="text-right tabular-nums">{timestamp(run.createdAt)}</dd>
<dt className="text-muted-foreground">Started</dt><dd className="text-right tabular-nums">{timestamp(run.startedAt)}</dd>
<dt className="text-muted-foreground">Updated</dt><dd className="text-right tabular-nums">{timestamp(run.updatedAt)}</dd>
<dt className="text-muted-foreground">Finished</dt><dd className="text-right tabular-nums">{timestamp(run.finishedAt)}</dd>
</dl>
</section>
</div>
<section className="mt-4" aria-label="Description generation counters">
<h3 className="thot-label mb-2">Progress</h3> <h3 className="thot-label mb-2">Progress</h3>
<div className="grid grid-cols-2 gap-2"> <div className="overflow-x-auto rounded-md border border-border bg-muted/20">
{([ <table className="w-full min-w-[28rem] table-fixed" aria-label="Description generation progress">
["total", "Total"], <thead>
["processed", "Processed"], <tr className="border-b border-border bg-muted/45">
["generated", "Generated"], {progressCounters.map(([name, label]) => (
["nonGeneratable", "Not generated"], <th key={name} scope="col" className="border-l border-border px-2 py-1.5 text-left text-[10px] font-semibold uppercase tracking-wide text-muted-foreground first:border-l-0">
["failed", "Failed"], {label}
] as const).map(([name, label]) => ( </th>
<div key={name} className="rounded-md border border-border bg-muted/25 px-3 py-2"> ))}
<p className="text-[11px] font-semibold uppercase tracking-wide text-muted-foreground">{label}</p> </tr>
<p data-count={name} className="mt-1 text-lg font-semibold tabular-nums">{run[name]}</p> </thead>
</div> <tbody>
))} <tr>
{progressCounters.map(([name]) => (
<td key={name} data-count={name} className="border-l border-border px-2 py-1.5 text-base font-semibold tabular-nums first:border-l-0">
{run[name]}
</td>
))}
</tr>
</tbody>
</table>
</div> </div>
</section> </section>
<section className="mt-5" aria-label="Description generation timestamps">
<h3 className="thot-label mb-2">Timestamps</h3>
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1.5 text-sm">
<dt className="text-muted-foreground">Created</dt><dd className="text-right">{timestamp(run.createdAt)}</dd>
<dt className="text-muted-foreground">Started</dt><dd className="text-right">{timestamp(run.startedAt)}</dd>
<dt className="text-muted-foreground">Updated</dt><dd className="text-right">{timestamp(run.updatedAt)}</dd>
<dt className="text-muted-foreground">Finished</dt><dd className="text-right">{timestamp(run.finishedAt)}</dd>
</dl>
</section>
{finalError ? ( {finalError ? (
<div className="mt-5 rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive"> <div className="mt-3 rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive">
{finalError} {finalError}
</div> </div>
) : null} ) : null}
<section className="mt-5" aria-label="Description generation log"> <section className="thot-catalog-drawer__events mt-3" aria-label="Description generation log">
<div className="mb-2 flex items-center justify-between"> <div className="mb-2 flex items-center justify-between">
<h3 className="thot-label">Events</h3> <h3 className="thot-label">Events</h3>
<span className="text-xs tabular-nums text-muted-foreground">{events.length}</span> <span className="text-xs tabular-nums text-muted-foreground">{events.length}</span>
@@ -397,7 +432,7 @@ export function DescriptionGenerationDrawer({
<div <div
role="log" role="log"
aria-label="Description generation events" aria-label="Description generation events"
className="max-h-72 overflow-y-auto rounded-md bg-zinc-950 p-3 font-mono text-xs leading-5 text-zinc-200" className="thot-catalog-drawer__event-log overflow-y-auto rounded-md bg-zinc-950 p-3 font-mono text-xs leading-5 text-zinc-200"
> >
{events.length === 0 ? <p className="text-zinc-500">Waiting for events…</p> : events.map((event) => ( {events.length === 0 ? <p className="text-zinc-500">Waiting for events…</p> : events.map((event) => (
<p key={event.sequence} className={event.level === "error" ? "text-red-300" : event.level === "warning" ? "text-amber-300" : undefined}> <p key={event.sequence} className={event.level === "error" ? "text-red-300" : event.level === "warning" ? "text-amber-300" : undefined}>
@@ -407,8 +442,9 @@ export function DescriptionGenerationDrawer({
</div> </div>
{eventQuery.isError ? <p role="alert" className="mt-2 text-sm text-destructive">{apiErrorMessage(eventQuery.error)}</p> : null} {eventQuery.isError ? <p role="alert" className="mt-2 text-sm text-destructive">{apiErrorMessage(eventQuery.error)}</p> : null}
</section> </section>
</div>
<section className="thot-catalog-drawer__history mt-5" aria-label="Description generation history"> <section className="thot-catalog-drawer__history" aria-label="Description generation history">
<div className="mb-2 flex items-center justify-between gap-3"> <div className="mb-2 flex items-center justify-between gap-3">
<h3 className="thot-label">Recent runs</h3> <h3 className="thot-label">Recent runs</h3>
<span className="text-xs tabular-nums text-muted-foreground"> <span className="text-xs tabular-nums text-muted-foreground">
@@ -424,7 +460,7 @@ export function DescriptionGenerationDrawer({
) : visibleHistory.length === 0 ? ( ) : visibleHistory.length === 0 ? (
<p className="text-sm text-muted-foreground">No description generation runs yet.</p> <p className="text-sm text-muted-foreground">No description generation runs yet.</p>
) : ( ) : (
<div className="max-h-56 divide-y divide-border overflow-y-auto rounded-md border border-border"> <div className="thot-catalog-drawer__history-list divide-y divide-border overflow-y-auto rounded-md border border-border">
{visibleHistory.map((item) => ( {visibleHistory.map((item) => (
<button <button
key={item.id} key={item.id}
@@ -15,6 +15,19 @@ const actions = [
}, },
] as const; ] as const;
test("disables the picker when no rows are selected", () => {
render(
<FleetActionSelector
actions={actions}
selectedCount={0}
onRun={vi.fn()}
onClear={vi.fn()}
/>,
);
expect(screen.getByRole("combobox", { name: "Batch action" })).toBeDisabled();
});
test("keeps unavailable commands discoverable and explains why Run is disabled", async () => { test("keeps unavailable commands discoverable and explains why Run is disabled", async () => {
const user = userEvent.setup(); const user = userEvent.setup();
render( render(
@@ -50,6 +63,7 @@ test("runs an enabled command explicitly and resets the picker", async () => {
await user.selectOptions(picker, "sync"); await user.selectOptions(picker, "sync");
expect(screen.getByText("All tables in this database")).toBeVisible(); expect(screen.getByText("All tables in this database")).toBeVisible();
expect(screen.getByText("Action scope")).toBeVisible(); expect(screen.getByText("Action scope")).toBeVisible();
expect(screen.getByRole("button", { name: "Run action" })).toHaveClass("bg-destructive");
await user.click(screen.getByRole("button", { name: "Run action" })); await user.click(screen.getByRole("button", { name: "Run action" }));
expect(onRun).toHaveBeenCalledWith("sync"); expect(onRun).toHaveBeenCalledWith("sync");
@@ -44,7 +44,6 @@ export function FleetActionSelector<ActionId extends string>({
selectedCount, selectedCount,
hiddenSelectedCount = 0, hiddenSelectedCount = 0,
busy = false, busy = false,
busyLabel = "Running…",
selectRef, selectRef,
onRun, onRun,
onClear, onClear,
@@ -102,7 +101,7 @@ export function FleetActionSelector<ActionId extends string>({
className="thot-fleet-action-selector__select" className="thot-fleet-action-selector__select"
aria-label={label} aria-label={label}
value={selectedActionId} value={selectedActionId}
disabled={busy} disabled={busy || selectedCount === 0}
onChange={(event) => setSelectedActionId(event.target.value as ActionId | "")} onChange={(event) => setSelectedActionId(event.target.value as ActionId | "")}
> >
<option value="">Choose an action…</option> <option value="">Choose an action…</option>
@@ -123,13 +122,14 @@ export function FleetActionSelector<ActionId extends string>({
<Button <Button
type="button" type="button"
size="sm" size="sm"
variant={selectedAction?.tone === "destructive" ? "destructive" : "default"} variant="destructive"
className="bg-destructive text-destructive-foreground hover:bg-destructive/90"
disabled={!selectedAction || selectedAction.disabled || busy} disabled={!selectedAction || selectedAction.disabled || busy}
aria-describedby={unavailableReason ? `${hintId}-reason` : undefined} aria-describedby={unavailableReason ? `${hintId}-reason` : undefined}
onClick={() => void run()} onClick={() => void run()}
> >
<Play aria-hidden="true" /> <Play aria-hidden="true" />
{busy ? busyLabel : selectedAction?.runLabel ?? "Run action"} Run action
</Button> </Button>
<Button type="button" size="sm" variant="ghost" disabled={busy} onClick={onClear}> <Button type="button" size="sm" variant="ghost" disabled={busy} onClick={onClear}>
<X aria-hidden="true" /> Clear <X aria-hidden="true" /> Clear
@@ -402,6 +402,7 @@
} }
.thot-fleet-action-selector__select { .thot-fleet-action-selector__select {
flex: 1 1 13rem;
width: min(100%, 22rem); width: min(100%, 22rem);
min-width: 13rem; min-width: 13rem;
height: 2rem; height: 2rem;
@@ -459,11 +460,11 @@
.thot-fleet-ledger__drawer { .thot-fleet-ledger__drawer {
position: absolute; position: absolute;
z-index: 50; z-index: 50;
inset-block: 0.75rem; inset-block: clamp(0.625rem, 2vh, 1rem);
inset-inline-start: 50%; inset-inline-start: 50%;
inset-inline-end: auto; inset-inline-end: auto;
display: flex; display: flex;
width: 60%; width: min(75rem, calc(100% - 2rem));
min-width: 0; min-width: 0;
flex-direction: column; flex-direction: column;
overflow: hidden; overflow: hidden;
@@ -479,10 +480,10 @@
.thot-fleet-ledger .thot-fleet-ledger__context-panel { .thot-fleet-ledger .thot-fleet-ledger__context-panel {
position: absolute; position: absolute;
inset-block: 0.75rem; inset-block: clamp(0.625rem, 2vh, 1rem);
inset-inline-start: 50%; inset-inline-start: 50%;
inset-inline-end: auto; inset-inline-end: auto;
width: 60%; width: min(75rem, calc(100% - 2rem));
max-width: none; max-width: none;
min-width: 0; min-width: 0;
transform: translateX(-50%); transform: translateX(-50%);
@@ -537,6 +538,20 @@
box-shadow: inset 0 -1px 0 oklch(var(--primary-foreground) / 0.18); box-shadow: inset 0 -1px 0 oklch(var(--primary-foreground) / 0.18);
} }
.thot-sensitive-run--success .thot-fleet-ledger__drawer-header,
.thot-sensitive-run--success .thot-catalog-drawer__header {
color: oklch(var(--primary-foreground));
background-color: oklch(var(--success));
border-color: oklch(var(--primary-foreground) / 0.18);
}
.thot-sensitive-run--failure .thot-fleet-ledger__drawer-header,
.thot-sensitive-run--failure .thot-catalog-drawer__header {
color: oklch(var(--destructive-foreground));
background-color: oklch(var(--destructive));
border-color: oklch(var(--destructive-foreground) / 0.18);
}
.thot-fleet-ledger__drawer-header .thot-fleet-ledger__drawer-title, .thot-fleet-ledger__drawer-header .thot-fleet-ledger__drawer-title,
.thot-fleet-ledger__drawer-header .thot-fleet-ledger__drawer-description, .thot-fleet-ledger__drawer-header .thot-fleet-ledger__drawer-description,
.thot-fleet-ledger__drawer-header .thot-fleet-ledger__eyebrow, .thot-fleet-ledger__drawer-header .thot-fleet-ledger__eyebrow,
@@ -575,6 +590,19 @@
box-shadow: inset 0 0 0 1px oklch(var(--success) / 0.5); box-shadow: inset 0 0 0 1px oklch(var(--success) / 0.5);
} }
.thot-catalog-drawer__event-log {
min-height: 8rem;
max-height: 18rem;
}
.thot-catalog-drawer__history-list {
max-height: 14rem;
}
.thot-catalog-drawer__run-layout > .thot-catalog-drawer__history {
margin-top: 1.25rem;
}
.thot-fleet-grid-surface { .thot-fleet-grid-surface {
position: relative; position: relative;
} }
@@ -583,6 +611,13 @@
background: oklch(var(--muted) / 0.32); background: oklch(var(--muted) / 0.32);
} }
@media (min-width: 48rem) {
.thot-fleet-grid-toolbar > .thot-fleet-action-selector {
flex: 1 1 33rem;
min-width: 33rem;
}
}
.thot-fleet-ledger-grid { .thot-fleet-ledger-grid {
--ag-header-background-color: oklch(var(--muted) / 0.84); --ag-header-background-color: oklch(var(--muted) / 0.84);
--ag-odd-row-background-color: oklch(var(--card)); --ag-odd-row-background-color: oklch(var(--card));
@@ -605,6 +640,14 @@
background: oklch(var(--muted) / 0.12); background: oklch(var(--muted) / 0.12);
} }
/* Keep selected rows readable without overpowering the status indicators. */
.thot-fleet-ledger-grid .ag-row.ag-row-selected,
.thot-fleet-ledger-grid .ag-row.ag-row-selected .ag-cell,
.thot-fleet-ledger-grid .ag-row.ag-row-selected .ag-grid-pinned-left-cells,
.thot-fleet-ledger-grid .ag-row.ag-row-selected .ag-grid-pinned-right-cells {
background-color: oklch(93% 0.002 17.2) !important;
}
.thot-fleet-icon-action { .thot-fleet-icon-action {
position: relative; position: relative;
} }
@@ -744,6 +787,10 @@
padding: 0; padding: 0;
} }
.thot-fleet-ledger__drawer-body.thot-catalog-drawer__run-layout {
padding: clamp(0.75rem, 1.5cqi, 1.25rem);
}
.thot-fleet-ledger__drawer-footer { .thot-fleet-ledger__drawer-footer {
display: flex; display: flex;
flex-wrap: wrap; flex-wrap: wrap;
@@ -784,8 +831,8 @@
@container fleet-surface (max-width: 47.999rem) { @container fleet-surface (max-width: 47.999rem) {
.thot-fleet-ledger__drawer, .thot-fleet-ledger__drawer,
.thot-fleet-ledger .thot-fleet-ledger__context-panel { .thot-fleet-ledger .thot-fleet-ledger__context-panel {
inset-block: 0.5rem; inset-block: 0.625rem;
width: calc(100% - 1rem); width: calc(100% - 1.5rem);
} }
} }
@@ -816,11 +863,11 @@
} }
} }
@media (min-width: 40rem) { @container work-area-panel (min-width: 48rem) {
.thot-catalog-drawer__history-layout { .thot-catalog-drawer__history-layout {
display: grid; display: grid;
grid-template-columns: minmax(0, 1fr) 13.5rem; grid-template-columns: minmax(0, 1fr) clamp(12.5rem, 25cqi, 15rem);
gap: 0 1rem; gap: 0 clamp(0.75rem, 2cqi, 1.25rem);
align-content: start; align-content: start;
} }
@@ -836,6 +883,45 @@
align-self: start; align-self: start;
margin-top: 0; margin-top: 0;
} }
.thot-catalog-drawer__run-layout {
height: 100%;
min-height: 0;
grid-template-rows: minmax(0, 1fr);
align-content: stretch;
overflow: hidden;
}
.thot-catalog-drawer__run-layout > .thot-catalog-drawer__run-main {
display: flex;
min-height: 0;
flex-direction: column;
grid-column: 1;
grid-row: 1;
}
.thot-catalog-drawer__run-layout > .thot-catalog-drawer__history {
position: static;
display: flex;
min-height: 0;
flex-direction: column;
align-self: stretch;
grid-row: 1;
}
.thot-catalog-drawer__run-layout .thot-catalog-drawer__events {
display: flex;
min-height: 10rem;
flex: 1;
flex-direction: column;
}
.thot-catalog-drawer__run-layout .thot-catalog-drawer__event-log,
.thot-catalog-drawer__run-layout .thot-catalog-drawer__history-list {
min-height: 0;
max-height: none;
flex: 1;
}
} }
@media (max-width: 47.999rem) { @media (max-width: 47.999rem) {
@@ -24,4 +24,17 @@ describe("Recent runs success styling contract", () => {
expect(styles).toContain('button[data-status="succeeded"]'); expect(styles).toContain('button[data-status="succeeded"]');
expect(styles).toMatch(/oklch\(var\(--success\)/); expect(styles).toMatch(/oklch\(var\(--success\)/);
}); });
test("run drawers keep responsive gutters and switch columns from their own width", () => {
const styles = readFileSync(resolve(sourceRoot, "FleetLedgerShell.css"), "utf8");
expect(styles).toContain("width: min(75rem, calc(100% - 2rem));");
expect(styles).toContain(
".thot-fleet-ledger__drawer-body.thot-catalog-drawer__run-layout",
);
expect(styles).toContain("padding: clamp(0.75rem, 1.5cqi, 1.25rem);");
expect(styles).toContain("@container work-area-panel (min-width: 48rem)");
expect(styles).toContain(".thot-catalog-drawer__history-list");
expect(styles).toContain(".thot-catalog-drawer__event-log");
});
}); });
@@ -97,6 +97,11 @@ test("shows sensitive suggestion results, safe events, and lets operators inspec
const drawer = await screen.findByRole("dialog", { name: "Sensitive suggestion history" }); const drawer = await screen.findByRole("dialog", { name: "Sensitive suggestion history" });
expect(within(drawer).getByRole("heading", { name: "Running" })).toBeVisible(); expect(within(drawer).getByRole("heading", { name: "Running" })).toBeVisible();
const progress = within(drawer).getByRole("table", { name: "Sensitive suggestion progress" });
expect(within(progress).getAllByRole("columnheader")).toHaveLength(3);
expect(within(drawer).getByRole("log", { name: "Sensitive suggestion events" })).toHaveClass(
"thot-catalog-drawer__event-log",
);
expect(await within(drawer).findByText("Classifying columns")).toBeVisible(); expect(await within(drawer).findByText("Classifying columns")).toBeVisible();
expect(within(drawer).queryByRole("button", { name: "Stop" })).not.toBeInTheDocument(); expect(within(drawer).queryByRole("button", { name: "Stop" })).not.toBeInTheDocument();
expect(within(drawer).queryByRole("button", { name: "Unlock stale run" })).not.toBeInTheDocument(); expect(within(drawer).queryByRole("button", { name: "Unlock stale run" })).not.toBeInTheDocument();
@@ -1,4 +1,5 @@
import { useEffect, useMemo } from "react"; import { useEffect, useMemo } from "react";
import { LoaderCircle } from "lucide-react";
import { useQuery, useQueryClient } from "@tanstack/react-query"; import { useQuery, useQueryClient } from "@tanstack/react-query";
import { apiErrorMessage } from "../../api/client"; import { apiErrorMessage } from "../../api/client";
import { import {
@@ -30,6 +31,12 @@ function isTerminal(run?: SensitiveDataSuggestionRun): boolean {
return Boolean(run && ["completed", "failed", "interrupted"].includes(run.status)); return Boolean(run && ["completed", "failed", "interrupted"].includes(run.status));
} }
function outcomeClass(run: SensitiveDataSuggestionRun): string {
if (run.status === "completed") return "thot-sensitive-run--success";
if (["failed", "interrupted"].includes(run.status)) return "thot-sensitive-run--failure";
return "";
}
function safeFinalError(summary: string | null): string | null { function safeFinalError(summary: string | null): string | null {
const value = summary?.trim(); const value = summary?.trim();
if (!value) return null; if (!value) return null;
@@ -133,56 +140,92 @@ export function SensitiveDataSuggestionHistoryDrawer({
const events = eventQuery.data ?? []; const events = eventQuery.data ?? [];
const finalError = safeFinalError(run.errorSummary); const finalError = safeFinalError(run.errorSummary);
const modelDisplay = modelLabel && modelLabel !== run.modelId
? `${modelLabel} (${run.modelId})`
: run.modelId;
const progressCounters = [
["total", "Total columns"],
["suggestedSensitive", "Sensitive"],
["suggestedNonSensitive", "Not sensitive"],
] as const;
return ( return (
<FleetLedgerDrawer <FleetLedgerDrawer
open open
className={outcomeClass(run)}
ariaLabel="Sensitive suggestion history" ariaLabel="Sensitive suggestion history"
eyebrow="Sensitive data suggestions" eyebrow="Sensitive data suggestions"
title={statusLabel(run)} title={(
description={`${modelLabel}${modelLabel !== run.modelId ? ` (${run.modelId})` : ""} · ${run.scope.replaceAll("_", " ")}`} <span className="inline-flex items-center gap-2">
{!isTerminal(run) ? <LoaderCircle className="size-5 animate-spin" aria-hidden="true" /> : null}
{statusLabel(run)}
</span>
)}
description={run.scope.replaceAll("_", " ")}
onClose={onClose} onClose={onClose}
closeLabel="Close sensitive suggestion history" closeLabel="Close sensitive suggestion history"
bodyClassName="thot-catalog-drawer__history-layout" bodyClassName="thot-catalog-drawer__history-layout thot-catalog-drawer__run-layout"
> >
{runQuery.isError ? ( <div className="thot-catalog-drawer__run-main">
{runQuery.isError ? (
<div role="alert" className="rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive"> <div role="alert" className="rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive">
{apiErrorMessage(runQuery.error)} {apiErrorMessage(runQuery.error)}
</div> </div>
) : null} ) : null}
<section aria-label="Sensitive suggestion counters"> <div className={`grid gap-4 sm:grid-cols-2${runQuery.isError ? " mt-4" : ""}`}>
<section aria-label="AI usage">
<h3 className="thot-label mb-2">AI usage</h3>
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1 text-xs">
<dt>Model</dt><dd className="truncate text-right" title={modelDisplay}>{modelDisplay}</dd>
<dt>Input tokens</dt><dd className="text-right tabular-nums">{(run.inputTokens ?? 0).toLocaleString("en-US")}</dd>
<dt>Cache tokens</dt><dd className="text-right tabular-nums">{(run.cacheReadTokens ?? 0).toLocaleString("en-US")}</dd>
<dt>Output tokens</dt><dd className="text-right tabular-nums">{(run.outputTokens ?? 0).toLocaleString("en-US")}</dd>
</dl>
</section>
<section aria-label="Sensitive suggestion timestamps">
<h3 className="thot-label mb-2">Timestamps</h3>
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1 text-xs">
<dt className="text-muted-foreground">Created</dt><dd className="text-right tabular-nums">{timestamp(run.createdAt)}</dd>
<dt className="text-muted-foreground">Started</dt><dd className="text-right tabular-nums">{timestamp(run.startedAt)}</dd>
<dt className="text-muted-foreground">Updated</dt><dd className="text-right tabular-nums">{timestamp(run.updatedAt)}</dd>
<dt className="text-muted-foreground">Finished</dt><dd className="text-right tabular-nums">{timestamp(run.finishedAt)}</dd>
</dl>
</section>
</div>
<section className="mt-4" aria-label="Sensitive suggestion counters">
<h3 className="thot-label mb-2">Results</h3> <h3 className="thot-label mb-2">Results</h3>
<div className="grid grid-cols-2 gap-2"> <div className="overflow-x-auto rounded-md border border-border bg-muted/20">
{([ <table className="w-full min-w-[20rem] table-fixed" aria-label="Sensitive suggestion progress">
["total", "Total columns"], <thead>
["suggestedSensitive", "Sensitive"], <tr className="border-b border-border bg-muted/45">
["suggestedNonSensitive", "Not sensitive"], {progressCounters.map(([name, label]) => (
] as const).map(([name, label]) => ( <th key={name} scope="col" className="border-l border-border px-2 py-1.5 text-left text-[10px] font-semibold uppercase tracking-wide text-muted-foreground first:border-l-0">
<div key={name} className="rounded-md border border-border bg-muted/25 px-3 py-2"> {label}
<p className="text-[11px] font-semibold uppercase tracking-wide text-muted-foreground">{label}</p> </th>
<p data-count={name} className="mt-1 text-lg font-semibold tabular-nums">{run[name]}</p> ))}
</div> </tr>
))} </thead>
<tbody>
<tr>
{progressCounters.map(([name]) => (
<td key={name} data-count={name} className="border-l border-border px-2 py-1.5 text-base font-semibold tabular-nums first:border-l-0">
{run[name]}
</td>
))}
</tr>
</tbody>
</table>
</div> </div>
</section> </section>
<section className="mt-5" aria-label="Sensitive suggestion timestamps">
<h3 className="thot-label mb-2">Timestamps</h3>
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1.5 text-sm">
<dt className="text-muted-foreground">Created</dt><dd className="text-right">{timestamp(run.createdAt)}</dd>
<dt className="text-muted-foreground">Started</dt><dd className="text-right">{timestamp(run.startedAt)}</dd>
<dt className="text-muted-foreground">Updated</dt><dd className="text-right">{timestamp(run.updatedAt)}</dd>
<dt className="text-muted-foreground">Finished</dt><dd className="text-right">{timestamp(run.finishedAt)}</dd>
</dl>
</section>
{finalError ? ( {finalError ? (
<div className="mt-5 rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive"> <div className="mt-3 rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive">
{finalError} {finalError}
</div> </div>
) : null} ) : null}
<section className="mt-5" aria-label="Sensitive suggestion log"> <section className="thot-catalog-drawer__events mt-3" aria-label="Sensitive suggestion log">
<div className="mb-2 flex items-center justify-between"> <div className="mb-2 flex items-center justify-between">
<h3 className="thot-label">Events</h3> <h3 className="thot-label">Events</h3>
<span className="text-xs tabular-nums text-muted-foreground">{events.length}</span> <span className="text-xs tabular-nums text-muted-foreground">{events.length}</span>
@@ -190,7 +233,7 @@ export function SensitiveDataSuggestionHistoryDrawer({
<div <div
role="log" role="log"
aria-label="Sensitive suggestion events" aria-label="Sensitive suggestion events"
className="max-h-72 overflow-y-auto rounded-md bg-zinc-950 p-3 font-mono text-xs leading-5 text-zinc-200" className="thot-catalog-drawer__event-log overflow-y-auto rounded-md bg-zinc-950 p-3 font-mono text-xs leading-5 text-zinc-200"
> >
{events.length === 0 ? <p className="text-zinc-500">Waiting for events…</p> : events.map((event) => ( {events.length === 0 ? <p className="text-zinc-500">Waiting for events…</p> : events.map((event) => (
<p key={event.sequence} className={event.level === "error" ? "text-red-300" : event.level === "warning" ? "text-amber-300" : undefined}> <p key={event.sequence} className={event.level === "error" ? "text-red-300" : event.level === "warning" ? "text-amber-300" : undefined}>
@@ -200,8 +243,9 @@ export function SensitiveDataSuggestionHistoryDrawer({
</div> </div>
{eventQuery.isError ? <p role="alert" className="mt-2 text-sm text-destructive">{apiErrorMessage(eventQuery.error)}</p> : null} {eventQuery.isError ? <p role="alert" className="mt-2 text-sm text-destructive">{apiErrorMessage(eventQuery.error)}</p> : null}
</section> </section>
</div>
<section className="thot-catalog-drawer__history mt-5" aria-label="Sensitive suggestion run history"> <section className="thot-catalog-drawer__history" aria-label="Sensitive suggestion run history">
<div className="mb-2 flex items-center justify-between gap-3"> <div className="mb-2 flex items-center justify-between gap-3">
<h3 className="thot-label">Recent runs</h3> <h3 className="thot-label">Recent runs</h3>
<span className="text-xs tabular-nums text-muted-foreground"> <span className="text-xs tabular-nums text-muted-foreground">
@@ -215,7 +259,7 @@ export function SensitiveDataSuggestionHistoryDrawer({
) : visibleHistory.length === 0 ? ( ) : visibleHistory.length === 0 ? (
<p className="text-sm text-muted-foreground">No sensitive suggestion runs yet.</p> <p className="text-sm text-muted-foreground">No sensitive suggestion runs yet.</p>
) : ( ) : (
<div className="max-h-56 divide-y divide-border overflow-y-auto rounded-md border border-border"> <div className="thot-catalog-drawer__history-list divide-y divide-border overflow-y-auto rounded-md border border-border">
{visibleHistory.map((item) => ( {visibleHistory.map((item) => (
<button <button
key={item.id} key={item.id}
+35 -1
View File
@@ -265,6 +265,36 @@ def _response_content(response: object) -> str:
return content return content
def _response_usage(response: object) -> dict[str, int] | None:
"""Normalize LiteLLM usage, tolerating providers that omit it."""
try:
usage = _field(response, "usage")
prompt = _field(usage, "prompt_tokens")
output = _field(usage, "completion_tokens")
if type(prompt) is not int or prompt < 0 or type(output) is not int or output < 0:
return None
cached = 0
for source, name in ((usage, "cache_read_input_tokens"), (usage, "cached_tokens")):
try:
value = _field(source, name)
except _InvalidResponse:
continue
if type(value) is int and value >= 0:
cached = value
break
try:
details = _field(usage, "prompt_tokens_details")
value = _field(details, "cached_tokens")
if type(value) is int and value >= 0:
cached = value
except _InvalidResponse:
pass
cached = min(cached, prompt)
return {"input": prompt - cached, "cacheRead": cached, "output": output}
except _InvalidResponse:
return None
def handle_request(request: object, *, completion: Completion | None = None) -> Result: def handle_request(request: object, *, completion: Completion | None = None) -> Result:
"""Validate one request and normalize one injected or LiteLLM completion.""" """Validate one request and normalize one injected or LiteLLM completion."""
try: try:
@@ -284,7 +314,11 @@ def handle_request(request: object, *, completion: Completion | None = None) ->
content = _response_content(response) content = _response_content(response)
except Exception: # noqa: BLE001 - arbitrary provider objects are untrusted response data except Exception: # noqa: BLE001 - arbitrary provider objects are untrusted response data
return {"ok": False, "error": "invalid_response"} return {"ok": False, "error": "invalid_response"}
return {"ok": True, "content": content} result: dict[str, object] = {"ok": True, "content": content}
usage = _response_usage(response)
if usage is not None:
result["usage"] = usage
return result
def _write_json(payload: Mapping[str, object]) -> None: def _write_json(payload: Mapping[str, object]) -> None:
@@ -75,6 +75,10 @@ func TestLoadAcceptsMixedKeyedAndKeylessMetadataGenerationModels(t *testing.T) {
label: DeepSeek V4 Pro label: DeepSeek V4 Pro
litellm: {provider: deepseek, model: deepseek-v4-pro} litellm: {provider: deepseek, model: deepseek-v4-pro}
apiKeyEnv: DEEPSEEK_API_KEY apiKeyEnv: DEEPSEEK_API_KEY
- id: deepseek-v4-flash
label: DeepSeek V4 Flash
litellm: {provider: deepseek, model: deepseek-v4-flash}
apiKeyEnv: DEEPSEEK_API_KEY
- id: glm-53 - id: glm-53
label: GLM 5.3 label: GLM 5.3
litellm: litellm:
@@ -95,10 +99,10 @@ func TestLoadAcceptsMixedKeyedAndKeylessMetadataGenerationModels(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("Load() error = %v", err) t.Fatalf("Load() error = %v", err)
} }
if len(installation.MetadataGeneration.Models) != 3 { if len(installation.MetadataGeneration.Models) != 4 {
t.Fatalf("metadata models = %#v", installation.MetadataGeneration.Models) t.Fatalf("metadata models = %#v", installation.MetadataGeneration.Models)
} }
qwen := installation.MetadataGeneration.Models[2] qwen := installation.MetadataGeneration.Models[3]
if qwen.APIKeyEnv != "" || !qwen.LiteLLM.DisableThinking || qwen.LiteLLM.Endpoint == nil { if qwen.APIKeyEnv != "" || !qwen.LiteLLM.DisableThinking || qwen.LiteLLM.Endpoint == nil {
t.Fatalf("keyless qwen model = %#v", qwen) t.Fatalf("keyless qwen model = %#v", qwen)
} }