feat: refine metadata catalog workflows

This commit is contained in:
Codex
2026-09-02 11:38:47 +02:00
parent 076c9742c5
commit 4531746038
48 changed files with 1345 additions and 493 deletions
+3 -4
View File
@@ -108,10 +108,9 @@ and staging only: `?db-ui=legacy` is honored in Vite development or when
prototype on port `5173` also remains temporary until owner acceptance of the integrated surface,
after which both migration aids can be removed.
Schema refresh is one durable asynchronous engine with database-table, database-column,
selected-table-column, relationship, and full-database actions. Database-level menus expose the
table, all-column, relationship, and full scopes separately; selecting tables exposes column
synchronization plus manual column and relationship cleanup for that subset. Database selections
Schema refresh is one durable asynchronous engine with database-table, selected-table-column,
relationship, and full-database actions. Database-level menus expose only the table, relationship,
and full scopes; selecting tables exposes column synchronization plus manual column and relationship cleanup for that subset. Database selections
also expose manual table and relationship cleanup. Cleanup selections are atomic and share the
one-active-operation-per-database exclusion with synchronization. Runs have leases and
restart recovery, atomic apply, destructive-diff confirmation with re-scan, cancellation before
+2 -2
View File
@@ -59,7 +59,7 @@ import { DescriptionGenerationWorker } from "./catalog/description-generation-wo
import { SensitiveDataSuggester } from "./catalog/sensitive-data-suggester.js";
import { SensitiveDataSuggestionRunner } from "./catalog/sensitive-data-suggestion-runner.js";
import {
PostgresDescriptionSourceSampler,
ConcreteDescriptionSourceSampler,
type DescriptionSourceSampler,
} from "./catalog/description-source-sampler.js";
import { catalogDescriptionGenerationRoutes } from "./routes/catalog-description-generation.js";
@@ -177,7 +177,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
{ connectTimeoutMs: config.workspaceDiagnosticTimeoutMs },
);
const descriptionSourceSampler = deps?.descriptionSourceSampler
?? new PostgresDescriptionSourceSampler(catalogPostgresAccess);
?? new ConcreteDescriptionSourceSampler(catalogPostgresAccess, workspaceSecretStore);
const descriptionGenerationWorker = new DescriptionGenerationWorker(
catalogRepository,
workspaceRegistry,
@@ -1,7 +1,7 @@
import { z } from "zod";
import type { WorkspaceRegistry } from "../workspaces/registry.js";
import type { MetadataGenerationModels, ResolvedMetadataGenerationModel } from "./metadata-generation-models.js";
import type { ModelCompleter, ModelCompletionMessage } from "./model-completer.js";
import type { ModelCompleter, ModelCompletionMessage, ModelCompletionResult } from "./model-completer.js";
import {
ModelCompletionCancelledError,
ModelCompletionProviderError,
@@ -38,6 +38,7 @@ const MAX_SAMPLE_FIELDS_PER_ROW = 4;
const MAX_SAMPLE_COLUMNS = 4;
const MAX_REPRESENTATIVE_VALUES_PER_REQUEST = 5;
const MAX_TARGET_SAMPLE_JSON_BYTES = 8 * 1024;
const MAX_COMPLETION_ATTEMPTS_PER_BATCH = 2;
const generatedOutcomeSchema = z.object({
targetId: z.uuid(),
outcome: z.literal("generated"),
@@ -55,7 +56,9 @@ const completionResponseSchema = z.object({
results: z.array(outcomeSchema).min(1).max(MAX_TARGETS_PER_BATCH),
}).strict();
class InvalidModelOutcomeError extends Error {}
class InvalidModelJsonError extends Error {}
class InvalidModelSchemaError extends Error {}
class MissingModelTargetsError extends Error {}
class DescriptionGenerationBatchError extends Error {
constructor(
readonly failure: unknown,
@@ -143,6 +146,9 @@ interface DescriptionGenerationCounters {
generated: number;
nonGeneratable: number;
failed: number;
inputTokens: number;
cacheReadTokens: number;
outputTokens: number;
consecutiveTechnicalFailures: number;
}
@@ -152,6 +158,9 @@ function persistedCounters(counters: DescriptionGenerationCounters) {
generated: counters.generated,
nonGeneratable: counters.nonGeneratable,
failed: counters.failed,
inputTokens: counters.inputTokens,
cacheReadTokens: counters.cacheReadTokens,
outputTokens: counters.outputTokens,
};
}
@@ -625,36 +634,47 @@ function messagesFor(
}
function parseOutcomes(content: string, expectedTargetIds: readonly string[]): Map<string, ParsedOutcome> {
const trimmed = content.trim();
const fenced = /^```(?:json)?[ \t]*\r?\n([\s\S]*?)\r?\n```$/iu.exec(trimmed);
let parsed: unknown;
try {
const trimmed = content.trim();
const fenced = /^```(?:json)?[ \t]*\r?\n([\s\S]*?)\r?\n```$/iu.exec(trimmed);
const outcomes = completionResponseSchema.parse(JSON.parse(fenced?.[1] ?? trimmed)).results;
const expected = new Set(expectedTargetIds);
if (outcomes.length !== expectedTargetIds.length || expected.size !== expectedTargetIds.length) {
throw new InvalidModelOutcomeError();
}
const mapped = new Map<string, ParsedOutcome>();
for (const outcome of outcomes) {
if (!expected.has(outcome.targetId) || mapped.has(outcome.targetId)) {
throw new InvalidModelOutcomeError();
}
mapped.set(outcome.targetId, outcome.outcome === "generated"
? { ...outcome, description: outcome.description.trim() }
: outcome);
}
if (mapped.size !== expected.size) throw new InvalidModelOutcomeError();
return mapped;
} catch (error) {
if (error instanceof InvalidModelOutcomeError) throw error;
throw new InvalidModelOutcomeError();
parsed = JSON.parse(fenced?.[1] ?? trimmed);
} catch {
throw new InvalidModelJsonError();
}
const response = completionResponseSchema.safeParse(parsed);
if (!response.success) throw new InvalidModelSchemaError();
const outcomes = response.data.results;
const expected = new Set(expectedTargetIds);
if (outcomes.length !== expectedTargetIds.length || expected.size !== expectedTargetIds.length) {
throw new MissingModelTargetsError();
}
const mapped = new Map<string, ParsedOutcome>();
for (const outcome of outcomes) {
if (!expected.has(outcome.targetId) || mapped.has(outcome.targetId)) {
throw new MissingModelTargetsError();
}
mapped.set(outcome.targetId, outcome.outcome === "generated"
? { ...outcome, description: outcome.description.trim() }
: outcome);
}
if (mapped.size !== expected.size) throw new MissingModelTargetsError();
return mapped;
}
function safeFailure(error: unknown): string {
if (error instanceof DescriptionGenerationFailureStreakError) return error.message;
const failure = error instanceof DescriptionGenerationBatchError ? error.failure : error;
if (failure instanceof ModelCompletionProviderError) return "The model provider request failed.";
if (failure instanceof InvalidModelOutcomeError) return "The model response was invalid.";
if (failure instanceof InvalidModelJsonError) return "The model response was not valid JSON.";
if (failure instanceof InvalidModelSchemaError) {
return "The model response did not match the required schema.";
}
if (failure instanceof MissingModelTargetsError) {
return "The model response was missing one or more requested targets.";
}
return "Description generation failed.";
}
@@ -907,6 +927,9 @@ export class DescriptionGenerationWorker {
generated: 0,
nonGeneratable: 0,
failed: 0,
inputTokens: 0,
cacheReadTokens: 0,
outputTokens: 0,
consecutiveTechnicalFailures: 0,
};
await this.processTargets(run, database, plan.columnTargets, model, counters, signal);
@@ -973,20 +996,41 @@ export class DescriptionGenerationWorker {
);
}
throwIfCancelled(signal);
let outcomes: Map<string, ParsedOutcome>;
try {
const content = await this.completer.complete({
model,
messages: messagesFor(database, batch, run.language, sourceSamples),
signal,
});
throwIfCancelled(signal);
outcomes = parseOutcomes(content, batch.map((target) => (
target.kind === "column" ? target.column.id : target.table.id
)));
} catch (error) {
if (error instanceof ModelCompletionCancelledError) throw error;
const batchError = new DescriptionGenerationBatchError(error, batch.map(failureTarget));
const expectedTargetIds = batch.map((target) => (
target.kind === "column" ? target.column.id : target.table.id
));
const messages = messagesFor(database, batch, run.language, sourceSamples);
let outcomes: Map<string, ParsedOutcome> | undefined;
let terminalFailure: unknown;
for (let attempt = 1; attempt <= MAX_COMPLETION_ATTEMPTS_PER_BATCH; attempt += 1) {
try {
const completion = await this.completer.complete({ model, messages, signal });
const result: ModelCompletionResult = typeof completion === "string"
? { content: completion, usage: { input: 0, cacheRead: 0, output: 0 } }
: completion;
counters.inputTokens += result.usage.input;
counters.cacheReadTokens += result.usage.cacheRead;
counters.outputTokens += result.usage.output;
throwIfCancelled(signal);
outcomes = parseOutcomes(result.content, expectedTargetIds);
break;
} catch (error) {
if (error instanceof ModelCompletionCancelledError) throw error;
terminalFailure = error;
if (attempt < MAX_COMPLETION_ATTEMPTS_PER_BATCH) {
await this.appendEvent(
run.id,
"warning",
`${safeFailure(error)} Retrying batch (attempt ${attempt + 1} of ${MAX_COMPLETION_ATTEMPTS_PER_BATCH}).`,
);
}
}
}
if (!outcomes) {
const batchError = new DescriptionGenerationBatchError(
terminalFailure,
batch.map(failureTarget),
);
counters.processed += batch.length;
counters.failed += batch.length;
counters.consecutiveTechnicalFailures += 1;
@@ -1009,7 +1053,10 @@ export class DescriptionGenerationWorker {
const targetId = target.kind === "column" ? target.column.id : target.table.id;
const outcome = outcomes.get(targetId);
if (!outcome) {
throw new DescriptionGenerationBatchError(new InvalidModelOutcomeError(), [failureTarget(target)]);
throw new DescriptionGenerationBatchError(
new MissingModelTargetsError(),
[failureTarget(target)],
);
}
const generatedDescription = outcome.outcome === "generated"
? outcome.description
+132 -43
View File
@@ -1,5 +1,8 @@
import { readFile } from "node:fs/promises";
import type { WorkspaceSecretStore } from "../workspaces/secret-store.js";
import type { CatalogPostgresAccess } from "./postgres-access.js";
import type { WorkspaceDatabase } from "./types.js";
import { CATALOG_SECRET_IDS } from "./secrets.js";
import { CatalogConnectorError, type WorkspaceDatabase } from "./types.js";
const MAX_SOURCE_ROWS = 5;
const MAX_REPRESENTATIVE_VALUES = 5;
@@ -79,15 +82,82 @@ function distinctKey(value: Exclude<DescriptionSourceSampleValue, null>): string
return `${typeof value}:${String(value)}`;
}
/** PostgreSQL-wire sampler. REST bindings remain unsupported by CatalogPostgresAccess. */
export class PostgresDescriptionSourceSampler implements DescriptionSourceSampler {
constructor(private readonly access: CatalogPostgresAccess) {}
function columnsFor(target: DescriptionSourceSamplingTarget): string[] {
return [...new Set(target.columnNames)].slice(0, MAX_SOURCE_COLUMNS_PER_TARGET);
}
function normalizedSample(
target: DescriptionSourceSamplingTarget,
columnNames: readonly string[],
sourceRows: readonly Record<string, unknown>[],
): DescriptionTargetSourceSample {
const rows = sourceRows.slice(0, MAX_SOURCE_ROWS).map((row) => ({
fields: columnNames.flatMap((name) => {
const value = normalizeValue(row[name]);
return value === undefined ? [] : [{ name, value }];
}),
}));
const valuesByColumn = new Map<string, Exclude<DescriptionSourceSampleValue, null>[]>();
const seenByColumn = new Map<string, Set<string>>();
let representativeValueCount = 0;
for (const row of rows) {
for (const field of row.fields) {
if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break;
if (field.value === null) continue;
const seen = seenByColumn.get(field.name) ?? new Set<string>();
const key = distinctKey(field.value);
if (seen.has(key)) continue;
seen.add(key);
seenByColumn.set(field.name, seen);
const values = valuesByColumn.get(field.name) ?? [];
values.push(field.value);
valuesByColumn.set(field.name, values);
representativeValueCount += 1;
}
if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break;
}
return {
targetId: target.targetId,
tableName: target.tableName,
rows,
representativeValues: columnNames.flatMap((column) => {
const values = valuesByColumn.get(column);
return values && values.length > 0 ? [{ column, values }] : [];
}),
};
}
function samplingSql(
database: WorkspaceDatabase,
target: DescriptionSourceSamplingTarget,
columns: readonly string[],
): string {
const projections = columns.map((columnName) => {
const identifier = quoteIdentifier(columnName);
return `LEFT((${identifier})::text, ${MAX_SOURCE_VALUE_BYTES}) AS ${identifier}`;
});
return [
`SELECT ${projections.join(", ")}`,
`FROM ${quoteIdentifier(database.schema)}.${quoteIdentifier(target.tableName)}`,
`LIMIT ${MAX_SOURCE_ROWS}`,
].join(" ");
}
/** Bounded source sampler that follows the database's PostgreSQL-wire or REST binding. */
export class ConcreteDescriptionSourceSampler implements DescriptionSourceSampler {
constructor(
private readonly access: CatalogPostgresAccess,
private readonly secretStore?: Pick<WorkspaceSecretStore, "materialize">,
) {}
async sample(
database: WorkspaceDatabase,
targets: readonly DescriptionSourceSamplingTarget[],
signal: AbortSignal,
): Promise<readonly DescriptionTargetSourceSample[]> {
if (database.binding.transport === "rest_api") {
return await this.sampleRest(database, targets, signal);
}
const client = await this.access.connect(database, signal);
let transactionOpen = false;
try {
@@ -95,7 +165,7 @@ export class PostgresDescriptionSourceSampler implements DescriptionSourceSample
transactionOpen = true;
const samples: DescriptionTargetSourceSample[] = [];
for (const target of targets) {
const columnNames = [...new Set(target.columnNames)].slice(0, MAX_SOURCE_COLUMNS_PER_TARGET);
const columnNames = columnsFor(target);
if (columnNames.length === 0) {
samples.push({
targetId: target.targetId,
@@ -115,44 +185,7 @@ export class PostgresDescriptionSourceSampler implements DescriptionSourceSample
"LIMIT $2",
].join(" ");
const result = await client.query(sql, [MAX_SOURCE_VALUE_BYTES, MAX_SOURCE_ROWS]);
const rows = result.rows.slice(0, MAX_SOURCE_ROWS).map((row) => ({
fields: columnNames.flatMap((name) => {
const value = normalizeValue(row[name]);
return value === undefined ? [] : [{ name, value }];
}),
}));
const valuesByColumn = new Map<
string,
Exclude<DescriptionSourceSampleValue, null>[]
>();
const seenByColumn = new Map<string, Set<string>>();
let representativeValueCount = 0;
for (const row of rows) {
for (const field of row.fields) {
if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break;
if (field.value === null) continue;
const seen = seenByColumn.get(field.name) ?? new Set<string>();
const key = distinctKey(field.value);
if (seen.has(key)) continue;
seen.add(key);
seenByColumn.set(field.name, seen);
const values = valuesByColumn.get(field.name) ?? [];
values.push(field.value);
valuesByColumn.set(field.name, values);
representativeValueCount += 1;
}
if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break;
}
const representativeValues = columnNames.flatMap((column) => {
const values = valuesByColumn.get(column);
return values && values.length > 0 ? [{ column, values }] : [];
});
samples.push({
targetId: target.targetId,
tableName: target.tableName,
rows,
representativeValues,
});
samples.push(normalizedSample(target, columnNames, result.rows));
}
return samples;
} finally {
@@ -160,4 +193,60 @@ export class PostgresDescriptionSourceSampler implements DescriptionSourceSample
await client.end().catch(() => undefined);
}
}
private async sampleRest(
database: WorkspaceDatabase,
targets: readonly DescriptionSourceSamplingTarget[],
signal: AbortSignal,
): Promise<readonly DescriptionTargetSourceSample[]> {
if (!this.secretStore) throw new CatalogConnectorError("REST source sampling is not configured");
const auth = database.binding.restAuth ?? "bearer";
const materialized = this.secretStore.materialize(
database.workspaceId,
auth === "none" ? [] : [CATALOG_SECRET_IDS.apiKey],
);
try {
const headers: Record<string, string> = { "content-type": "application/json" };
if (auth !== "none") {
const credentialFile = materialized.files.get(CATALOG_SECRET_IDS.apiKey);
if (!credentialFile) throw new CatalogConnectorError("REST API key is not configured");
const credential = (await readFile(credentialFile, "utf8")).trim();
if (auth === "bearer") headers.authorization = `Bearer ${credential}`;
else headers["x-api-key"] = credential;
}
const baseUrl = database.binding.baseUrl?.replace(/\/+$/, "");
if (!baseUrl) throw new CatalogConnectorError("Database binding is incomplete");
const samples: DescriptionTargetSourceSample[] = [];
for (const target of targets) {
const columnNames = columnsFor(target);
if (columnNames.length === 0) {
samples.push(normalizedSample(target, columnNames, []));
continue;
}
const response = await fetch(`${baseUrl}/rpc/run_query`, {
method: "POST",
headers,
body: JSON.stringify({ query_text: samplingSql(database, target, columnNames) }),
signal,
});
if (!response.ok) throw new CatalogConnectorError("REST source sampling failed");
const body: unknown = await response.json();
if (!Array.isArray(body)
|| body.some((row) => !row || typeof row !== "object" || Array.isArray(row))) {
throw new CatalogConnectorError("REST source sampling response is invalid");
}
samples.push(normalizedSample(
target,
columnNames,
body as Array<Record<string, unknown>>,
));
}
return samples;
} catch (error) {
if (error instanceof CatalogConnectorError) throw error;
throw new CatalogConnectorError("REST source sampling failed");
} finally {
materialized.release();
}
}
}
+8 -2
View File
@@ -335,7 +335,10 @@ export class MemoryCatalogRepository implements CatalogRepository {
processed: 0,
generated: 0,
nonGeneratable: 0,
failed: 0,
failed: 0,
inputTokens: 0,
cacheReadTokens: 0,
outputTokens: 0,
createdAt: now,
startedAt: null,
updatedAt: now,
@@ -444,7 +447,10 @@ export class MemoryCatalogRepository implements CatalogRepository {
status: "running",
total: 0,
suggestedSensitive: 0,
suggestedNonSensitive: 0,
suggestedNonSensitive: 0,
inputTokens: 0,
cacheReadTokens: 0,
outputTokens: 0,
createdAt: now,
startedAt: now,
updatedAt: now,
+2
View File
@@ -11,6 +11,7 @@ import * as descriptionGenerationRunsMigration from "./migrations/005_descriptio
import * as sensitiveDataFlagMigration from "./migrations/006_sensitive_data_flag.js";
import * as sensitiveDataSuggestionRunsMigration from "./migrations/007_sensitive_data_suggestion_runs.js";
import * as catalogLogicalRelationshipsMigration from "./migrations/008_catalog_logical_relationships.js";
import * as aiTokenUsageMigration from "./migrations/009_ai_token_usage.js";
const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL;
const host = process.env.THT_CATALOG_DB_HOST;
@@ -44,6 +45,7 @@ const provider: MigrationProvider = {
"006_sensitive_data_flag": sensitiveDataFlagMigration,
"007_sensitive_data_suggestion_runs": sensitiveDataSuggestionRunsMigration,
"008_catalog_logical_relationships": catalogLogicalRelationshipsMigration,
"009_ai_token_usage": aiTokenUsageMigration,
};
},
};
@@ -0,0 +1,20 @@
import type { Kysely } from "kysely";
import type { CatalogDatabase } from "../repository.js";
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
for (const table of ["description_generation_runs", "sensitive_data_suggestion_runs"] as const) {
await db.schema.alterTable(table)
.addColumn("input_tokens", "integer", (col) => col.notNull().defaultTo(0))
.addColumn("cache_read_tokens", "integer", (col) => col.notNull().defaultTo(0))
.addColumn("output_tokens", "integer", (col) => col.notNull().defaultTo(0))
.execute();
}
}
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
for (const table of ["sensitive_data_suggestion_runs", "description_generation_runs"] as const) {
await db.schema.alterTable(table)
.dropColumn("input_tokens").dropColumn("cache_read_tokens").dropColumn("output_tokens")
.execute();
}
}
+7 -5
View File
@@ -4,7 +4,7 @@ import type { ResolvedMetadataGenerationModel } from "./metadata-generation-mode
const MAX_HELPER_OUTPUT_BYTES = 64 * 1024;
const helperOutputSchema = z.discriminatedUnion("ok", [
z.object({ ok: z.literal(true), content: z.string() }).strict(),
z.object({ ok: z.literal(true), content: z.string(), usage: z.object({ input: z.number().int().nonnegative(), cacheRead: z.number().int().nonnegative(), output: z.number().int().nonnegative() }).strict().optional() }).strict(),
z.object({ ok: z.literal(false), error: z.literal("provider_failure") }).strict(),
]);
@@ -18,10 +18,12 @@ export interface ModelCompletionRequest {
messages: readonly ModelCompletionMessage[];
signal: AbortSignal;
}
export interface ModelCompletionUsage { input: number; cacheRead: number; output: number; }
export interface ModelCompletionResult { content: string; usage: ModelCompletionUsage; }
/** The provider boundary used by Description Generation. */
export interface ModelCompleter {
complete(request: ModelCompletionRequest): Promise<string>;
complete(request: ModelCompletionRequest): Promise<string | ModelCompletionResult>;
}
export class ModelCompletionProviderError extends Error {
@@ -47,7 +49,7 @@ export class PythonModelCompleter implements ModelCompleter {
terminationGraceMs?: number;
}) {}
async complete(request: ModelCompletionRequest): Promise<string> {
async complete(request: ModelCompletionRequest): Promise<ModelCompletionResult> {
if (request.signal.aborted) throw new ModelCompletionCancelledError();
const payload = {
model: `${request.model.provider}/${request.model.model}`,
@@ -62,7 +64,7 @@ export class PythonModelCompleter implements ModelCompleter {
...(request.model.disableThinking === true ? { disable_thinking: true } : {}),
};
return await new Promise<string>((resolve, reject) => {
return await new Promise<ModelCompletionResult>((resolve, reject) => {
const child = spawn(
this.options.pythonExecutable,
["-m", this.options.helperModule ?? "tht.internal.litellm_completion"],
@@ -135,7 +137,7 @@ export class PythonModelCompleter implements ModelCompleter {
if (!output.ok) return fail();
settled = true;
cleanup();
resolve(output.content);
resolve({ content: output.content, usage: output.usage ?? { input: 0, cacheRead: 0, output: 0 } });
} catch {
fail();
}
+12
View File
@@ -171,6 +171,9 @@ interface DescriptionGenerationRunTable {
generated: number;
nonGeneratable: number;
failed: number;
inputTokens: number;
cacheReadTokens: number;
outputTokens: number;
createdAt: Timestamp;
startedAt: Timestamp | null;
updatedAt: Timestamp;
@@ -195,6 +198,9 @@ interface SensitiveDataSuggestionRunTable {
total: number;
suggestedSensitive: number;
suggestedNonSensitive: number;
inputTokens: number;
cacheReadTokens: number;
outputTokens: number;
createdAt: Timestamp;
startedAt: Timestamp;
updatedAt: Timestamp;
@@ -816,6 +822,9 @@ export class KyselyCatalogRepository implements CatalogRepository {
generated: 0,
nonGeneratable: 0,
failed: 0,
inputTokens: 0,
cacheReadTokens: 0,
outputTokens: 0,
startedAt: null,
finishedAt: null,
errorSummary: null,
@@ -943,6 +952,9 @@ export class KyselyCatalogRepository implements CatalogRepository {
total: 0,
suggestedSensitive: 0,
suggestedNonSensitive: 0,
inputTokens: 0,
cacheReadTokens: 0,
outputTokens: 0,
finishedAt: null,
errorSummary: null,
}).returningAll().executeTakeFirstOrThrow();
@@ -1,6 +1,6 @@
import { z } from "zod";
import type { MetadataGenerationModels } from "./metadata-generation-models.js";
import type { ModelCompleter, ModelCompletionMessage } from "./model-completer.js";
import type { ModelCompleter, ModelCompletionMessage, ModelCompletionResult, ModelCompletionUsage } from "./model-completer.js";
import type {
CatalogColumn,
CatalogRepository,
@@ -201,6 +201,8 @@ export class SensitiveDataSuggester {
targetIds: readonly string[],
signal: AbortSignal,
onPrepared?: (total: number) => void | Promise<void>,
onProgress?: (processed: number, suggestions: readonly SensitiveDataSuggestion[]) => void | Promise<void>,
onUsage?: (usage: ModelCompletionUsage) => void | Promise<void>,
): Promise<readonly SensitiveDataSuggestion[]> {
const database = await this.repository.get(databaseId);
if (!database) throw new SensitiveDataSuggestionTargetNotFoundError("database");
@@ -212,11 +214,16 @@ export class SensitiveDataSuggester {
for (const batch of batchesFor(database, columns)) {
let received: Map<string, { columnId: string; sensitive: boolean }> | undefined;
for (let attempt = 0; attempt < 2 && !received; attempt += 1) {
const content = await this.completer.complete({
const completion = await this.completer.complete({
model,
signal,
messages: [systemMessage, { role: "user", content: userContent(database, batch) }],
});
const result: ModelCompletionResult = typeof completion === "string"
? { content: completion, usage: { input: 0, cacheRead: 0, output: 0 } }
: completion;
await onUsage?.(result.usage);
const content = result.content;
try {
const parsed = responseSchema.parse(JSON.parse(content));
const expected = new Set(batch.map((column) => column.columnId));
@@ -240,6 +247,7 @@ export class SensitiveDataSuggester {
currentSensitive: column.currentSensitive,
sensitive: received!.get(column.columnId)!.sensitive,
})));
await onProgress?.(suggestions.length, suggestions.slice(-batch.length));
}
return suggestions;
}
@@ -10,6 +10,7 @@ import type {
SensitiveDataSuggestionRun,
SensitiveDataSuggestionScope,
} from "./types.js";
import type { ModelCompletionUsage } from "./model-completer.js";
const interruptedMessage = "Sensitive-field suggestion generation was interrupted by backend restart.";
const failedMessage = "Sensitive-field suggestion generation failed.";
@@ -73,6 +74,31 @@ export class SensitiveDataSuggestionRunner {
});
if (!prepared) throw new Error("Sensitive Data Suggestion Run disappeared");
},
async (processed, batch) => {
const suggestedSensitive = batch.filter((suggestion) => suggestion.sensitive).length;
const suggestedNonSensitive = batch.length - suggestedSensitive;
const current = await this.repository.getSensitiveDataSuggestionRun(started.id);
if (!current) throw new Error("Sensitive Data Suggestion Run disappeared");
const progress = await this.repository.updateSensitiveDataSuggestionRun(started.id, {
suggestedSensitive: current.suggestedSensitive + suggestedSensitive,
suggestedNonSensitive: current.suggestedNonSensitive + suggestedNonSensitive,
});
if (!progress) throw new Error("Sensitive Data Suggestion Run disappeared");
await this.repository.appendSensitiveDataSuggestionEvent(
started.id,
"info",
`Classified ${processed} of ${progress.total} columns.`,
);
},
async (usage: ModelCompletionUsage) => {
const current = await this.repository.getSensitiveDataSuggestionRun(started.id);
if (!current) throw new Error("Sensitive Data Suggestion Run disappeared");
await this.repository.updateSensitiveDataSuggestionRun(started.id, {
inputTokens: current.inputTokens + usage.input,
cacheReadTokens: current.cacheReadTokens + usage.cacheRead,
outputTokens: current.outputTokens + usage.output,
});
},
);
const suggestedSensitive = suggestions.filter((suggestion) => suggestion.sensitive).length;
const suggestedNonSensitive = suggestions.length - suggestedSensitive;
+12
View File
@@ -234,6 +234,9 @@ export interface DescriptionGenerationRun {
generated: number;
nonGeneratable: number;
failed: number;
inputTokens: number;
cacheReadTokens: number;
outputTokens: number;
createdAt: string;
startedAt: string | null;
updatedAt: string;
@@ -250,6 +253,9 @@ export interface DescriptionGenerationRunUpdate {
startedAt?: string | null;
finishedAt?: string | null;
errorSummary?: string | null;
inputTokens?: number;
cacheReadTokens?: number;
outputTokens?: number;
}
export interface DescriptionGenerationEvent {
@@ -272,6 +278,9 @@ export interface SensitiveDataSuggestionRun {
total: number;
suggestedSensitive: number;
suggestedNonSensitive: number;
inputTokens: number;
cacheReadTokens: number;
outputTokens: number;
createdAt: string;
startedAt: string;
updatedAt: string;
@@ -286,6 +295,9 @@ export interface SensitiveDataSuggestionRunUpdate {
suggestedNonSensitive?: number;
finishedAt?: string | null;
errorSummary?: string | null;
inputTokens?: number;
cacheReadTokens?: number;
outputTokens?: number;
}
export interface SensitiveDataSuggestionEvent {
@@ -101,6 +101,9 @@ function publicRun(run: DescriptionGenerationRun) {
generated: run.generated,
nonGeneratable: run.nonGeneratable,
failed: run.failed,
inputTokens: run.inputTokens,
cacheReadTokens: run.cacheReadTokens,
outputTokens: run.outputTokens,
createdAt: run.createdAt,
startedAt: run.startedAt,
updatedAt: run.updatedAt,
@@ -129,6 +132,9 @@ function publicSensitiveDataSuggestionRun(run: SensitiveDataSuggestionRun) {
total: run.total,
suggestedSensitive: run.suggestedSensitive,
suggestedNonSensitive: run.suggestedNonSensitive,
inputTokens: run.inputTokens,
cacheReadTokens: run.cacheReadTokens,
outputTokens: run.outputTokens,
createdAt: run.createdAt,
startedAt: run.startedAt,
updatedAt: run.updatedAt,
@@ -218,6 +218,12 @@ test("suggests sensitive flags from structural metadata without persisting them"
runId: responseBody.run.id,
sequence: 2,
level: "info",
message: "Classified 1 of 1 columns.",
},
{
runId: responseBody.run.id,
sequence: 3,
level: "info",
message: "Sensitive-field suggestion generation completed for 1 column.",
},
]);
@@ -671,6 +677,7 @@ test("generates one selected Catalog Column from a single JSON code fence", asyn
errorSummary: null,
});
expect(Object.keys(start.json()).sort()).toEqual([
"cacheReadTokens",
"createdAt",
"databaseId",
"errorSummary",
@@ -678,9 +685,11 @@ test("generates one selected Catalog Column from a single JSON code fence", asyn
"finishedAt",
"generated",
"id",
"inputTokens",
"language",
"modelId",
"nonGeneratable",
"outputTokens",
"processed",
"scope",
"startedAt",
@@ -1272,7 +1281,7 @@ test("Stop aborts source sampling before any model request", async () => {
test("an isolated exhausted technical batch failure allows completion with errors", async () => {
const modelCompleter: ModelCompleter = {
complete: vi.fn(async (request) => {
if (vi.mocked(modelCompleter.complete).mock.calls.length === 1) {
if (vi.mocked(modelCompleter.complete).mock.calls.length <= 2) {
throw new ModelCompletionProviderError();
}
const context = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n"));
@@ -1320,7 +1329,7 @@ test("an isolated exhausted technical batch failure allows completion with error
failed: 10,
errorSummary: "Description generation completed with errors.",
});
expect(modelCompleter.complete).toHaveBeenCalledTimes(2);
expect(modelCompleter.complete).toHaveBeenCalledTimes(3);
const updated = new Map(
(await repository.listColumns(database.id, table.id)).map((column) => [column.id, column]),
);
@@ -1346,10 +1355,10 @@ test("success resets the technical-failure streak and the third later failure st
const modelCompleter: ModelCompleter = {
complete: vi.fn(async (request) => {
const call = vi.mocked(modelCompleter.complete).mock.calls.length;
if ([1, 2, 4, 5, 6].includes(call)) {
if ([1, 2, 4, 5, 6, 7, 8, 9].includes(call)) {
throw Object.assign(new ModelCompletionProviderError(), { message: sensitiveDiagnostic });
}
if (call > 6) throw new Error("a later batch must not start");
if (call > 9) throw new Error("a later batch must not start");
const context = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n"));
return JSON.stringify({
results: context.targets.map((target: { targetId: string }) => ({
@@ -1389,25 +1398,25 @@ test("success resets the technical-failure streak and the third later failure st
expect(run).toMatchObject({
status: "failed",
total: 61,
processed: 60,
processed: 50,
generated: 10,
nonGeneratable: 0,
failed: 50,
failed: 40,
errorSummary: "Description generation stopped after three consecutive technical batch failures.",
});
expect(modelCompleter.complete).toHaveBeenCalledTimes(6);
expect(modelCompleter.complete).toHaveBeenCalledTimes(9);
const requests = vi.mocked(modelCompleter.complete).mock.calls.map(([request]) => request);
expect(new Set(requests.map((request) => request.signal)).size).toBe(1);
expect(new Set(requests.map((request) => request.model.id))).toEqual(new Set([configuredModel.id]));
const updated = new Map(
(await repository.listColumns(database.id, table.id)).map((column) => [column.id, column]),
);
targetIds.slice(20, 30).forEach((targetId) => {
targetIds.slice(10, 20).forEach((targetId) => {
expect(updated.get(targetId)?.generatedDescription).toBe("Successful reset batch.");
});
expect(updated.get(targetIds[60]!)?.generatedDescription).toBeNull();
expect(updated.get(targetIds[50]!)?.generatedDescription).toBeNull();
const events = await repository.listDescriptionGenerationEvents(run.id);
expect(events.filter((event) => event.message.includes("model provider request failed"))).toHaveLength(5);
expect(events.filter((event) => event.message.includes("model provider request failed"))).toHaveLength(8);
expect(events.at(-1)).toEqual(expect.objectContaining({
level: "error",
message: "Description generation stopped after three consecutive technical batch failures.",
@@ -1660,6 +1669,7 @@ test("Description Generation history is newest-first, bounded, and exposes only
expect(response.statusCode).toBe(200);
expect(response.json().map((run: { id: string }) => run.id)).toEqual(ids.slice(1).reverse());
expect(Object.keys(response.json()[0]).sort()).toEqual([
"cacheReadTokens",
"createdAt",
"databaseId",
"errorSummary",
@@ -1667,9 +1677,11 @@ test("Description Generation history is newest-first, bounded, and exposes only
"finishedAt",
"generated",
"id",
"inputTokens",
"language",
"modelId",
"nonGeneratable",
"outputTokens",
"processed",
"scope",
"startedAt",
@@ -1945,7 +1957,7 @@ test("retains completed batch writes when a later batch response is malformed",
});
const { run } = await waitForTerminalRun(app, start.json().id);
expect(modelCompleter.complete).toHaveBeenCalledTimes(2);
expect(modelCompleter.complete).toHaveBeenCalledTimes(3);
expect(run).toMatchObject({
status: "completed_with_errors",
total: 11,
@@ -1972,9 +1984,14 @@ test("retains completed batch writes when a later batch response is malformed",
expect(events.slice(2, 12).map((event) => event.message)).toEqual(
orderedIds.slice(0, 10).map((targetId) => `Generated description for Catalog Column ${targetId}.`),
);
expect(events.find((event) => event.level === "warning" && event.message.includes("Retrying batch"))).toEqual(
expect.objectContaining({
message: "The model response did not match the required schema. Retrying batch (attempt 2 of 2).",
}),
);
expect(events.find((event) => event.level === "error")).toEqual(expect.objectContaining({
level: "error",
message: `The model response was invalid. Affected Catalog Column target: ${orderedIds[10]}.`,
message: `The model response did not match the required schema. Affected Catalog Column target: ${orderedIds[10]}.`,
}));
} finally {
await app.close();
@@ -2186,7 +2203,7 @@ test("Generate Missing skips prior partial results and includes null, empty, and
const metadata = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n"));
if (mode === "partial") {
partialCall += 1;
if (partialCall === 2) throw new ModelCompletionProviderError();
if (partialCall === 2 || partialCall === 3) throw new ModelCompletionProviderError();
return JSON.stringify({
results: metadata.targets.map((target: { targetId: string }) => ({
targetId: target.targetId,
@@ -2512,6 +2529,58 @@ test("localizes valid non-generatable Catalog Column results in English", async
}
});
test("retries invalid JSON once and completes the batch when the second response is valid", async () => {
const modelCompleter: ModelCompleter = {
complete: vi.fn(async (request) => {
if (vi.mocked(modelCompleter.complete).mock.calls.length === 1) {
return { content: "not-json", usage: { input: 11, cacheRead: 3, output: 2 } };
}
const context = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n"));
return {
content: JSON.stringify({
results: context.targets.map((target: { targetId: string }) => ({
targetId: target.targetId,
outcome: "generated",
description: "Generated after the application retry.",
})),
}),
usage: { input: 7, cacheRead: 1, output: 5 },
};
}),
};
const { app, repository, database, table, column } = await setup(modelCompleter);
try {
const start = await app.inject({
method: "POST",
url: `/catalog/databases/${database.id}/description-generation-runs`,
payload: { modelId: configuredModel.id, scope: "selected_columns", targetIds: [column.id] },
});
const { run } = await waitForTerminalRun(app, start.json().id);
expect(run).toMatchObject({
status: "completed",
processed: 1,
generated: 1,
failed: 0,
inputTokens: 18,
cacheReadTokens: 4,
outputTokens: 7,
});
expect(modelCompleter.complete).toHaveBeenCalledTimes(2);
expect(await repository.getColumn(database.id, table.id, column.id)).toMatchObject({
generatedDescription: "Generated after the application retry.",
});
expect(await repository.listDescriptionGenerationEvents(run.id)).toEqual(expect.arrayContaining([
expect.objectContaining({
level: "warning",
message: "The model response was not valid JSON. Retrying batch (attempt 2 of 2).",
}),
]));
} finally {
await app.close();
}
});
test("fails safely when the provider fails and redacts provider diagnostics", async () => {
const sensitiveDiagnostic = "test-provider-secret private prompt raw provider payload";
const modelCompleter: ModelCompleter = {
@@ -2528,6 +2597,7 @@ test("fails safely when the provider fails and redacts provider diagnostics", as
});
const { run } = await waitForTerminalRun(app, start.json().id);
expect(modelCompleter.complete).toHaveBeenCalledTimes(2);
expect(run).toMatchObject({
status: "completed_with_errors",
processed: 1,
@@ -2565,7 +2635,7 @@ test.each([
["duplicate mappings", (targetIds: readonly string[]) => ({ results: [
{ targetId: targetIds[0], outcome: "generated", description: "First valid value" },
{ targetId: targetIds[0], outcome: "non_generatable" },
] })],
] }), "The model response was missing one or more requested targets."],
["unknown mappings", (targetIds: readonly string[]) => ({ results: [
{ targetId: targetIds[0], outcome: "non_generatable" },
{
@@ -2573,15 +2643,15 @@ test.each([
outcome: "generated",
description: "Unknown target value",
},
] })],
] }), "The model response was missing one or more requested targets."],
["missing mappings", (targetIds: readonly string[]) => ({ results: [
{ targetId: targetIds[0], outcome: "generated", description: "Only one result" },
] })],
] }), "The model response was missing one or more requested targets."],
["malformed mappings", (targetIds: readonly string[]) => ({ results: [
{ targetId: targetIds[0], outcome: "generated", description: "First valid value" },
{ targetId: targetIds[1], outcome: "generated", description: " " },
] })],
] as const)("rejects %s without applying any result from the batch", async (_name, responseFor) => {
] }), "The model response did not match the required schema."],
] as const)("rejects %s without applying any result from the batch", async (_name, responseFor, failureMessage) => {
let selectedColumnIds: string[] = [];
const modelCompleter: ModelCompleter = {
complete: vi.fn(async () => JSON.stringify(responseFor(selectedColumnIds))),
@@ -2625,6 +2695,7 @@ test.each([
});
const { run } = await waitForTerminalRun(app, start.json().id);
expect(modelCompleter.complete).toHaveBeenCalledTimes(2);
expect(run).toMatchObject({
status: "completed_with_errors",
processed: 2,
@@ -2644,7 +2715,7 @@ test.each([
expect((await repository.listDescriptionGenerationEvents(run.id)).find((event) => event.level === "error")).toEqual(
expect.objectContaining({
level: "error",
message: `The model response was invalid. Affected Catalog Column targets: ${selectedColumnIds.join(", ")}.`,
message: `${failureMessage} Affected Catalog Column targets: ${selectedColumnIds.join(", ")}.`,
}),
);
} finally {
@@ -13,6 +13,7 @@ import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema
import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js";
import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js";
import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js";
import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js";
import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js";
import { loadConfig } from "../src/config.js";
import type { WorkspaceRegistry } from "../src/workspaces/registry.js";
@@ -48,6 +49,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
await upSensitiveDataFlag(db);
await upDescriptionGeneration(db);
await upSensitiveSuggestionRuns(db);
await upAiTokenUsage(db);
const repository = new KyselyCatalogRepository(db);
const database = await repository.create({
workspaceId: "psd-clinical",
@@ -124,7 +126,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
description: "Elenco dei pazienti e dei loro dati clinici.",
}] });
}
if (call === 4) {
if (call === 5) {
return JSON.stringify({ results: [
{
targetId: birthDate.id,
@@ -138,14 +140,14 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
},
] });
}
if (call === 5) {
if (call === 6) {
return JSON.stringify({ results: [{
targetId: table.id,
outcome: "generated",
description: "Descrizione rigenerata della tabella pazienti.",
}] });
}
if (call === 6) {
if (call === 7) {
return JSON.stringify({ results: [{
targetId: birthDate.id,
outcome: "generated",
@@ -340,7 +342,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
expect(await repository.getColumn(database.id, table.id, birthDate.id)).toMatchObject({
generatedDescription: "Descrizione recuperata della data di nascita.",
});
expect(modelCompleter.complete).toHaveBeenCalledTimes(6);
expect(modelCompleter.complete).toHaveBeenCalledTimes(7);
expect(JSON.stringify(vi.mocked(modelCompleter.complete).mock.calls)).toContain(persistedSampleSecret);
const runIds = [
@@ -1,6 +1,9 @@
import { expect, test, vi } from "vitest";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
PostgresDescriptionSourceSampler,
ConcreteDescriptionSourceSampler,
type DescriptionSourceSamplingTarget,
} from "../src/catalog/description-source-sampler.js";
import type {
@@ -8,6 +11,8 @@ import type {
CatalogPostgresAccess,
} from "../src/catalog/postgres-access.js";
import type { WorkspaceDatabase } from "../src/catalog/types.js";
import type { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
import { CATALOG_SECRET_IDS } from "../src/catalog/secrets.js";
const database: WorkspaceDatabase = {
id: "11111111-1111-4111-8111-111111111111",
@@ -51,7 +56,7 @@ test("samples at most five source rows and five distinct non-null examples in a
const access: CatalogPostgresAccess = {
connect: vi.fn(async () => ({ query, end }) as CatalogDatabaseClient),
};
const sampler = new PostgresDescriptionSourceSampler(access);
const sampler = new ConcreteDescriptionSourceSampler(access);
const controller = new AbortController();
const samples = await sampler.sample(database, [target], controller.signal);
@@ -92,13 +97,71 @@ test("samples at most five source rows and five distinct non-null examples in a
expect(end).toHaveBeenCalledOnce();
});
test("samples source rows through the configured REST run_query binding", async () => {
const root = mkdtempSync(join(tmpdir(), "tht-source-rest-"));
const credentialFile = join(root, "api-key");
writeFileSync(credentialFile, "test-api-key\n", { mode: 0o600 });
const release = vi.fn();
const secretStore = {
materialize: vi.fn(() => ({
files: new Map([[CATALOG_SECRET_IDS.apiKey, credentialFile]]),
release,
})),
} as unknown as WorkspaceSecretStore;
const fetchMock = vi.fn(async () => new Response(JSON.stringify([
{ 'status"code': "active", ward: null },
{ 'status"code': "pending", ward: "A" },
]), { status: 200, headers: { "content-type": "application/json" } }));
vi.stubGlobal("fetch", fetchMock);
const access: CatalogPostgresAccess = {
connect: vi.fn(async () => { throw new Error("PostgreSQL access must not be used"); }),
};
const sampler = new ConcreteDescriptionSourceSampler(access, secretStore);
const restDatabase: WorkspaceDatabase = {
...database,
binding: {
transport: "rest_api",
baseUrl: "https://dwh.example.test/root/",
restPath: "/health",
restAuth: "x-api-key",
},
};
try {
await expect(sampler.sample(restDatabase, [target], new AbortController().signal)).resolves.toEqual([{
targetId: target.targetId,
tableName: target.tableName,
rows: [
{ fields: [{ name: 'status"code', value: "active" }, { name: "ward", value: null }] },
{ fields: [{ name: 'status"code', value: "pending" }, { name: "ward", value: "A" }] },
],
representativeValues: [
{ column: 'status"code', values: ["active", "pending"] },
{ column: "ward", values: ["A"] },
],
}]);
expect(access.connect).not.toHaveBeenCalled();
expect(fetchMock).toHaveBeenCalledWith("https://dwh.example.test/root/rpc/run_query", expect.objectContaining({
method: "POST",
headers: { "content-type": "application/json", "x-api-key": "test-api-key" },
body: JSON.stringify({
query_text: 'SELECT LEFT(("status""code")::text, 256) AS "status""code", LEFT(("ward")::text, 256) AS "ward" FROM "clinical""data"."patient""facts" LIMIT 5',
}),
}));
expect(release).toHaveBeenCalledOnce();
} finally {
vi.unstubAllGlobals();
rmSync(root, { recursive: true, force: true });
}
});
test("does not issue a SELECT when a protected target has no source columns", async () => {
const query = vi.fn(async () => ({ rows: [] }));
const end = vi.fn(async () => undefined);
const access: CatalogPostgresAccess = {
connect: vi.fn(async () => ({ query, end }) as CatalogDatabaseClient),
};
const sampler = new PostgresDescriptionSourceSampler(access);
const sampler = new ConcreteDescriptionSourceSampler(access);
const samples = await sampler.sample(database, [{
targetId: target.targetId,
@@ -129,7 +192,7 @@ test("rolls back and closes the source connection when sampling fails", async ()
const access: CatalogPostgresAccess = {
connect: vi.fn(async () => ({ query, end }) as CatalogDatabaseClient),
};
const sampler = new PostgresDescriptionSourceSampler(access);
const sampler = new ConcreteDescriptionSourceSampler(access);
const controller = new AbortController();
await expect(sampler.sample(database, [target], controller.signal)).rejects.toThrow();
@@ -13,6 +13,7 @@ import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_des
import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js";
import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js";
import { up as upLogicalRelationships } from "../src/catalog/migrations/008_catalog_logical_relationships.js";
import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js";
const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
@@ -28,6 +29,9 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
await upSchemaSync(db);
await upSensitiveDataFlag(db);
await upLogicalRelationships(db);
await upDescriptionGeneration(db);
await upSensitiveSuggestionRuns(db);
await upAiTokenUsage(db);
await sql`CREATE ROLE thothii_catalog_runtime`.execute(db);
await upRuntimeSequencePrivileges(db);
const sequencePrivilege = await sql<{ allowed: boolean }>`
@@ -386,6 +390,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
await upLogicalRelationships(db);
await upDescriptionGeneration(db);
await upSensitiveSuggestionRuns(db);
await upAiTokenUsage(db);
const repository = new KyselyCatalogRepository(db);
const firstDatabase = await repository.create({
workspaceId: "generation-one",
@@ -127,7 +127,7 @@ test("resolves only a configured selection for the later generation boundary", (
expect(() => models.resolve("unknown-model")).toThrow(MetadataGenerationModelUnavailableError);
});
test("loads DeepSeek, GLM, and an explicit keyless Qwen endpoint from installation setup", () => {
test("loads DeepSeek models, GLM, and an explicit keyless Qwen endpoint from installation setup", () => {
const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration:
default: glm-53
models:
@@ -135,6 +135,10 @@ test("loads DeepSeek, GLM, and an explicit keyless Qwen endpoint from installati
label: DeepSeek V4 Pro
litellm: {provider: deepseek, model: deepseek-v4-pro}
apiKeyEnv: DEEPSEEK_API_KEY
- id: deepseek-v4-flash
label: DeepSeek V4 Flash
litellm: {provider: deepseek, model: deepseek-v4-flash}
apiKeyEnv: DEEPSEEK_API_KEY
- id: glm-53
label: GLM 5.3
litellm:
@@ -156,6 +160,7 @@ test("loads DeepSeek, GLM, and an explicit keyless Qwen endpoint from installati
expect(models.catalog()).toEqual({
models: [
{ id: "deepseek-v4-pro", label: "DeepSeek V4 Pro" },
{ id: "deepseek-v4-flash", label: "DeepSeek V4 Flash" },
{ id: "glm-53", label: "GLM 5.3" },
{ id: "qwen-36", label: "Qwen 3.6" },
],
+2 -2
View File
@@ -64,7 +64,7 @@ sys.stdout.write(json.dumps({"ok": True, "content": "Descrizione italiana"}))
signal: new AbortController().signal,
});
expect(content).toBe("Descrizione italiana");
expect(content).toEqual({ content: "Descrizione italiana", usage: { input: 0, cacheRead: 0, output: 0 } });
const captured = JSON.parse(readFileSync(join(roots[0]!, "request.json"), "utf8"));
expect(captured.request).toEqual({
model: "openai/gpt-4.1-mini",
@@ -100,7 +100,7 @@ sys.stdout.write(json.dumps({"ok": True, "content": "Descrizione Qwen"}))
},
messages: [{ role: "user", content: "Describe invented metadata." }],
signal: new AbortController().signal,
})).resolves.toBe("Descrizione Qwen");
})).resolves.toEqual({ content: "Descrizione Qwen", usage: { input: 0, cacheRead: 0, output: 0 } });
expect(JSON.parse(readFileSync(join(roots[0]!, "request.json"), "utf8"))).toEqual({
model: "openai/qwen3.6-35b-a3b",
@@ -17,6 +17,12 @@ metadataGeneration:
provider: deepseek
model: deepseek-v4-pro
apiKeyEnv: DEEPSEEK_API_KEY
- id: deepseek-v4-flash
label: DeepSeek V4 Flash
litellm:
provider: deepseek
model: deepseek-v4-flash
apiKeyEnv: DEEPSEEK_API_KEY
- id: glm-53
label: GLM 5.3
litellm:
+1
View File
@@ -108,6 +108,7 @@ not supported by this installation until the declarative contract is extended ge
| Model | Level | Why | Visibility |
|---|---|---|---|
| `deepseek/deepseek-v4-pro` | Built-in Pi | Known public API, already in the build | All projects |
| `deepseek/deepseek-v4-flash` | Built-in Pi | Known public API, already in the build | All projects |
| `zai/glm-5.3` | `deploy/pi/models.json` | Custom OpenAI-compatible endpoint | ThothII installation |
| `local-qwen/qwen3.6-35b-a3b` | `deploy/pi/models.json` | Locally configured OpenAI-compatible endpoint | ThothII installation |
@@ -179,12 +179,12 @@ canonico; i percorsi e i comportamenti descrivono il sorgente disponibile il 202
e Schema Sync. Il renderer e il runtime delle sessioni NL→SQL restano fuori scope e continuano a
rifiutarla finché non verrà deciso il relativo cutover.
55. I menu di azione a livello Workspace Database espongono separatamente `Synchronize tables`,
`Synchronize all columns`, `Synchronize relationships` e `Synchronize all`. Su una selezione di
`Synchronize relationships` e `Synchronize all`. Su una selezione di
database lo scope scelto viene avviato per ogni database idoneo; non viene sostituito
implicitamente con una sincronizzazione completa.
56. Per lo scope Columns, `tableIds` vuoto significa tutte le Catalog Table correnti del Workspace
Database; `tableIds` valorizzato limita invece la riconciliazione alle tabelle indicate. La grid
Tables espone `Synchronize columns` sulle tabelle selezionate.
56. Lo scope Columns è disponibile dalla grid Tables e limita la riconciliazione alle tabelle
selezionate; la pagina Columns non espone azioni di sincronizzazione. La grid Tables espone
`Synchronize columns` sulle tabelle selezionate.
## Correzione del modello mentale corrente
@@ -570,9 +570,9 @@ Le griglie che dispongono di azioni massive usano checkbox e una toolbar contest
menu `Actions` e cancellazione della selezione. La selezione identifica ID espliciti, può essere
accumulata attraverso i filtri e viene azzerata dopo successo, nuova sincronizzazione o uscita
dalla pagina; un'azione è all-or-nothing se un elemento non è idoneo. I menu a livello database
espongono gli scope fisici come azioni distinte: `Synchronize tables`, `Synchronize all columns`,
`Synchronize relationships` e `Synchronize all`. La grid Tables espone invece `Synchronize
columns` per le tabelle selezionate. Le selezioni database aggiungono `Delete all tables` e
espongono gli scope fisici come azioni distinte: `Synchronize tables`, `Synchronize relationships`
e `Synchronize all`. La grid Tables espone invece `Synchronize columns` per le tabelle selezionate;
la pagina Columns non espone sincronizzazione. Le selezioni database aggiungono `Delete all tables` e
`Delete all relationships`; le selezioni tabelle aggiungono `Delete all columns` e `Delete all
relationships`. Queste operazioni sono atomiche, richiedono conferma e non modificano database
esterno, binding, configurazione o segreti. Test connection resta un'azione distinta; griglie senza
@@ -17,6 +17,7 @@ The installation-level model catalog contained these entries, with `glm-53` as t
| Selection ID | LiteLLM route | Authentication |
| --- | --- | --- |
| `deepseek-v4-pro` | `deepseek/deepseek-v4-pro` | Protected `DEEPSEEK_API_KEY` value already used by core |
| `deepseek-v4-flash` | `deepseek/deepseek-v4-flash` | Protected `DEEPSEEK_API_KEY` value already used by core |
| `glm-53` | `openai/glm-5.3` on the Z.AI coding endpoint | Protected `ZAI_API_KEY` value already used by core |
| `qwen-36` | `openai/qwen3.6-35b-a3b` on the VPN-only AritmoLab endpoint | No operator API key |
@@ -72,6 +72,28 @@ const table = {
updatedAt: "2026-08-27T09:00:00Z",
};
const descriptionRun = {
id: "77777777-7777-4777-8777-777777777777",
databaseId,
scope: "selected_columns",
modelId: "glm-53",
language: "it",
status: "completed",
total: 5,
processed: 5,
generated: 4,
nonGeneratable: 1,
failed: 0,
inputTokens: 1200,
cacheReadTokens: 80,
outputTokens: 420,
createdAt: "2026-08-31T09:00:00Z",
startedAt: "2026-08-31T09:00:01Z",
updatedAt: "2026-08-31T09:00:04Z",
finishedAt: "2026-08-31T09:00:04Z",
errorSummary: null,
};
function metrics(requestedDatabaseId: string | null) {
return {
scope: requestedDatabaseId ? "database" : "global",
@@ -133,7 +155,8 @@ async function expectContextPanelGeometry(page: Page, accessibleName: string) {
const managerCenter = managerBox.x + managerBox.width / 2;
const panelCenter = panelBox.x + panelBox.width / 2;
expect(Math.abs(panelCenter - managerCenter)).toBeLessThanOrEqual(1);
expect(Math.abs(panelBox.width / managerBox.width - 0.6)).toBeLessThanOrEqual(0.005);
const expectedPanelWidth = Math.min(1200, managerBox.width - (managerBox.width <= 768 ? 24 : 32));
expect(Math.abs(panelBox.width - expectedPanelWidth)).toBeLessThanOrEqual(1);
expect(panelBox.x).toBeGreaterThanOrEqual(managerBox.x - 1);
expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(managerBox.x + managerBox.width + 1);
expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(sessionRailBox.x + 1);
@@ -297,7 +320,8 @@ async function expectWorkAreaPanelGeometry(page: Page, accessibleName: string) {
expect(Math.abs(
panelBox.x + panelBox.width / 2 - (workAreaBox.x + workAreaBox.width / 2),
)).toBeLessThanOrEqual(1);
expect(Math.abs(panelBox.width / workAreaBox.width - 0.6)).toBeLessThanOrEqual(0.005);
const expectedPanelWidth = Math.min(1200, workAreaBox.width - (workAreaBox.width <= 768 ? 24 : 32));
expect(Math.abs(panelBox.width - expectedPanelWidth)).toBeLessThanOrEqual(1);
expect(panelBox.x).toBeGreaterThanOrEqual(workAreaBox.x - 1);
expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(workAreaBox.x + workAreaBox.width + 1);
expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(sessionRailBox.x + 1);
@@ -344,12 +368,20 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb
"GET /api/catalog/metadata-generation/models": {
models: [
{ id: "deepseek-v4-pro", label: "DeepSeek V4 Pro" },
{ id: "deepseek-v4-flash", label: "DeepSeek V4 Flash" },
{ id: "glm-53", label: "GLM 5.3" },
{ id: "qwen-36", label: "AritmoLab Qwen 3.6 35B A3B" },
],
default: "glm-53",
},
"GET /api/catalog/description-generation-runs?limit=50": [],
"GET /api/catalog/description-generation-runs?limit=50": [descriptionRun],
[`GET /api/catalog/description-generation-runs/${descriptionRun.id}`]: descriptionRun,
[`GET /api/catalog/description-generation-runs/${descriptionRun.id}/events-list?after=0`]: [{
sequence: 1,
level: "info",
message: "Description generation completed.",
createdAt: "2026-08-31T09:00:04Z",
}],
"GET /api/catalog/sensitive-data-suggestion-runs?limit=50": [],
[`GET /api/catalog/databases/${databaseId}/tables`]: [table],
};
@@ -387,6 +419,7 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb
const adminNavigationRail = page.getByRole("complementary", { name: "Session navigation" });
const administration = adminNavigationRail.getByRole("button", { name: "Administration", exact: true });
await expect(administration).toHaveCSS("font-family", /Manrope/);
await expect(administration).toHaveAttribute("aria-expanded", "false");
await expect(adminNavigationRail.getByRole("region", { name: "Administration" })).toHaveCount(0);
await administration.click();
@@ -435,6 +468,7 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb
await expect(metadataModelSelector).toHaveValue("glm-53");
await expect(metadataModelSelector.locator("option")).toHaveText([
"DeepSeek V4 Pro",
"DeepSeek V4 Flash",
"GLM 5.3",
"AritmoLab Qwen 3.6 35B A3B",
]);
@@ -488,10 +522,55 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb
.click();
await expectContextPanelGeometry(page, "Description generation");
await expectContextPanelHeaderUsesPrimary(page, "Description generation");
await page
.getByRole("dialog", { name: "Description generation" })
.getByRole("button", { name: /close/i })
.click();
const descriptionPanel = page.getByRole("dialog", { name: "Description generation" });
const descriptionBody = descriptionPanel.locator(
':scope > [data-catalog-panel-region="body"]',
);
const progressTable = descriptionPanel.getByRole("table", {
name: "Description generation progress",
});
const eventLog = descriptionPanel.getByRole("log", {
name: "Description generation events",
});
const recentRuns = descriptionPanel.getByRole("region", {
name: "Description generation history",
});
await expect(progressTable.getByRole("columnheader")).toHaveCount(5);
await expect(eventLog).toBeVisible();
await expect(recentRuns).toBeVisible();
const [bodyBox, eventLogBox, recentRunsBox] = await Promise.all([
descriptionBody.boundingBox(),
eventLog.boundingBox(),
recentRuns.boundingBox(),
]);
expect(bodyBox).not.toBeNull();
expect(eventLogBox).not.toBeNull();
expect(recentRunsBox).not.toBeNull();
if (bodyBox && eventLogBox && recentRunsBox) {
const eventBottomGutter = bodyBox.y + bodyBox.height - eventLogBox.y - eventLogBox.height;
const recentBottomGutter = bodyBox.y + bodyBox.height - recentRunsBox.y - recentRunsBox.height;
expect(eventBottomGutter).toBeGreaterThanOrEqual(10);
expect(recentBottomGutter).toBeGreaterThanOrEqual(10);
expect(eventLogBox.x + eventLogBox.width).toBeLessThan(recentRunsBox.x);
}
await page.setViewportSize({ width: 720, height: 800 });
const [compactManagerBox, compactDescriptionPanelBox] = await Promise.all([
page.getByTestId("conversation-column").getByRole("main", {
name: "Database management",
}).boundingBox(),
descriptionPanel.boundingBox(),
]);
expect(compactManagerBox).not.toBeNull();
expect(compactDescriptionPanelBox).not.toBeNull();
if (compactManagerBox && compactDescriptionPanelBox) {
expect(compactManagerBox.width - compactDescriptionPanelBox.width).toBeGreaterThanOrEqual(20);
}
expect(await descriptionBody.evaluate((element) => getComputedStyle(element).display)).not.toBe("grid");
await page.setViewportSize({ width: 1910, height: 911 });
await descriptionPanel.getByRole("button", { name: /close/i }).click();
const applicationBar = page.locator(
'main[aria-label="Database management"] .thot-fleet-ledger__header',
@@ -734,7 +813,7 @@ test("Workspace and Pi management share the centered work-area panel without cov
expect(compactWorkAreaBox).not.toBeNull();
expect(compactPanelBox).not.toBeNull();
if (compactWorkAreaBox && compactPanelBox) {
expect(Math.abs(compactPanelBox.width - compactWorkAreaBox.width + 16)).toBeLessThanOrEqual(1);
expect(Math.abs(compactPanelBox.width - compactWorkAreaBox.width + 24)).toBeLessThanOrEqual(1);
expect(Math.abs(
compactPanelBox.x + compactPanelBox.width / 2
- (compactWorkAreaBox.x + compactWorkAreaBox.width / 2),
@@ -269,7 +269,6 @@ export function FleetSelectionActions({ props, direction = "row" }: { props: Var
if (state.inventoryLevel === "databases") actions = [
{ id: "databases-test", group: "Connection", label: "Test connections", icon: <TestTube2 size={14} />, disabled, onClick: run(`Testing ${count} selected connection${count === 1 ? "" : "s"}`) },
{ id: "databases-sync-tables", group: "Synchronization", label: "Synchronize tables", icon: <RefreshCw size={14} />, disabled, onClick: run("Table synchronization started", "running") },
{ id: "databases-sync-columns", group: "Synchronization", label: "Synchronize all columns", icon: <Columns3 size={14} />, disabled, onClick: run("Column synchronization started", "running") },
{ id: "databases-sync-relationships", group: "Synchronization", label: "Synchronize relationships", icon: <Activity size={14} />, disabled, onClick: run("Relationship synchronization started", "running") },
{ id: "databases-sync-all", group: "Synchronization", label: "Synchronize all", icon: <RefreshCw size={14} />, disabled, onClick: run("Full schema synchronization started", "running") },
{ id: "databases-generate-all", group: "Descriptions", label: "Generate all descriptions", icon: <Sparkles size={14} />, disabled, onClick: run("Generate all confirmation opened") },
+6
View File
@@ -157,6 +157,9 @@ export interface SensitiveDataSuggestionRun {
total: number;
suggestedSensitive: number;
suggestedNonSensitive: number;
inputTokens?: number;
cacheReadTokens?: number;
outputTokens?: number;
createdAt: string;
startedAt: string | null;
updatedAt: string;
@@ -297,6 +300,9 @@ export interface DescriptionGenerationRun {
generated: number;
nonGeneratable: number;
failed: number;
inputTokens?: number;
cacheReadTokens?: number;
outputTokens?: number;
createdAt: string;
startedAt: string | null;
updatedAt: string;
+1 -1
View File
@@ -4,7 +4,7 @@ import { cva, type VariantProps } from "class-variance-authority"
import { cn } from "@/lib/utils"
const buttonVariants = cva(
"group/button inline-flex shrink-0 items-center justify-center rounded-md border border-transparent bg-clip-padding text-sm font-semibold tracking-[0.005em] whitespace-nowrap transition-all outline-none select-none focus-visible:ring-3 focus-visible:ring-ring/25 disabled:pointer-events-none disabled:opacity-50 aria-invalid:border-destructive aria-invalid:ring-3 aria-invalid:ring-destructive/20 dark:aria-invalid:border-destructive/50 dark:aria-invalid:ring-destructive/40 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
"group/button inline-flex shrink-0 items-center justify-center rounded-md border border-transparent bg-clip-padding font-sans text-sm font-semibold tracking-[0.005em] whitespace-nowrap transition-all outline-none select-none focus-visible:ring-3 focus-visible:ring-ring/25 disabled:pointer-events-none disabled:opacity-50 aria-invalid:border-destructive aria-invalid:ring-3 aria-invalid:ring-destructive/20 dark:aria-invalid:border-destructive/50 dark:aria-invalid:ring-destructive/40 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
{
variants: {
variant: {
@@ -67,6 +67,7 @@ describe("authenticated shell permissions", () => {
const sessionNavigation = screen.getByRole("complementary", { name: "Session navigation" });
const trigger = await within(sessionNavigation).findByRole("button", { name: "Administration" });
expect(trigger).toHaveClass("font-sans");
expect(trigger).toHaveAttribute("aria-expanded", "false");
expect(within(sessionNavigation).queryByRole("region", { name: "Administration" })).not.toBeInTheDocument();
+24 -2
View File
@@ -45,6 +45,23 @@ interface AppShellProps {
type ActiveSurface = "core" | "database-management";
type ActiveManagementPanel = "workspace" | "pi" | null;
const SESSION_SCOPES: readonly SessionScope[] = ["mine", "all"];
const APP_NAVIGATION_STORAGE_KEY = "thothii:app-navigation";
function readPersistedNavigation(): { surface: ActiveSurface; panel: ActiveManagementPanel } {
if (import.meta.env.MODE === "test") return { surface: "core", panel: null };
try {
const parsed = JSON.parse(window.localStorage.getItem(APP_NAVIGATION_STORAGE_KEY) ?? "null") as {
surface?: unknown;
panel?: unknown;
} | null;
return {
surface: parsed?.surface === "database-management" ? "database-management" : "core",
panel: parsed?.panel === "workspace" || parsed?.panel === "pi" ? parsed.panel : null,
};
} catch {
return { surface: "core", panel: null };
}
}
function sessionScopeTabClass(selected: boolean): string {
const base = "relative -mb-px flex h-8 w-full cursor-pointer items-center justify-center rounded-t-md border border-b-2 px-2 text-xs font-semibold tracking-[0.005em] outline-none focus-visible:z-10 focus-visible:ring-3 focus-visible:ring-[oklch(var(--nav-active-border)/0.28)]";
@@ -166,7 +183,8 @@ export function AppShell({ canLogout }: AppShellProps) {
const queryClient = useQueryClient();
const [showActivity, setShowActivity] = useState(false);
const [activeSurface, setActiveSurface] = useState<ActiveSurface>("core");
const persistedNavigation = readPersistedNavigation();
const [activeSurface, setActiveSurface] = useState<ActiveSurface>(persistedNavigation.surface);
const databaseManagementPresentation = resolveDatabaseManagementPresentation({
isDevelopment: import.meta.env.DEV,
mode: import.meta.env.MODE,
@@ -177,7 +195,7 @@ export function AppShell({ canLogout }: AppShellProps) {
const updateDatabaseNavigationState = useCallback((state: { dirty: boolean; busy: boolean }) => {
databaseNavigationRef.current = state;
}, []);
const [activeManagementPanel, setActiveManagementPanel] = useState<ActiveManagementPanel>(null);
const [activeManagementPanel, setActiveManagementPanel] = useState<ActiveManagementPanel>(persistedNavigation.panel);
const [adminNavigationValue, setAdminNavigationValue] = useState<string[]>([]);
const [activeOpen, setActiveOpen] = useState(true);
const [archiveOpen, setArchiveOpen] = useState(false);
@@ -188,6 +206,10 @@ export function AppShell({ canLogout }: AppShellProps) {
const [collapsedGroups, setCollapsedGroups] = useState<Record<string, boolean>>({});
const [renameGroupTarget, setRenameGroupTarget] = useState<string | null>(null);
const operationEpochRef = useRef(0);
useEffect(() => {
if (import.meta.env.MODE === "test") return;
window.localStorage.setItem(APP_NAVIGATION_STORAGE_KEY, JSON.stringify({ surface: activeSurface, panel: activeManagementPanel }));
}, [activeManagementPanel, activeSurface]);
useEffect(() => () => { operationEpochRef.current += 1; }, []);
const groups = useMemo(
@@ -197,7 +197,6 @@ function registerCompletedSyncRun(run: CatalogSyncRun) {
const synchronizationScopes = [
{ scope: "tables", label: "Synchronize tables" },
{ scope: "columns", label: "Synchronize all columns" },
{ scope: "relationships", label: "Synchronize relationships" },
{ scope: "all", label: "Synchronize all" },
] as const;
@@ -234,9 +233,7 @@ test("renders Fleet Ledger with real metrics, conceptual row tooltips, and persi
expect(screen.queryByText("Inspect physical metadata, curate descriptions and control catalog operations.")).not.toBeInTheDocument();
expect(screen.queryByRole("button", { name: "Back to workspace" })).not.toBeInTheDocument();
expect(screen.queryByRole("button", { name: /Add database/i })).not.toBeInTheDocument();
expect(screen.getByRole("columnheader", { name: /Revision \/ Evidence/ })).toBeVisible();
expect(screen.getByRole("columnheader", { name: /NL→SQL runtime/ })).toBeVisible();
expect(screen.getByRole("columnheader", { name: /Metadata Catalog/ })).toBeVisible();
expect(screen.getByRole("columnheader", { name: /Catalog status/ })).toBeVisible();
const summary = screen.getByRole("region", { name: "Fleet summary" });
expect(await within(summary).findByText("2,275")).toBeVisible();
expect(within(summary).getByText("75%")).toBeVisible();
@@ -261,13 +258,14 @@ test("renders Fleet Ledger with real metrics, conceptual row tooltips, and persi
await user.click(within(restoredDatabaseRow).getByRole("checkbox", { name: /toggle row selection/i }));
const actionPicker = screen.getByRole("combobox", { name: "Batch action" });
expect(within(actionPicker).getByRole("option", { name: /^Synchronize all(?:,|$)/ })).toBeVisible();
expect(within(actionPicker).queryByRole("option", { name: "Synchronize all columns" })).not.toBeInTheDocument();
await user.selectOptions(actionPicker, "generate-missing");
expect(screen.getAllByRole("combobox", { name: "Metadata-generation LLM model" })).toHaveLength(1);
expect(modelSelector).toHaveValue("local-qwen");
});
test("shows repository, NL→SQL runtime, and Metadata Catalog states independently", async () => {
test("shows synchronization, description, and sensitive-data states independently", async () => {
const configured = makeDatabase({ connectionStatus: "reachable", testedVersion: 3 });
const needsRuntimeConfiguration = makeDatabase({
...unconfigured,
@@ -281,21 +279,58 @@ test("shows repository, NL→SQL runtime, and Metadata Catalog states independen
renderPage({ rows: [configured, needsRuntimeConfiguration, orphan], presentation: "fleet" });
const configuredRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
expect(within(configuredRow).getByText("Active revision aaaaaaa")).toBeVisible();
expect(within(configuredRow).getByText("Evidence materialized · filesystem")).toBeVisible();
expect(within(configuredRow).getByText("Ready")).toBeVisible();
expect(within(configuredRow).getByText("Configured")).toBeVisible();
expect(within(configuredRow).getByText("Connection reachable")).toBeVisible();
expect(within(configuredRow).getByText("Not synchronized")).toBeVisible();
expect(within(configuredRow).getByText(/Descriptions ·/)).toBeVisible();
expect(within(configuredRow).getByText(/Sensitive ·/)).toBeVisible();
const needsConfigurationRow = screen.getByRole("row", { name: /Research laboratory/ });
expect(within(needsConfigurationRow).getByText("Evidence credentials required · http")).toBeVisible();
expect(within(needsConfigurationRow).getByText("Configuration required")).toBeVisible();
expect(within(needsConfigurationRow).getByText("Not configured")).toBeVisible();
expect(within(needsConfigurationRow).getByText("Not synchronized")).toBeVisible();
expect(within(needsConfigurationRow).getByText(/Descriptions ·/)).toBeVisible();
const orphanRow = screen.getByRole("row", { name: /Retired workspace/ });
expect(within(orphanRow).getByText("Workspace missing")).toBeVisible();
expect(within(orphanRow).getByText("Unavailable")).toBeVisible();
expect(within(orphanRow).getByText("Orphaned configuration")).toBeVisible();
expect(within(orphanRow).getByText("Not synchronized")).toBeVisible();
});
test("shows the configured access type and endpoint in catalog status", async () => {
const direct = makeDatabase();
const rest = makeDatabase({
id: "33333333-3333-4333-8333-333333333333",
workspaceId: "rest-lab",
workspaceName: "REST laboratory",
binding: {
transport: "rest_api",
baseUrl: "https://203.0.113.18/dwh/",
restPath: "/health",
restAuth: "x-api-key",
},
});
const ssh = makeDatabase({
id: "44444444-4444-4444-8444-444444444444",
workspaceId: "ssh-lab",
workspaceName: "SSH laboratory",
binding: {
transport: "ssh_tunnel",
sshHost: "bastion.example.test",
sshPort: 2222,
sshUsername: "operator",
sshTargetHost: "postgres.internal",
sshTargetPort: 5432,
username: "reader",
},
});
renderPage({ rows: [direct, rest, ssh], presentation: "fleet" });
const directRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
expect(within(directRow).getByText("Direct PostgreSQL")).toBeVisible();
expect(within(directRow).getByText("db.internal:5432")).toBeVisible();
const restRow = screen.getByRole("row", { name: /REST laboratory/ });
expect(within(restRow).getByText("REST")).toBeVisible();
expect(within(restRow).getByText("203.0.113.18")).toBeVisible();
const sshRow = screen.getByRole("row", { name: /SSH laboratory/ });
expect(within(sshRow).getByText("SSH")).toBeVisible();
expect(within(sshRow).getByText("bastion.example.test:2222")).toBeVisible();
});
test("keeps Fleet Ledger visible when a catalog response omits the Evidence projection", async () => {
@@ -306,7 +341,7 @@ test("keeps Fleet Ledger visible when a catalog response omits the Evidence proj
expect(await screen.findByRole("heading", { name: "Database management" })).toBeVisible();
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
expect(within(databaseRow).getByText("Evidence state unavailable")).toBeVisible();
expect(within(databaseRow).getByText("Not synchronized")).toBeVisible();
});
test("opens the relationship map directly from a Fleet database and restores focus on return", async () => {
@@ -359,7 +394,7 @@ test("offers catalog configuration directly on an unconfigured Fleet workspace",
expect(within(drawer).getByLabelText("Workspace")).toHaveValue("Research laboratory");
});
test("keeps only the local back action while browsing Fleet Ledger columns", async () => {
test("keeps the table back action in the Fleet Ledger breadcrumb while browsing columns", async () => {
const user = userEvent.setup();
server.use(
http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([patientsTable])),
@@ -372,9 +407,17 @@ test("keeps only the local back action while browsing Fleet Ledger columns", asy
name: "View tables for Policlinico San Donato",
}));
expect(await screen.findByRole("button", { name: "Back to databases" })).toBeVisible();
expect(screen.getByRole("combobox", { name: "Table action" })).toBeDisabled();
expect(screen.queryByRole("button", { name: "Sync history" })).not.toBeInTheDocument();
expect(screen.queryByRole("button", { name: "Description history" })).not.toBeInTheDocument();
expect(screen.queryByRole("button", { name: "Sensitive history" })).not.toBeInTheDocument();
await user.click(await screen.findByRole("button", { name: "View columns for patients" }));
expect(await screen.findByRole("button", { name: "Back to tables" })).toBeVisible();
const tableBackAction = await screen.findByRole("button", { name: "Back to tables" });
expect(tableBackAction).toBeVisible();
expect(screen.getByRole("combobox", { name: "Column action" })).toBeDisabled();
expect(screen.getByRole("navigation", { name: "Database hierarchy" }).parentElement)
.toContainElement(tableBackAction);
await waitFor(() => {
expect(screen.queryByRole("button", { name: "Back to databases" })).not.toBeInTheDocument();
});
@@ -427,7 +470,7 @@ test("reopens database synchronization history when no run is active", async ()
renderPage({ rows: [makeDatabase()], presentation: "fleet" });
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
await user.click(within(databaseRow).getByRole("button", { name: "View synchronization history for Policlinico San Donato" }));
await user.click(within(databaseRow).getByText("Policlinico San Donato"));
const drawer = await screen.findByRole("dialog", { name: "Schema synchronization" });
expect(drawer).toBeVisible();
@@ -628,6 +671,7 @@ test.each(synchronizationScopes)(
await user.click(screen.getByRole("menuitem", { name: label }));
await waitFor(() => expect(startBody).toEqual({ version: 3, scope, tableIds: [] }));
expect(screen.getByText("1 selected")).toBeVisible();
},
);
@@ -1001,15 +1045,16 @@ test("selected database Actions confirms and deletes catalog tables for the full
],
target: "tables",
}));
await waitFor(() => expect(screen.queryByText("2 selected")).not.toBeInTheDocument());
await waitFor(() => expect(screen.getByRole("textbox", { name: "Search databases" })).toHaveFocus());
expect(screen.getByText("2 selected")).toBeVisible();
await user.click(screen.getByRole("button", { name: "Clear" }));
expect(screen.queryByText("2 selected")).not.toBeInTheDocument();
});
test("presents completed synchronization steps as success and skips unneeded confirmation", async () => {
test("presents completed database synchronization steps as success and skips unneeded confirmation", async () => {
const user = userEvent.setup();
const run = {
...makeSyncRun("columns"),
counts: { tables: 163, columns: 2_275 },
...makeSyncRun("all"),
counts: { tables: 163, columns: 2_275, relationships: 18 },
};
registerCompletedSyncRun(run);
server.use(http.post("/api/catalog/databases/:databaseId/sync-runs", () => (
@@ -1022,7 +1067,7 @@ test("presents completed synchronization steps as success and skips unneeded con
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
await user.click(within(databaseRow).getByRole("checkbox", { name: /toggle row selection/i }));
await user.click(screen.getByRole("button", { name: "Actions" }));
await user.click(await screen.findByRole("menuitem", { name: "Synchronize all columns" }));
await user.click(await screen.findByRole("menuitem", { name: "Synchronize all" }));
expect(await screen.findByRole("heading", { name: "Succeeded" })).toBeVisible();
const steps = screen.getByRole("list", { name: "Synchronization steps" });
@@ -1039,7 +1084,7 @@ test("presents completed synchronization steps as success and skips unneeded con
expect(steps.querySelectorAll(".text-primary")).toHaveLength(0);
});
test("database Overview exposes every synchronization scope", async () => {
test("database Overview exposes the database synchronization scopes", async () => {
const user = userEvent.setup();
let startBody: unknown;
const run = makeSyncRun("relationships");
@@ -1520,6 +1565,37 @@ test("starts description generation for multiple selected tables", async () => {
expect(screen.getByText("2 selected")).toBeVisible();
});
test("generates descriptions for every column in selected tables", async () => {
const user = userEvent.setup();
const queuedRun = makeDescriptionGenerationRun({ scope: "selected_columns", total: 1 });
let startBody: unknown;
server.use(
http.get("/api/catalog/metadata-generation/models", () => HttpResponse.json({ models: [{ id: "local-qwen", label: "Local Qwen" }], default: "local-qwen" })),
http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([patientsTable])),
http.get(`/api/catalog/databases/:databaseId/tables/${patientsTable.id}/columns`, () => HttpResponse.json([patientIdColumn])),
http.post("/api/catalog/databases/:databaseId/description-generation-runs", async ({ request }) => {
startBody = await request.json();
return HttpResponse.json(queuedRun, { status: 202 });
}),
http.get("/api/catalog/description-generation-runs/:runId", () => HttpResponse.json(queuedRun)),
http.get("/api/catalog/description-generation-runs/:runId/events-list", () => HttpResponse.json([])),
);
renderPage({ rows: [makeDatabase({ connectionStatus: "reachable", testedVersion: 3 })] });
await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" }));
await user.click(screen.getByRole("tab", { name: "Tables" }));
await user.click(within(await screen.findByRole("row", { name: /patients/ })).getByRole("checkbox"));
await user.click(screen.getByRole("button", { name: "Actions" }));
await user.click(await screen.findByRole("menuitem", { name: "Generate descriptions for all columns" }));
await waitFor(() => expect(startBody).toEqual({
modelId: "local-qwen",
scope: "selected_columns",
targetIds: [patientIdColumn.id],
}));
expect(await screen.findByText("Column description generation started for 1 column")).toBeVisible();
});
test("observes an active run from another browser and reopens a terminal run from history", async () => {
const user = userEvent.setup();
const activeRun = makeDescriptionGenerationRun({
@@ -1913,6 +1989,12 @@ test("starts one selected column with the configured default model", async () =>
await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" }));
await user.click(screen.getByRole("tab", { name: "Tables" }));
await user.click(await screen.findByRole("button", { name: "View columns for patients" }));
expect(screen.queryByRole("button", { name: "Sync columns" })).not.toBeInTheDocument();
expect(screen.getByRole("columnheader", { name: /Select/i })).toBeVisible();
expect(await screen.findByRole("checkbox", { name: "Sensitive data for id" })).toHaveAttribute(
"title",
"Mark this column as sensitive; this does not select it for actions.",
);
const columnRow = (await screen.findAllByRole("row", { name: /Patient identifier/ }))
.find((row) => within(row).queryByRole("checkbox", { name: /toggle row selection/i }));
expect(columnRow).toBeDefined();
+92 -8
View File
@@ -80,6 +80,20 @@ const SYNC_STARTED_MESSAGES: Record<CatalogSyncScope, string> = {
relationships: "Relationship synchronization started",
all: "Full schema synchronization started",
};
const DATABASE_SCREEN_STORAGE_KEY = "thothii:database-management-screen";
function readPersistedDatabaseScreen(): DatabaseScreen {
if (import.meta.env.MODE === "test") return { kind: "list" };
try {
const parsed = JSON.parse(window.localStorage.getItem(DATABASE_SCREEN_STORAGE_KEY) ?? "null") as { kind?: unknown; workspaceId?: unknown } | null;
if ((parsed?.kind === "tables" || parsed?.kind === "relationships") && typeof parsed.workspaceId === "string") {
return { kind: parsed.kind, workspaceId: parsed.workspaceId };
}
} catch {
// Ignore malformed persisted navigation and return to the safe list view.
}
return { kind: "list" };
}
interface Props {
canManage: boolean;
@@ -156,12 +170,13 @@ export function DatabaseManagementPage({
(run) => run.status === "running",
);
const [screen, setScreen] = useState<DatabaseScreen>({ kind: "list" });
const [screen, setScreen] = useState<DatabaseScreen>(readPersistedDatabaseScreen);
const [selectedMetadataModel, setSelectedMetadataModel] = useState("");
const [draft, setDraft] = useState<DatabaseFormDraft | null>(null);
const [baseline, setBaseline] = useState("");
const [formSource, setFormSource] = useState<FormSource | null>(null);
const [search, setSearch] = useState("");
const [selectedDatabaseId, setSelectedDatabaseId] = useState<string | null>(null);
const [busyAction, setBusyAction] = useState<DatabaseBusyAction>(null);
const [stale, setStale] = useState(false);
const [staleBannerOpen, setStaleBannerOpen] = useState(true);
@@ -171,6 +186,11 @@ export function DatabaseManagementPage({
busy: false,
});
const [tableNestedNavigationActive, setTableNestedNavigationActive] = useState(false);
const [tableNestedNavigationBack, setTableNestedNavigationBack] = useState<(() => void) | null>(null);
const handleTableNestedNavigationChange = useCallback((active: boolean, onBack?: () => void) => {
setTableNestedNavigationActive(active);
setTableNestedNavigationBack(() => (active ? onBack ?? null : null));
}, []);
const [activeSyncRun, setActiveSyncRun] = useState<CatalogSyncRun | null>(null);
const [syncHistoryDatabaseId, setSyncHistoryDatabaseId] = useState<string | null>(null);
const [syncDrawerOpen, setSyncDrawerOpen] = useState(false);
@@ -184,6 +204,12 @@ export function DatabaseManagementPage({
suggestions: SensitiveDataSuggestion[];
} | null>(null);
useEffect(() => {
if (import.meta.env.MODE === "test") return;
const persisted = screen.kind === "tables" || screen.kind === "relationships" ? screen : { kind: "list" };
window.localStorage.setItem(DATABASE_SCREEN_STORAGE_KEY, JSON.stringify(persisted));
}, [screen]);
const originRef = useRef<HTMLElement | null>(null);
const searchInputRef = useRef<HTMLInputElement>(null);
const formHeadingRef = useRef<HTMLHeadingElement>(null);
@@ -284,6 +310,7 @@ export function DatabaseManagementPage({
setPartialSecretFailure(null);
setTablesNavigationState({ dirty: false, busy: false });
setTableNestedNavigationActive(false);
setTableNestedNavigationBack(null);
restoreListFocus();
}, [restoreListFocus]);
@@ -346,6 +373,7 @@ export function DatabaseManagementPage({
setPartialSecretFailure(null);
setTablesNavigationState({ dirty: false, busy: false });
setTableNestedNavigationActive(false);
setTableNestedNavigationBack(null);
setScreen({ kind: "tables", workspaceId: row.workspaceId });
}, []);
@@ -358,6 +386,8 @@ export function DatabaseManagementPage({
setStale(false);
setPartialSecretFailure(null);
setTablesNavigationState({ dirty: false, busy: false });
setTableNestedNavigationActive(false);
setTableNestedNavigationBack(null);
setScreen({ kind: "relationships", workspaceId: row.workspaceId });
}, []);
@@ -856,6 +886,11 @@ export function DatabaseManagementPage({
}
}, [invalidateCatalogMetrics, queryClient]);
const clearActiveCatalogTables = useCallback(async () => {
if (!activeRow?.id) return;
await deleteSelectedMetadata([activeRow], "tables");
}, [activeRow, deleteSelectedMetadata]);
const syncDatabase = useCallback(async (scope: CatalogSyncScope) => {
if (!activeRow?.id || !activeRow.configured) return;
try {
@@ -1038,13 +1073,12 @@ export function DatabaseManagementPage({
onOpenOverview={() => openOverview(activeRow)}
onOpenRelationships={() => openRelationships(activeRow)}
onNavigationStateChange={setTablesNavigationState}
onNestedNavigationChange={setTableNestedNavigationActive}
onNestedNavigationChange={handleTableNestedNavigationChange}
onRunStarted={rememberSyncRun}
onOpenSync={() => openSync(activeRow)}
onOpenDescriptionHistory={openDescriptionGenerationHistory}
onOpenSensitiveHistory={openSensitiveDataSuggestionHistory}
onDescriptionGenerationRunStarted={rememberDescriptionGenerationRun}
onSuggestSensitive={suggestActiveDatabaseSensitiveFields}
onClearCatalogTables={clearActiveCatalogTables}
onCatalogMetricsChanged={invalidateCatalogMetrics}
/>
) : screen.kind === "relationships" && relationshipsVisible ? (
@@ -1088,12 +1122,14 @@ export function DatabaseManagementPage({
onEdit={editRow}
onDelete={deleteRow}
onOpenSync={openSync}
onSelectionChange={(selected) => setSelectedDatabaseId(selected.length === 1 ? selected[0].id ?? null : null)}
onTestSelected={testSelected}
onSyncSelected={syncSelected}
selectedMetadataModel={selectedMetadataModelAvailable ? selectedMetadataModel : null}
descriptionGenerationActive={descriptionGenerationActive}
onGenerateDescriptions={generateDatabaseDescriptions}
onSuggestSensitive={suggestDatabaseSensitiveFields}
sensitiveDataSuggestionRuns={sensitiveDataSuggestionRuns}
onDeleteMetadataSelected={deleteSelectedMetadata}
onRefresh={refreshList}
/>
@@ -1125,6 +1161,45 @@ export function DatabaseManagementPage({
actions={(
<>
{fleetMetadataModelControl}
<Button
type="button"
variant="outline"
size="sm"
className="whitespace-nowrap"
aria-label="View schema synchronization logs"
title="View schema synchronization progress and history"
disabled={!canManage || (!selectedDatabaseId && !activeSyncRun?.databaseId)}
onClick={() => openSync(selectedDatabaseId ? rows.find((row) => row.id === selectedDatabaseId) : undefined)}
>
<RefreshCw aria-hidden="true" />
{currentActiveRun ? "Schema sync active" : "Schema sync logs"}
</Button>
<Button
type="button"
variant="outline"
size="sm"
className="whitespace-nowrap"
aria-label="View AI description generation logs"
title="View AI description generation progress and history"
disabled={!canManage}
onClick={openDescriptionGenerationHistory}
>
<History aria-hidden="true" />
{descriptionGenerationActive ? "AI descriptions active" : "AI description logs"}
</Button>
<Button
type="button"
variant="outline"
size="sm"
className="whitespace-nowrap"
aria-label="View sensitive analysis progress"
title="View sensitive analysis progress and history"
disabled={!canManage}
onClick={openSensitiveDataSuggestionHistory}
>
<History aria-hidden="true" />
{observedActiveSensitiveDataSuggestionRun ? "Sensitive analysis active" : "Sensitive analysis history"}
</Button>
</>
)}
/>
@@ -1144,9 +1219,17 @@ export function DatabaseManagementPage({
navigation={(
<FleetLedgerBreadcrumb
items={fleetBreadcrumb}
back={screen.kind === "relationships" || (screen.kind === "tables" && !tableNestedNavigationActive)
back={screen.kind === "relationships"
? { label: "Back to databases", onBack: showList, disabled: navigationBusy }
: undefined}
: screen.kind === "tables" && tableNestedNavigationActive
? {
label: "Back to tables",
onBack: () => tableNestedNavigationBack?.(),
disabled: navigationBusy || !tableNestedNavigationBack,
}
: screen.kind === "tables"
? { label: "Back to databases", onBack: showList, disabled: navigationBusy }
: undefined}
/>
)}
children={<FleetLedgerContent busy={isLoading || isFetching}>{fleetContent}</FleetLedgerContent>}
@@ -1246,10 +1329,12 @@ export function DatabaseManagementPage({
onEdit={editRow}
onDelete={deleteRow}
onOpenSync={openSync}
onSelectionChange={(selected) => setSelectedDatabaseId(selected.length === 1 ? selected[0].id ?? null : null)}
onTestSelected={testSelected}
onSyncSelected={syncSelected}
selectedMetadataModel={selectedMetadataModelAvailable ? selectedMetadataModel : null}
descriptionGenerationActive={descriptionGenerationActive}
sensitiveDataSuggestionRuns={sensitiveDataSuggestionRuns}
onGenerateDescriptions={generateDatabaseDescriptions}
onSuggestSensitive={suggestDatabaseSensitiveFields}
onDeleteMetadataSelected={deleteSelectedMetadata}
@@ -1304,10 +1389,9 @@ export function DatabaseManagementPage({
onNavigationStateChange={setTablesNavigationState}
onRunStarted={rememberSyncRun}
onOpenSync={() => openSync(activeRow)}
onOpenDescriptionHistory={openDescriptionGenerationHistory}
onOpenSensitiveHistory={openSensitiveDataSuggestionHistory}
onDescriptionGenerationRunStarted={rememberDescriptionGenerationRun}
onSuggestSensitive={suggestActiveDatabaseSensitiveFields}
onClearCatalogTables={clearActiveCatalogTables}
onCatalogMetricsChanged={invalidateCatalogMetrics}
/>
) : null}
@@ -285,7 +285,12 @@ export function CatalogSyncDrawer({
open
ariaLabel="Schema synchronization"
eyebrow="Schema synchronization"
title={run ? `${stateLabel(run)[0].toUpperCase()}${stateLabel(run).slice(1)}` : "Synchronization history"}
title={run ? (
<span className="inline-flex items-center gap-2">
{!terminal(run) ? <LoaderCircle className="size-5 animate-spin" aria-hidden="true" /> : null}
{stateLabel(run)[0].toUpperCase()}{stateLabel(run).slice(1)}
</span>
) : "Synchronization history"}
description={run ? `${run.scope} · ${elapsed(run, now)}` : undefined}
onClose={onClose}
closeLabel="Close synchronization drawer"
@@ -30,9 +30,7 @@ interface Props {
descriptionGenerationActive: boolean;
onDescriptionGenerationRunStarted: (run: DescriptionGenerationRun) => void;
onNavigationStateChange: (state: DatabaseNavigationState) => void;
onSync: () => void;
onSuggestSensitive: (selection: SensitiveDataSuggestionRequest, scopeLabel: string) => Promise<void>;
bindingReady?: boolean;
catalogOperationActive?: boolean;
onCatalogMetricsChanged?: () => void | Promise<void>;
presentation?: "legacy" | "fleet";
@@ -79,6 +77,7 @@ function SensitiveCell({ data, context }: ICellRendererParams<CatalogColumn, unk
checked={data.sensitive}
disabled={!context.canManage || context.busy}
aria-label={`Sensitive data for ${data.name}`}
title="Mark this column as sensitive; this does not select it for actions."
onChange={(event) => context.onSensitiveChange(data, event.target.checked)}
onClick={(event) => event.stopPropagation()}
/>
@@ -94,9 +93,7 @@ export function DatabaseColumns({
descriptionGenerationActive,
onDescriptionGenerationRunStarted,
onNavigationStateChange,
onSync,
onSuggestSensitive,
bindingReady = true,
catalogOperationActive = false,
onCatalogMetricsChanged,
presentation = "legacy",
@@ -291,7 +288,6 @@ export function DatabaseColumns({
| "generate-descriptions"
| "consolidate-descriptions"
| "suggest-sensitive"
| "sync-columns"
| "save-sensitive";
const fleetActions: readonly FleetActionOption<FleetColumnAction>[] = [
@@ -347,23 +343,6 @@ export function DatabaseColumns({
? "Another action is running."
: undefined,
},
{
id: "sync-columns",
label: "Synchronize columns",
group: "Synchronization",
scopeLabel: "All columns in this table",
runLabel: "Synchronize",
disabled: !canManage || !bindingReady || catalogOperationActive || busy,
disabledReason: !canManage
? "You do not have permission to synchronize the catalog."
: !bindingReady
? "Test the current database binding first."
: catalogOperationActive
? "Wait for the active catalog operation to finish."
: busy
? "Another action is running."
: undefined,
},
{
id: "save-sensitive",
label: "Save sensitive-field changes",
@@ -386,7 +365,6 @@ export function DatabaseColumns({
if (action === "generate-descriptions") await generateDescriptions();
else if (action === "consolidate-descriptions") await consolidateDescriptions();
else if (action === "suggest-sensitive") await suggestSensitive();
else if (action === "sync-columns") onSync();
else await saveSensitive();
};
@@ -397,7 +375,7 @@ export function DatabaseColumns({
const columns = useMemo<ColDef<CatalogColumn>[]>(() => [
{ field: "ordinalPosition", headerName: "#", width: 64, maxWidth: 64, filter: "agNumberColumnFilter" },
{ field: "sensitive", headerName: "Sensitive", minWidth: 110, width: 110, sortable: false, filter: false, resizable: false, cellRenderer: SensitiveCell },
{ field: "sensitive", headerName: "Sensitive", headerTooltip: "Mark this column as sensitive; this does not select it for actions.", minWidth: 110, width: 110, sortable: false, filter: false, resizable: false, cellRenderer: SensitiveCell },
{ field: "name", headerName: "Name", minWidth: 190, flex: 1, cellClass: "font-mono text-xs" },
{ field: "dataType", headerName: "Type", minWidth: 150, flex: 0.8, cellClass: "font-mono text-xs" },
{ headerName: "Keys", minWidth: 125, width: 125, sortable: false, filter: false, cellRenderer: KeyCell },
@@ -512,7 +490,6 @@ export function DatabaseColumns({
<span className="text-xs tabular-nums text-muted-foreground">{data.length}</span>
{changedSensitiveColumns.length > 0 ? <Button type="button" disabled={!canManage || busy} onClick={() => void saveSensitive()}><Save />{sensitiveAction === "save" ? "Saving…" : "Save sensitive fields"}</Button> : null}
<Button type="button" variant="outline" disabled={isFetching || busy} onClick={() => void refetch()}><RefreshCw className={isFetching ? "animate-spin" : ""} />Refresh</Button>
<Button type="button" disabled={!canManage || busy} onClick={onSync}><RefreshCw />Sync columns</Button>
</>
)
)}
@@ -529,6 +506,17 @@ export function DatabaseColumns({
defaultColDef={{ sortable: true, filter: true, resizable: true }}
getRowId={({ data: row }) => row.id}
rowSelection={{ mode: "multiRow", selectAll: "filtered", enableClickSelection: false }}
selectionColumnDef={{
headerName: "Select",
headerTooltip: "Select columns for actions",
width: 72,
minWidth: 72,
maxWidth: 72,
pinned: "left",
sortable: false,
resizable: false,
suppressMovable: true,
}}
onSelectionChanged={({ api }) => setSelectedIds(api.getSelectedRows().map((column) => column.id))}
rowHeight={44}
headerHeight={38}
@@ -115,7 +115,6 @@ test("Fleet columns distinguish a failed query from an empty catalog and retry i
descriptionGenerationActive={false}
onDescriptionGenerationRunStarted={noop}
onNavigationStateChange={noop}
onSync={noop}
onSuggestSensitive={noopAsync}
/>,
);
@@ -12,7 +12,6 @@ import { Button } from "../../components/ui/button";
import type {
CatalogDatabase,
CatalogSecretName,
CatalogSyncScope,
CatalogSyncRun,
DatabaseBinding,
DatabaseTransport,
@@ -23,7 +22,7 @@ import type {
DatabaseFormMode,
} from "./model";
import { statusLabel } from "./model";
import { DatabaseSyncMenu } from "./DatabaseSyncMenu";
import { DatabaseSyncMenu, type DatabaseSyncScope } from "./DatabaseSyncMenu";
const inputClass = "h-9 w-full rounded-md border border-input bg-card px-3 text-sm outline-none transition focus:border-primary/60 focus:ring-3 focus:ring-ring/15 read-only:bg-muted/40 read-only:text-muted-foreground disabled:bg-muted disabled:text-muted-foreground";
const labelClass = "grid min-w-0 gap-1.5 text-xs font-medium text-foreground";
@@ -135,7 +134,7 @@ interface DatabaseFormProps {
onRetrySecrets: () => void;
onOpenTables: (origin: HTMLButtonElement) => void;
onOpenRelationships: () => void;
onSync: (scope: CatalogSyncScope) => void;
onSync: (scope: DatabaseSyncScope) => void;
onOpenSync: () => void;
onOpenDescriptionHistory?: () => void;
onOpenSensitiveHistory?: () => void;
@@ -1,4 +1,5 @@
import { useEffect, useMemo, useRef, useState, type RefObject } from "react";
import { useQueries } from "@tanstack/react-query";
import { Menu } from "@base-ui/react/menu";
import { AgGridReact } from "ag-grid-react";
import {
@@ -11,13 +12,15 @@ import "ag-grid-community/styles/ag-grid.css";
import "ag-grid-community/styles/ag-theme-alpine.css";
import { ChevronDown, Eye, History, Info, Link2, Pencil, RefreshCw, Rows3, Sparkles, Trash2, X } from "lucide-react";
import { Button } from "../../components/ui/button";
import { getCatalogMetrics } from "../../api/catalog-databases";
import type {
CatalogDatabase,
CatalogMetrics,
SensitiveDataSuggestionRun,
CatalogDatabaseMetadataDeleteTarget,
CatalogSyncScope,
DescriptionGenerationScope,
} from "../../api/catalog-databases";
import { databaseSyncItemClass, databaseSyncScopes } from "./DatabaseSyncMenu";
import { databaseSyncItemClass, databaseSyncScopes, type DatabaseSyncScope } from "./DatabaseSyncMenu";
import { FleetActionSelector, type FleetActionOption } from "./FleetActionSelector";
import { NO_METADATA_GENERATION_LLM_MODEL_MESSAGE } from "./MetadataGenerationModelSelector";
@@ -37,10 +40,12 @@ interface DatabaseGridProps {
onEdit: (row: CatalogDatabase, origin: HTMLButtonElement) => void;
onDelete: (row: CatalogDatabase, origin: HTMLButtonElement) => void;
onOpenSync: (row: CatalogDatabase) => void;
onSelectionChange?: (rows: CatalogDatabase[]) => void;
onTestSelected: (rows: CatalogDatabase[]) => Promise<void>;
onSyncSelected: (rows: CatalogDatabase[], scope: CatalogSyncScope) => Promise<void>;
onSyncSelected: (rows: CatalogDatabase[], scope: DatabaseSyncScope) => Promise<void>;
selectedMetadataModel: string | null;
descriptionGenerationActive: boolean;
sensitiveDataSuggestionRuns?: SensitiveDataSuggestionRun[];
onGenerateDescriptions: (
rows: CatalogDatabase[],
scope: Extract<DescriptionGenerationScope, "all" | "missing">,
@@ -68,7 +73,6 @@ interface DatabaseGridContext {
type DatabaseFleetAction =
| "test"
| "sync-tables"
| "sync-columns"
| "sync-relationships"
| "sync-all"
| "generate-all"
@@ -124,87 +128,63 @@ function StateCell({
);
}
function evidenceStatus(row: CatalogDatabase): { label: string; tone: StateTone } {
if (!row.workspaceEvidence) {
return { label: "Evidence state unavailable", tone: "warning" };
}
switch (row.workspaceEvidence.state) {
case "materialized_current_revision":
return { label: "Evidence materialized", tone: "success" };
case "configuration_required":
return { label: "Evidence credentials required", tone: "warning" };
case "configured_unverified":
return { label: "Evidence configured, unverified", tone: "info" };
case "workspace_unavailable":
return { label: "Evidence unavailable", tone: "danger" };
default:
return { label: "No Evidence declared", tone: "neutral" };
}
function endpoint(host: string | undefined, port: number | undefined): string {
if (!host) return "Endpoint not configured";
return port ? `${host}:${port}` : host;
}
function RevisionEvidenceCell({ row }: { row: CatalogDatabase }) {
if (!row.workspaceRevision) {
return <StateCell label="Workspace missing" detail="Revision and Evidence unavailable" tone="danger" />;
function accessSummary(row: CatalogDatabase): { label: string; detail: string } {
const transport = row.configured
? row.binding.transport
: row.runtimeBinding?.transport ?? row.binding.transport;
if (transport === "rest_api") {
if (!row.binding.baseUrl) return { label: "REST", detail: "Endpoint not configured" };
try {
return { label: "REST", detail: new URL(row.binding.baseUrl).host };
} catch {
return { label: "REST", detail: row.binding.baseUrl };
}
}
const evidence = evidenceStatus(row);
const source = row.workspaceEvidence?.sourceType ? ` · ${row.workspaceEvidence.sourceType}` : "";
if (transport === "ssh_tunnel") {
return { label: "SSH", detail: endpoint(row.binding.sshHost, row.binding.sshPort) };
}
return {
label: "Direct PostgreSQL",
detail: endpoint(row.binding.host, row.binding.port),
};
}
function coverageStatus(total: number, complete: number): { label: string; detail: string; tone: StateTone } {
if (total === 0) return { label: "Not started", detail: "No catalog items", tone: "danger" };
if (complete >= total) return { label: "Complete", detail: `${complete}/${total}`, tone: "success" };
if (complete > 0) return { label: "Partial", detail: `${complete}/${total}`, tone: "warning" };
return { label: "Not started", detail: `0/${total}`, tone: "danger" };
}
function CatalogStatusCells({ row, metrics, sensitiveRun }: { row: CatalogDatabase; metrics?: CatalogMetrics; sensitiveRun?: SensitiveDataSuggestionRun }) {
const access = accessSummary(row);
const synchronized = row.schemaSyncedVersion === row.version;
const syncStatus = !row.schemaSyncedVersion
? { label: "Not synchronized", detail: "Schema not synchronized", tone: "danger" as const }
: synchronized
? { label: "Synchronized", detail: "Current schema", tone: "success" as const }
: { label: "Partial", detail: "Schema revision changed", tone: "warning" as const };
const descriptions = coverageStatus(metrics?.descriptionTargets ?? 0, metrics?.describedTargets ?? 0);
const sensitiveProcessed = sensitiveRun ? sensitiveRun.suggestedSensitive + sensitiveRun.suggestedNonSensitive : 0;
const sensitive = !sensitiveRun
? { label: "Not started", detail: "No analysis run", tone: "danger" as const }
: sensitiveRun.status === "completed" && sensitiveProcessed >= sensitiveRun.total
? { label: "Complete", detail: `${sensitiveProcessed}/${sensitiveRun.total}`, tone: "success" as const }
: ["failed", "interrupted"].includes(sensitiveRun.status)
? { label: "Incomplete", detail: `${sensitiveProcessed}/${sensitiveRun.total}`, tone: "danger" as const }
: { label: "Partial", detail: `${sensitiveProcessed}/${sensitiveRun.total}`, tone: "warning" as const };
return (
<StateCell
label={`Active revision ${row.workspaceRevision.commit.slice(0, 7)}`}
detail={`${evidence.label}${source}`}
tone={evidence.tone}
/>
);
}
function transportLabel(row: CatalogDatabase): string {
const transport = row.runtimeBinding?.transport;
if (transport === "postgres_direct") return "Direct PostgreSQL";
if (transport === "rest_api") return "REST API";
if (transport === "ssh_tunnel") return "SSH tunnel";
return "No runtime binding";
}
function RuntimeBindingCell({ row }: { row: CatalogDatabase }) {
if (!row.runtimeBinding) {
return <StateCell label="Unavailable" detail="Workspace is not in the active revision" tone="danger" />;
}
if (!row.runtimeBinding.sessionTransportSupported) {
return <StateCell label="Unsupported for NL→SQL" detail={transportLabel(row)} tone="danger" />;
}
if (row.runtimeBinding.configurationState === "configuration_required") {
return <StateCell label="Configuration required" detail={transportLabel(row)} tone="warning" />;
}
return <StateCell label="Ready" detail={transportLabel(row)} tone="success" />;
}
function catalogConnectionDetail(row: CatalogDatabase): { label: string; className?: string } {
if (row.activeSyncRun) return { label: "Metadata sync in progress", className: "text-primary" };
if (row.testedVersion !== undefined && row.testedVersion !== row.version) {
return { label: "Connection retest required", className: "text-amber-700 dark:text-amber-300" };
}
if (row.connectionStatus === "reachable") return { label: "Connection reachable" };
if (row.connectionStatus === "failed") {
return { label: "Connection test failed", className: "text-destructive" };
}
return { label: "Connection not tested" };
}
function MetadataCatalogCell({ row }: { row: CatalogDatabase }) {
if (!row.workspaceAvailable) {
return <StateCell label="Orphaned configuration" detail="Workspace missing from repository" tone="danger" />;
}
if (!row.configured) {
return <StateCell label="Not configured" detail="Configure this workspace" tone="neutral" />;
}
const connection = catalogConnectionDetail(row);
return (
<StateCell
label="Configured"
detail={connection.label}
detailClassName={connection.className}
tone="success"
/>
<div className="grid h-full min-w-[540px] grid-cols-4 gap-3 py-1">
<StateCell label={access.label} detail={access.detail} tone="info" />
<StateCell label={syncStatus.label} detail={syncStatus.detail} tone={syncStatus.tone} />
<StateCell label={descriptions.label} detail={`Descriptions · ${descriptions.detail}`} tone={descriptions.tone} />
<StateCell label={sensitive.label} detail={`Sensitive · ${sensitive.detail}`} tone={sensitive.tone} />
</div>
);
}
@@ -259,11 +239,6 @@ function DatabaseActionsCell({
return (
<div className="flex h-full items-center justify-end gap-0.5" onClick={(event) => event.stopPropagation()}>
{data.activeSyncRun || (context.presentation === "fleet" && data.configured && data.id) ? (
<Button type="button" variant="ghost" size="icon-lg" className={context.presentation === "fleet" ? "thot-fleet-icon-action" : undefined} data-tooltip={context.presentation === "fleet" ? "Synchronization history" : undefined} title={context.presentation === "legacy" ? `View synchronization for ${data.workspaceName}` : undefined} aria-label={`View synchronization history for ${data.workspaceName}`} onClick={() => context.onOpenSync(data)}>
{data.activeSyncRun ? <RefreshCw className="animate-spin" aria-hidden="true" /> : <History aria-hidden="true" />}
</Button>
) : null}
{context.presentation === "fleet" && context.onOpenTables ? (
<span className="thot-fleet-icon-action inline-flex" data-tooltip="Tables">
<Button
@@ -357,16 +332,37 @@ export function DatabaseGrid({
onEdit,
onDelete,
onOpenSync,
onSelectionChange,
onTestSelected,
onSyncSelected,
selectedMetadataModel,
descriptionGenerationActive,
sensitiveDataSuggestionRuns = [],
onGenerateDescriptions,
onSuggestSensitive,
onDeleteMetadataSelected,
onRefresh,
presentation = "legacy",
}: DatabaseGridProps) {
const metricQueries = useQueries({
queries: rows.filter((row) => row.id).map((row) => ({
queryKey: ["catalog-metrics", row.id],
queryFn: () => getCatalogMetrics(row.id!),
enabled: canManage,
staleTime: 10_000,
})),
});
const metricsByDatabase = useMemo(() => {
const result: Record<string, CatalogMetrics> = {};
rows.filter((row) => row.id).forEach((row, index) => {
const metrics = metricQueries[index]?.data;
if (metrics && row.id) result[row.id] = metrics;
});
return result;
}, [metricQueries, rows]);
const sensitiveRunsByDatabase = useMemo(() => new Map(
rows.filter((row) => row.id).map((row) => [row.id!, sensitiveDataSuggestionRuns.find((run) => run.databaseId === row.id)]),
), [rows, sensitiveDataSuggestionRuns]);
const compact = useCompactViewport();
const gridRef = useRef<AgGridReact<CatalogDatabase>>(null);
const actionsTriggerRef = useRef<HTMLButtonElement>(null);
@@ -390,68 +386,16 @@ export function DatabaseGrid({
flex: compact ? undefined : 1.35,
width: compact ? 112 : undefined,
};
const revisionEvidence: ColDef<CatalogDatabase> = {
headerName: "Revision / Evidence",
minWidth: 215,
flex: 1.2,
valueGetter: ({ data }) => {
if (!data) return "";
const revision = data.workspaceRevision?.commit.slice(0, 7) ?? "workspace missing";
return `${revision} ${evidenceStatus(data).label}`;
const status: ColDef<CatalogDatabase> = {
headerName: "Catalog status",
minWidth: 580,
flex: 2.5,
cellRenderer: ({ data }: ICellRendererParams<CatalogDatabase>) => {
const databaseId = data?.id;
return data
? <CatalogStatusCells row={data} metrics={databaseId ? metricsByDatabase[databaseId] : undefined} sensitiveRun={databaseId ? sensitiveRunsByDatabase.get(databaseId) : undefined} />
: null;
},
cellRenderer: ({ data }: ICellRendererParams<CatalogDatabase>) => (
data ? <RevisionEvidenceCell row={data} /> : null
),
};
const runtimeBinding: ColDef<CatalogDatabase> = {
headerName: "NL→SQL runtime",
minWidth: 175,
flex: 1,
valueGetter: ({ data }) => {
if (!data?.runtimeBinding) return "Unavailable";
if (!data.runtimeBinding.sessionTransportSupported) return "Unsupported for NL→SQL";
return data.runtimeBinding.configurationState === "ready" ? "Ready" : "Configuration required";
},
cellRenderer: ({ data }: ICellRendererParams<CatalogDatabase>) => (
data ? <RuntimeBindingCell row={data} /> : null
),
};
const database: ColDef<CatalogDatabase> = {
field: "databaseName", headerName: "Database", minWidth: 145, flex: 1,
};
const schema: ColDef<CatalogDatabase> = {
field: "schema", headerName: "Schema", minWidth: 140, flex: 0.9,
};
const endpoint: ColDef<CatalogDatabase> = {
headerName: "Endpoint",
minWidth: 190,
flex: 1.2,
valueGetter: ({ data }) => {
if (!data) return "";
if (data.binding.transport === "rest_api") return data.binding.baseUrl ?? "";
if (data.binding.transport === "ssh_tunnel") return data.binding.sshHost ?? "";
return data.binding.host ? `${data.binding.host}:${data.binding.port ?? 5432}` : "";
},
};
const metadataCatalog: ColDef<CatalogDatabase> = {
headerName: "Metadata Catalog",
minWidth: 185,
flex: 1.05,
valueGetter: ({ data }) => {
if (!data) return "";
if (!data.workspaceAvailable) return "Orphaned configuration";
return data.configured ? `Configured ${catalogConnectionDetail(data).label}` : "Not configured";
},
cellRenderer: ({ data }: ICellRendererParams<CatalogDatabase>) => (
data ? <MetadataCatalogCell row={data} /> : null
),
};
const updated: ColDef<CatalogDatabase> = {
field: "updatedAt",
headerName: "Updated",
minWidth: 170,
flex: 0.9,
valueFormatter: ({ value }) => value ? new Date(String(value)).toLocaleString() : "",
};
const actionsWidth = presentation === "fleet" ? 232 : 210;
const actions: ColDef<CatalogDatabase> = {
@@ -471,10 +415,8 @@ export function DatabaseGrid({
cellRenderer: DatabaseActionsCell,
};
return compact
? [workspace, revisionEvidence, runtimeBinding, metadataCatalog, database, schema, endpoint, updated, actions]
: [workspace, revisionEvidence, runtimeBinding, database, schema, endpoint, metadataCatalog, updated, actions];
}, [compact, presentation]);
return [workspace, status, actions];
}, [compact, metricsByDatabase, presentation, sensitiveRunsByDatabase]);
const configuredCount = rows.filter((row) => row.configured).length;
const hiddenSelected = selectedRows.filter((row) => {
@@ -533,7 +475,6 @@ export function DatabaseGrid({
return [
{ id: "test", label: "Test connections", group: "Connection", disabled: !canTestSelection, disabledReason: testReason },
{ id: "sync-tables", label: "Synchronize tables", group: "Synchronization", disabled: !canSyncSelection, disabledReason: syncReason },
{ id: "sync-columns", label: "Synchronize all columns", group: "Synchronization", disabled: !canSyncSelection, disabledReason: syncReason },
{ id: "sync-relationships", label: "Synchronize relationships", group: "Synchronization", disabled: !canSyncSelection, disabledReason: syncReason },
{ id: "sync-all", label: "Synchronize all", group: "Synchronization", disabled: !canSyncSelection, disabledReason: syncReason },
{ id: "generate-missing", label: "Generate missing descriptions", group: "Descriptions", disabled: !canGenerateDescriptions, disabledReason: generationReason },
@@ -580,15 +521,10 @@ export function DatabaseGrid({
const perform = async (
kind: "test" | "sync" | "generate" | "suggest" | "delete",
operation: () => Promise<void>,
clearSelection = true,
) => {
setAction(kind);
try {
await operation();
if (clearSelection) {
gridRef.current?.api.deselectAll();
setSelectedRows([]);
}
setPendingDelete(null);
setPendingGenerationScope(null);
if (kind === "delete") window.setTimeout(() => searchInputRef.current?.focus(), 0);
@@ -612,11 +548,11 @@ export function DatabaseGrid({
return;
}
if (selectedAction.startsWith("sync-")) {
const scope = selectedAction.slice("sync-".length) as CatalogSyncScope;
const scope = selectedAction.slice("sync-".length) as DatabaseSyncScope;
await perform("sync", () => onSyncSelected(selectedRows, scope));
return;
}
await perform("suggest", () => onSuggestSensitive(selectedRows), false);
await perform("suggest", () => onSuggestSensitive(selectedRows));
};
return (
@@ -785,7 +721,7 @@ export function DatabaseGrid({
variant="outline"
disabled={!canManage || action !== null || descriptionGenerationActive}
title={descriptionGenerationActive ? "Wait for the active description generation to finish" : undefined}
onClick={() => void perform("suggest", () => onSuggestSensitive(selectedRows), false)}
onClick={() => void perform("suggest", () => onSuggestSensitive(selectedRows))}
>
<Sparkles />{action === "suggest" ? "Suggesting…" : "Suggest sensitive fields"}
</Button>
@@ -833,7 +769,9 @@ export function DatabaseGrid({
onSelectionChanged={({ api }) => {
if (pendingDelete && action === null) setPendingDelete(null);
if (pendingGenerationScope && action === null) setPendingGenerationScope(null);
setSelectedRows(api.getSelectedRows());
const rows = api.getSelectedRows();
setSelectedRows(rows);
onSelectionChange?.(rows);
}}
rowHeight={52}
headerHeight={38}
@@ -3,14 +3,15 @@ import { ChevronDown, RefreshCw } from "lucide-react";
import { buttonVariants } from "../../components/ui/button";
import type { CatalogSyncScope } from "../../api/catalog-databases";
export type DatabaseSyncScope = Exclude<CatalogSyncScope, "columns">;
export const databaseSyncItemClass = [
"flex cursor-default select-none items-center rounded-md px-3 py-2 text-sm outline-none",
"data-[highlighted]:bg-muted data-[disabled]:opacity-45",
].join(" ");
export const databaseSyncScopes: ReadonlyArray<{ scope: CatalogSyncScope; label: string }> = [
export const databaseSyncScopes: ReadonlyArray<{ scope: DatabaseSyncScope; label: string }> = [
{ scope: "tables", label: "Synchronize tables" },
{ scope: "columns", label: "Synchronize all columns" },
{ scope: "relationships", label: "Synchronize relationships" },
{ scope: "all", label: "Synchronize all" },
];
@@ -18,7 +19,7 @@ export const databaseSyncScopes: ReadonlyArray<{ scope: CatalogSyncScope; label:
interface DatabaseSyncMenuProps {
disabled: boolean;
disabledReason?: string;
onSelect: (scope: CatalogSyncScope) => void;
onSelect: (scope: DatabaseSyncScope) => void;
}
export function DatabaseSyncMenu({ disabled, disabledReason, onSelect }: DatabaseSyncMenuProps) {
@@ -1,15 +1,16 @@
import { useEffect, useMemo, useRef, useState } from "react";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { Menu } from "@base-ui/react/menu";
import { useQuery, useQueryClient } from "@tanstack/react-query";
import { AgGridReact } from "ag-grid-react";
import type { ColDef, ICellRendererParams } from "ag-grid-community";
import { ArrowLeft, ChevronDown, Columns3, History, Pencil, RefreshCw, Save, Sparkles, Trash2, X } from "lucide-react";
import { ArrowLeft, ChevronDown, Columns3, Pencil, RefreshCw, Save, Sparkles, Trash2, X } from "lucide-react";
import { toast } from "sonner";
import { Button } from "../../components/ui/button";
import { ApiError, apiErrorMessage } from "../../api/client";
import {
consolidateCatalogDescriptions,
deleteCatalogTableMetadata,
listCatalogColumns,
listCatalogTables,
startCatalogSync,
startDescriptionGenerationRun,
@@ -37,11 +38,10 @@ interface Props {
onOpenOverview: () => void;
onOpenRelationships: () => void;
onNavigationStateChange: (state: DatabaseNavigationState) => void;
onNestedNavigationChange?: (active: boolean) => void;
onNestedNavigationChange?: (active: boolean, onBack?: () => void) => void;
onRunStarted: (run: CatalogSyncRun) => void;
onOpenSync: () => void;
onOpenDescriptionHistory?: () => void;
onOpenSensitiveHistory?: () => void;
onClearCatalogTables?: () => Promise<void>;
onDescriptionGenerationRunStarted: (run: DescriptionGenerationRun) => void;
onSuggestSensitive: (selection: SensitiveDataSuggestionRequest, scopeLabel: string) => Promise<void>;
onCatalogMetricsChanged?: () => void | Promise<void>;
@@ -95,8 +95,7 @@ export function DatabaseTables({
onNestedNavigationChange,
onRunStarted,
onOpenSync,
onOpenDescriptionHistory,
onOpenSensitiveHistory,
onClearCatalogTables,
onDescriptionGenerationRunStarted,
onSuggestSensitive,
onCatalogMetricsChanged,
@@ -134,9 +133,22 @@ export function DatabaseTables({
const nestedNavigationActive = presentation === "fleet" && Boolean(activeTable) && tableSection === "columns";
const bindingReady = database.connectionStatus === "reachable" && database.testedVersion === database.version;
const currentRun = activeRun ?? database.activeSyncRun;
const clearAllCatalogTables = async () => {
if (!onClearCatalogTables || !window.confirm("Clear all catalog tables, columns, and relationships for this database? The source database will not be changed.")) return;
await onClearCatalogTables();
};
const leaveTable = useCallback(() => {
if (navigationBusy) return;
if (dirty && !window.confirm("Discard unsaved metadata?")) return;
setActiveTableId(null);
setTableSection("overview");
window.setTimeout(() => originRef.current?.focus(), 0);
}, [dirty, navigationBusy]);
useEffect(() => { onNavigationStateChange({ dirty, busy: navigationBusy }); }, [dirty, navigationBusy, onNavigationStateChange]);
useEffect(() => { onNestedNavigationChange?.(nestedNavigationActive); }, [nestedNavigationActive, onNestedNavigationChange]);
useEffect(() => {
onNestedNavigationChange?.(nestedNavigationActive, nestedNavigationActive ? leaveTable : undefined);
}, [leaveTable, nestedNavigationActive, onNestedNavigationChange]);
useEffect(() => () => { onNestedNavigationChange?.(false); }, [onNestedNavigationChange]);
useEffect(() => {
if (!activeTableId || !activeTable || editorVersion === activeTable.version || busy) return;
@@ -168,13 +180,6 @@ export function DatabaseTables({
setStaleBannerOpen(true);
setColumnNavigation({ dirty: false, busy: false });
};
const leaveTable = () => {
if (navigationBusy) return;
if (dirty && !window.confirm("Discard unsaved metadata?")) return;
setActiveTableId(null);
setTableSection("overview");
window.setTimeout(() => originRef.current?.focus(), 0);
};
const navigateDatabase = (target: "databases" | "overview" | "relationships") => {
if (navigationBusy) return;
if (dirty && !window.confirm("Discard unsaved metadata?")) return;
@@ -281,6 +286,28 @@ export function DatabaseTables({
toast.error(apiErrorMessage(error));
} finally { setBusy(null); }
};
const generateColumnDescriptions = async () => {
if (selectedIds.length === 0 || !selectedMetadataModel) return;
setBusy("generate");
try {
const columns = (await Promise.all(selectedIds.map((tableId) => listCatalogColumns(databaseId, tableId)))).flat();
const columnIds = columns.map((column) => column.id);
if (columnIds.length === 0) {
toast.error("The selected tables have no synchronized catalog columns.");
return;
}
const run = await startDescriptionGenerationRun(
databaseId,
selectedMetadataModel,
"selected_columns",
columnIds,
);
onDescriptionGenerationRunStarted(run);
toast.success(`Column description generation started for ${columnIds.length} column${columnIds.length === 1 ? "" : "s"}`);
} catch (error) {
toast.error(apiErrorMessage(error));
} finally { setBusy(null); }
};
const suggestSensitive = async () => {
if (selectedIds.length === 0) return;
setBusy("suggest");
@@ -298,6 +325,7 @@ export function DatabaseTables({
type FleetTableAction =
| "generate-descriptions"
| "generate-column-descriptions"
| "sync-tables"
| "sync-columns"
| "consolidate-descriptions"
@@ -324,6 +352,24 @@ export function DatabaseTables({
? "Another action is running."
: undefined,
},
{
id: "generate-column-descriptions",
label: "Generate column descriptions",
group: "Descriptions",
runLabel: "Generate",
disabled: !canManage || selectedIds.length === 0 || !selectedMetadataModel || descriptionGenerationActive || busy !== null || Boolean(currentRun),
disabledReason: !canManage
? "You do not have permission to generate descriptions."
: selectedIds.length === 0
? "Select at least one table."
: !selectedMetadataModel
? NO_METADATA_GENERATION_LLM_MODEL_MESSAGE
: descriptionGenerationActive || Boolean(currentRun)
? "Wait for the active catalog operation to finish."
: busy !== null
? "Another action is running."
: undefined,
},
{
id: "consolidate-descriptions",
label: "Move generated to Description",
@@ -421,6 +467,7 @@ export function DatabaseTables({
const runFleetAction = async (action: FleetTableAction) => {
if (action === "generate-descriptions") await generateDescriptions();
else if (action === "generate-column-descriptions") await generateColumnDescriptions();
else if (action === "sync-tables") await synchronize("tables");
else if (action === "consolidate-descriptions") await consolidateDescriptions();
else if (action === "sync-columns") await synchronize("columns", selectedIds);
@@ -464,7 +511,7 @@ export function DatabaseTables({
const databaseHeader = (
<div className="px-4 pt-5 sm:px-5">
<Button type="button" variant="ghost" className="-ml-2 mb-4" disabled={navigationBusy} onClick={() => navigateDatabase("databases")}><ArrowLeft />Back to databases</Button>
<div className="flex flex-wrap items-start justify-between gap-4 border-b border-border pb-5">
<div className="flex flex-wrap items-start justify-between gap-4 border-b border-border pb-5">
<div>
<p className="thot-label mb-1">Database management</p>
<h2 ref={headingRef} tabIndex={-1} className="font-heading text-2xl font-semibold tracking-tight outline-none">{database.workspaceName}</h2>
@@ -472,6 +519,7 @@ export function DatabaseTables({
</div>
<div className="flex gap-2">
{currentRun ? <Button type="button" variant="outline" onClick={onOpenSync}><RefreshCw className="animate-spin" />View sync</Button> : null}
{onClearCatalogTables ? <Button type="button" variant="outline" className="text-destructive" disabled={!canManage || busy !== null || Boolean(currentRun)} onClick={() => void clearAllCatalogTables()}>Clear catalog tables</Button> : null}
<Button type="button" disabled={!canManage || !bindingReady || busy !== null || Boolean(currentRun)} onClick={() => void synchronize("all")}><RefreshCw />Sync all</Button>
</div>
</div>
@@ -488,18 +536,6 @@ export function DatabaseTables({
return (
<section aria-label={`Columns for ${activeTable.name}`} className="flex min-h-0 flex-1 flex-col overflow-hidden bg-background">
<div className="flex min-h-12 flex-wrap items-center gap-3 border-b border-border bg-muted/25 px-3 py-2">
<Button
type="button"
variant="ghost"
size="sm"
className="thot-fleet-icon-action thot-fleet-back-action"
disabled={navigationBusy}
data-tooltip="Tables"
aria-label="Back to tables"
onClick={leaveTable}
>
<ArrowLeft aria-hidden="true" /> Back to tables
</Button>
<div className="min-w-0">
<p className="thot-label">Physical table</p>
<h3 className="truncate font-heading text-lg font-semibold">{activeTable.name}</h3>
@@ -513,9 +549,7 @@ export function DatabaseTables({
descriptionGenerationActive={descriptionGenerationActive}
onDescriptionGenerationRunStarted={onDescriptionGenerationRunStarted}
onNavigationStateChange={setColumnNavigation}
onSync={() => void synchronize("columns", [activeTable.id])}
onSuggestSensitive={onSuggestSensitive}
bindingReady={bindingReady}
catalogOperationActive={Boolean(currentRun)}
onCatalogMetricsChanged={onCatalogMetricsChanged}
presentation="fleet"
@@ -626,7 +660,6 @@ export function DatabaseTables({
descriptionGenerationActive={descriptionGenerationActive}
onDescriptionGenerationRunStarted={onDescriptionGenerationRunStarted}
onNavigationStateChange={setColumnNavigation}
onSync={() => void synchronize("columns", [activeTable.id])}
onSuggestSensitive={onSuggestSensitive}
onCatalogMetricsChanged={onCatalogMetricsChanged}
/>
@@ -728,6 +761,7 @@ export function DatabaseTables({
<Menu.Positioner side="bottom" align="start" sideOffset={4}>
<Menu.Popup className="z-50 min-w-64 rounded-lg bg-popover p-1 text-popover-foreground shadow-md ring-1 ring-foreground/10 outline-none">
<Menu.Item className="rounded-md px-3 py-2 text-sm outline-none data-[highlighted]:bg-muted data-[disabled]:opacity-45" disabled={!canManage || !selectedMetadataModel || descriptionGenerationActive || busy !== null || Boolean(currentRun)} onClick={() => void generateDescriptions()}>Generate {selectedIds.length === 1 ? "description" : "descriptions"}</Menu.Item>
<Menu.Item className="rounded-md px-3 py-2 text-sm outline-none data-[highlighted]:bg-muted data-[disabled]:opacity-45" disabled={!canManage || !selectedMetadataModel || descriptionGenerationActive || busy !== null || Boolean(currentRun)} onClick={() => void generateColumnDescriptions()}>Generate descriptions for all columns</Menu.Item>
<Menu.Item className="rounded-md px-3 py-2 text-sm outline-none data-[highlighted]:bg-muted data-[disabled]:opacity-45" disabled={!canManage || !bindingReady || busy !== null || Boolean(currentRun)} onClick={() => void synchronize("columns", selectedIds)}>Synchronize columns</Menu.Item>
<Menu.Item className="rounded-md px-3 py-2 text-sm outline-none data-[highlighted]:bg-muted data-[disabled]:opacity-45" disabled={!canManage || busy !== null || Boolean(currentRun)} onClick={() => void consolidateDescriptions()}>Move generated description to Description</Menu.Item>
<Menu.Separator className="my-1 h-px bg-border" />
@@ -769,13 +803,6 @@ export function DatabaseTables({
</>
)
)}
{presentation === "fleet" ? (
<Button type="button" size="sm" variant="outline" className="thot-fleet-nav-action thot-fleet-nav-action--below" data-tooltip="Synchronization history" onClick={onOpenSync}>
{currentRun ? <RefreshCw className="animate-spin" aria-hidden="true" /> : <History aria-hidden="true" />} Sync history
</Button>
) : null}
{onOpenDescriptionHistory ? <Button type="button" size="sm" variant="outline" onClick={onOpenDescriptionHistory}><History aria-hidden="true" /> Description history</Button> : null}
{onOpenSensitiveHistory ? <Button type="button" size="sm" variant="outline" onClick={onOpenSensitiveHistory}><History aria-hidden="true" /> Sensitive history</Button> : null}
</div>
{!bindingReady ? <div className="border-b border-border bg-muted/35 px-4 py-3 text-sm text-muted-foreground">Test the current database binding from Overview before synchronizing tables.</div> : null}
{fleetQueryError ?? <div className="relative min-h-[280px] flex-1">
@@ -109,6 +109,9 @@ test("replays ordered SSE events, reconnects from the latest sequence, and keeps
renderDrawer();
const log = await screen.findByRole("log", { name: "Description generation events" });
const progress = screen.getByRole("table", { name: "Description generation progress" });
expect(within(progress).getAllByRole("columnheader")).toHaveLength(5);
expect(log).toHaveClass("thot-catalog-drawer__event-log");
expect(await within(log).findByText("Run queued")).toBeVisible();
await waitFor(() => expect(FakeEventSource.instances).toHaveLength(1));
expect(FakeEventSource.instances[0].url).toBe(
@@ -300,6 +300,16 @@ export function DescriptionGenerationDrawer({
}
const apparentlyStale = isApparentlyStale(run);
const finalError = safeFinalError(run.errorSummary);
const modelDisplay = modelLabel && modelLabel !== run.modelId
? `${modelLabel} (${run.modelId})`
: run.modelId;
const progressCounters = [
["total", "Total"],
["processed", "Processed"],
["generated", "Generated"],
["nonGeneratable", "Not generated"],
["failed", "Failed"],
] as const;
const footer = ["queued", "running"].includes(run.status) ? (
<div className="w-full">
{apparentlyStale ? (
@@ -341,55 +351,80 @@ export function DescriptionGenerationDrawer({
open
ariaLabel="Description generation"
eyebrow="Description generation"
title={statusLabel(run)}
description={`${modelLabel}${modelLabel !== run.modelId ? ` (${run.modelId})` : ""} · ${run.scope.replaceAll("_", " ")}`}
title={(
<span className="inline-flex items-center gap-2">
{!isTerminal(run) ? <LoaderCircle className="size-5 animate-spin" aria-hidden="true" /> : null}
{statusLabel(run)}
</span>
)}
description={run.scope.replaceAll("_", " ")}
onClose={onClose}
closeLabel="Close description generation"
busy={action !== null}
bodyClassName="thot-catalog-drawer__history-layout"
bodyClassName="thot-catalog-drawer__history-layout thot-catalog-drawer__run-layout"
footer={footer}
>
{runQuery.isError ? (
<div className="thot-catalog-drawer__run-main">
{runQuery.isError ? (
<div role="alert" className="rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive">
{apiErrorMessage(runQuery.error)}
</div>
) : null}
<section aria-label="Description generation counters">
<div className={`grid gap-4 sm:grid-cols-2${runQuery.isError ? " mt-4" : ""}`}>
<section aria-label="AI usage">
<h3 className="thot-label mb-2">AI usage</h3>
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1 text-xs">
<dt>Model</dt><dd className="truncate text-right" title={modelDisplay}>{modelDisplay}</dd>
<dt>Input tokens</dt><dd className="text-right tabular-nums">{(run.inputTokens ?? 0).toLocaleString("en-US")}</dd>
<dt>Cache tokens</dt><dd className="text-right tabular-nums">{(run.cacheReadTokens ?? 0).toLocaleString("en-US")}</dd>
<dt>Output tokens</dt><dd className="text-right tabular-nums">{(run.outputTokens ?? 0).toLocaleString("en-US")}</dd>
</dl>
</section>
<section aria-label="Description generation timestamps">
<h3 className="thot-label mb-2">Timestamps</h3>
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1 text-xs">
<dt className="text-muted-foreground">Created</dt><dd className="text-right tabular-nums">{timestamp(run.createdAt)}</dd>
<dt className="text-muted-foreground">Started</dt><dd className="text-right tabular-nums">{timestamp(run.startedAt)}</dd>
<dt className="text-muted-foreground">Updated</dt><dd className="text-right tabular-nums">{timestamp(run.updatedAt)}</dd>
<dt className="text-muted-foreground">Finished</dt><dd className="text-right tabular-nums">{timestamp(run.finishedAt)}</dd>
</dl>
</section>
</div>
<section className="mt-4" aria-label="Description generation counters">
<h3 className="thot-label mb-2">Progress</h3>
<div className="grid grid-cols-2 gap-2">
{([
["total", "Total"],
["processed", "Processed"],
["generated", "Generated"],
["nonGeneratable", "Not generated"],
["failed", "Failed"],
] as const).map(([name, label]) => (
<div key={name} className="rounded-md border border-border bg-muted/25 px-3 py-2">
<p className="text-[11px] font-semibold uppercase tracking-wide text-muted-foreground">{label}</p>
<p data-count={name} className="mt-1 text-lg font-semibold tabular-nums">{run[name]}</p>
</div>
))}
<div className="overflow-x-auto rounded-md border border-border bg-muted/20">
<table className="w-full min-w-[28rem] table-fixed" aria-label="Description generation progress">
<thead>
<tr className="border-b border-border bg-muted/45">
{progressCounters.map(([name, label]) => (
<th key={name} scope="col" className="border-l border-border px-2 py-1.5 text-left text-[10px] font-semibold uppercase tracking-wide text-muted-foreground first:border-l-0">
{label}
</th>
))}
</tr>
</thead>
<tbody>
<tr>
{progressCounters.map(([name]) => (
<td key={name} data-count={name} className="border-l border-border px-2 py-1.5 text-base font-semibold tabular-nums first:border-l-0">
{run[name]}
</td>
))}
</tr>
</tbody>
</table>
</div>
</section>
<section className="mt-5" aria-label="Description generation timestamps">
<h3 className="thot-label mb-2">Timestamps</h3>
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1.5 text-sm">
<dt className="text-muted-foreground">Created</dt><dd className="text-right">{timestamp(run.createdAt)}</dd>
<dt className="text-muted-foreground">Started</dt><dd className="text-right">{timestamp(run.startedAt)}</dd>
<dt className="text-muted-foreground">Updated</dt><dd className="text-right">{timestamp(run.updatedAt)}</dd>
<dt className="text-muted-foreground">Finished</dt><dd className="text-right">{timestamp(run.finishedAt)}</dd>
</dl>
</section>
{finalError ? (
<div className="mt-5 rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive">
<div className="mt-3 rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive">
{finalError}
</div>
) : null}
<section className="mt-5" aria-label="Description generation log">
<section className="thot-catalog-drawer__events mt-3" aria-label="Description generation log">
<div className="mb-2 flex items-center justify-between">
<h3 className="thot-label">Events</h3>
<span className="text-xs tabular-nums text-muted-foreground">{events.length}</span>
@@ -397,7 +432,7 @@ export function DescriptionGenerationDrawer({
<div
role="log"
aria-label="Description generation events"
className="max-h-72 overflow-y-auto rounded-md bg-zinc-950 p-3 font-mono text-xs leading-5 text-zinc-200"
className="thot-catalog-drawer__event-log overflow-y-auto rounded-md bg-zinc-950 p-3 font-mono text-xs leading-5 text-zinc-200"
>
{events.length === 0 ? <p className="text-zinc-500">Waiting for events…</p> : events.map((event) => (
<p key={event.sequence} className={event.level === "error" ? "text-red-300" : event.level === "warning" ? "text-amber-300" : undefined}>
@@ -407,8 +442,9 @@ export function DescriptionGenerationDrawer({
</div>
{eventQuery.isError ? <p role="alert" className="mt-2 text-sm text-destructive">{apiErrorMessage(eventQuery.error)}</p> : null}
</section>
</div>
<section className="thot-catalog-drawer__history mt-5" aria-label="Description generation history">
<section className="thot-catalog-drawer__history" aria-label="Description generation history">
<div className="mb-2 flex items-center justify-between gap-3">
<h3 className="thot-label">Recent runs</h3>
<span className="text-xs tabular-nums text-muted-foreground">
@@ -424,7 +460,7 @@ export function DescriptionGenerationDrawer({
) : visibleHistory.length === 0 ? (
<p className="text-sm text-muted-foreground">No description generation runs yet.</p>
) : (
<div className="max-h-56 divide-y divide-border overflow-y-auto rounded-md border border-border">
<div className="thot-catalog-drawer__history-list divide-y divide-border overflow-y-auto rounded-md border border-border">
{visibleHistory.map((item) => (
<button
key={item.id}
@@ -15,6 +15,19 @@ const actions = [
},
] as const;
test("disables the picker when no rows are selected", () => {
render(
<FleetActionSelector
actions={actions}
selectedCount={0}
onRun={vi.fn()}
onClear={vi.fn()}
/>,
);
expect(screen.getByRole("combobox", { name: "Batch action" })).toBeDisabled();
});
test("keeps unavailable commands discoverable and explains why Run is disabled", async () => {
const user = userEvent.setup();
render(
@@ -50,6 +63,7 @@ test("runs an enabled command explicitly and resets the picker", async () => {
await user.selectOptions(picker, "sync");
expect(screen.getByText("All tables in this database")).toBeVisible();
expect(screen.getByText("Action scope")).toBeVisible();
expect(screen.getByRole("button", { name: "Run action" })).toHaveClass("bg-destructive");
await user.click(screen.getByRole("button", { name: "Run action" }));
expect(onRun).toHaveBeenCalledWith("sync");
@@ -44,7 +44,6 @@ export function FleetActionSelector<ActionId extends string>({
selectedCount,
hiddenSelectedCount = 0,
busy = false,
busyLabel = "Running…",
selectRef,
onRun,
onClear,
@@ -102,7 +101,7 @@ export function FleetActionSelector<ActionId extends string>({
className="thot-fleet-action-selector__select"
aria-label={label}
value={selectedActionId}
disabled={busy}
disabled={busy || selectedCount === 0}
onChange={(event) => setSelectedActionId(event.target.value as ActionId | "")}
>
<option value="">Choose an action…</option>
@@ -123,13 +122,14 @@ export function FleetActionSelector<ActionId extends string>({
<Button
type="button"
size="sm"
variant={selectedAction?.tone === "destructive" ? "destructive" : "default"}
variant="destructive"
className="bg-destructive text-destructive-foreground hover:bg-destructive/90"
disabled={!selectedAction || selectedAction.disabled || busy}
aria-describedby={unavailableReason ? `${hintId}-reason` : undefined}
onClick={() => void run()}
>
<Play aria-hidden="true" />
{busy ? busyLabel : selectedAction?.runLabel ?? "Run action"}
Run action
</Button>
<Button type="button" size="sm" variant="ghost" disabled={busy} onClick={onClear}>
<X aria-hidden="true" /> Clear
@@ -402,6 +402,7 @@
}
.thot-fleet-action-selector__select {
flex: 1 1 13rem;
width: min(100%, 22rem);
min-width: 13rem;
height: 2rem;
@@ -459,11 +460,11 @@
.thot-fleet-ledger__drawer {
position: absolute;
z-index: 50;
inset-block: 0.75rem;
inset-block: clamp(0.625rem, 2vh, 1rem);
inset-inline-start: 50%;
inset-inline-end: auto;
display: flex;
width: 60%;
width: min(75rem, calc(100% - 2rem));
min-width: 0;
flex-direction: column;
overflow: hidden;
@@ -479,10 +480,10 @@
.thot-fleet-ledger .thot-fleet-ledger__context-panel {
position: absolute;
inset-block: 0.75rem;
inset-block: clamp(0.625rem, 2vh, 1rem);
inset-inline-start: 50%;
inset-inline-end: auto;
width: 60%;
width: min(75rem, calc(100% - 2rem));
max-width: none;
min-width: 0;
transform: translateX(-50%);
@@ -537,6 +538,20 @@
box-shadow: inset 0 -1px 0 oklch(var(--primary-foreground) / 0.18);
}
.thot-sensitive-run--success .thot-fleet-ledger__drawer-header,
.thot-sensitive-run--success .thot-catalog-drawer__header {
color: oklch(var(--primary-foreground));
background-color: oklch(var(--success));
border-color: oklch(var(--primary-foreground) / 0.18);
}
.thot-sensitive-run--failure .thot-fleet-ledger__drawer-header,
.thot-sensitive-run--failure .thot-catalog-drawer__header {
color: oklch(var(--destructive-foreground));
background-color: oklch(var(--destructive));
border-color: oklch(var(--destructive-foreground) / 0.18);
}
.thot-fleet-ledger__drawer-header .thot-fleet-ledger__drawer-title,
.thot-fleet-ledger__drawer-header .thot-fleet-ledger__drawer-description,
.thot-fleet-ledger__drawer-header .thot-fleet-ledger__eyebrow,
@@ -575,6 +590,19 @@
box-shadow: inset 0 0 0 1px oklch(var(--success) / 0.5);
}
.thot-catalog-drawer__event-log {
min-height: 8rem;
max-height: 18rem;
}
.thot-catalog-drawer__history-list {
max-height: 14rem;
}
.thot-catalog-drawer__run-layout > .thot-catalog-drawer__history {
margin-top: 1.25rem;
}
.thot-fleet-grid-surface {
position: relative;
}
@@ -583,6 +611,13 @@
background: oklch(var(--muted) / 0.32);
}
@media (min-width: 48rem) {
.thot-fleet-grid-toolbar > .thot-fleet-action-selector {
flex: 1 1 33rem;
min-width: 33rem;
}
}
.thot-fleet-ledger-grid {
--ag-header-background-color: oklch(var(--muted) / 0.84);
--ag-odd-row-background-color: oklch(var(--card));
@@ -605,6 +640,14 @@
background: oklch(var(--muted) / 0.12);
}
/* Keep selected rows readable without overpowering the status indicators. */
.thot-fleet-ledger-grid .ag-row.ag-row-selected,
.thot-fleet-ledger-grid .ag-row.ag-row-selected .ag-cell,
.thot-fleet-ledger-grid .ag-row.ag-row-selected .ag-grid-pinned-left-cells,
.thot-fleet-ledger-grid .ag-row.ag-row-selected .ag-grid-pinned-right-cells {
background-color: oklch(93% 0.002 17.2) !important;
}
.thot-fleet-icon-action {
position: relative;
}
@@ -744,6 +787,10 @@
padding: 0;
}
.thot-fleet-ledger__drawer-body.thot-catalog-drawer__run-layout {
padding: clamp(0.75rem, 1.5cqi, 1.25rem);
}
.thot-fleet-ledger__drawer-footer {
display: flex;
flex-wrap: wrap;
@@ -784,8 +831,8 @@
@container fleet-surface (max-width: 47.999rem) {
.thot-fleet-ledger__drawer,
.thot-fleet-ledger .thot-fleet-ledger__context-panel {
inset-block: 0.5rem;
width: calc(100% - 1rem);
inset-block: 0.625rem;
width: calc(100% - 1.5rem);
}
}
@@ -816,11 +863,11 @@
}
}
@media (min-width: 40rem) {
@container work-area-panel (min-width: 48rem) {
.thot-catalog-drawer__history-layout {
display: grid;
grid-template-columns: minmax(0, 1fr) 13.5rem;
gap: 0 1rem;
grid-template-columns: minmax(0, 1fr) clamp(12.5rem, 25cqi, 15rem);
gap: 0 clamp(0.75rem, 2cqi, 1.25rem);
align-content: start;
}
@@ -836,6 +883,45 @@
align-self: start;
margin-top: 0;
}
.thot-catalog-drawer__run-layout {
height: 100%;
min-height: 0;
grid-template-rows: minmax(0, 1fr);
align-content: stretch;
overflow: hidden;
}
.thot-catalog-drawer__run-layout > .thot-catalog-drawer__run-main {
display: flex;
min-height: 0;
flex-direction: column;
grid-column: 1;
grid-row: 1;
}
.thot-catalog-drawer__run-layout > .thot-catalog-drawer__history {
position: static;
display: flex;
min-height: 0;
flex-direction: column;
align-self: stretch;
grid-row: 1;
}
.thot-catalog-drawer__run-layout .thot-catalog-drawer__events {
display: flex;
min-height: 10rem;
flex: 1;
flex-direction: column;
}
.thot-catalog-drawer__run-layout .thot-catalog-drawer__event-log,
.thot-catalog-drawer__run-layout .thot-catalog-drawer__history-list {
min-height: 0;
max-height: none;
flex: 1;
}
}
@media (max-width: 47.999rem) {
@@ -24,4 +24,17 @@ describe("Recent runs success styling contract", () => {
expect(styles).toContain('button[data-status="succeeded"]');
expect(styles).toMatch(/oklch\(var\(--success\)/);
});
test("run drawers keep responsive gutters and switch columns from their own width", () => {
const styles = readFileSync(resolve(sourceRoot, "FleetLedgerShell.css"), "utf8");
expect(styles).toContain("width: min(75rem, calc(100% - 2rem));");
expect(styles).toContain(
".thot-fleet-ledger__drawer-body.thot-catalog-drawer__run-layout",
);
expect(styles).toContain("padding: clamp(0.75rem, 1.5cqi, 1.25rem);");
expect(styles).toContain("@container work-area-panel (min-width: 48rem)");
expect(styles).toContain(".thot-catalog-drawer__history-list");
expect(styles).toContain(".thot-catalog-drawer__event-log");
});
});
@@ -97,6 +97,11 @@ test("shows sensitive suggestion results, safe events, and lets operators inspec
const drawer = await screen.findByRole("dialog", { name: "Sensitive suggestion history" });
expect(within(drawer).getByRole("heading", { name: "Running" })).toBeVisible();
const progress = within(drawer).getByRole("table", { name: "Sensitive suggestion progress" });
expect(within(progress).getAllByRole("columnheader")).toHaveLength(3);
expect(within(drawer).getByRole("log", { name: "Sensitive suggestion events" })).toHaveClass(
"thot-catalog-drawer__event-log",
);
expect(await within(drawer).findByText("Classifying columns")).toBeVisible();
expect(within(drawer).queryByRole("button", { name: "Stop" })).not.toBeInTheDocument();
expect(within(drawer).queryByRole("button", { name: "Unlock stale run" })).not.toBeInTheDocument();
@@ -1,4 +1,5 @@
import { useEffect, useMemo } from "react";
import { LoaderCircle } from "lucide-react";
import { useQuery, useQueryClient } from "@tanstack/react-query";
import { apiErrorMessage } from "../../api/client";
import {
@@ -30,6 +31,12 @@ function isTerminal(run?: SensitiveDataSuggestionRun): boolean {
return Boolean(run && ["completed", "failed", "interrupted"].includes(run.status));
}
function outcomeClass(run: SensitiveDataSuggestionRun): string {
if (run.status === "completed") return "thot-sensitive-run--success";
if (["failed", "interrupted"].includes(run.status)) return "thot-sensitive-run--failure";
return "";
}
function safeFinalError(summary: string | null): string | null {
const value = summary?.trim();
if (!value) return null;
@@ -133,56 +140,92 @@ export function SensitiveDataSuggestionHistoryDrawer({
const events = eventQuery.data ?? [];
const finalError = safeFinalError(run.errorSummary);
const modelDisplay = modelLabel && modelLabel !== run.modelId
? `${modelLabel} (${run.modelId})`
: run.modelId;
const progressCounters = [
["total", "Total columns"],
["suggestedSensitive", "Sensitive"],
["suggestedNonSensitive", "Not sensitive"],
] as const;
return (
<FleetLedgerDrawer
open
className={outcomeClass(run)}
ariaLabel="Sensitive suggestion history"
eyebrow="Sensitive data suggestions"
title={statusLabel(run)}
description={`${modelLabel}${modelLabel !== run.modelId ? ` (${run.modelId})` : ""} · ${run.scope.replaceAll("_", " ")}`}
title={(
<span className="inline-flex items-center gap-2">
{!isTerminal(run) ? <LoaderCircle className="size-5 animate-spin" aria-hidden="true" /> : null}
{statusLabel(run)}
</span>
)}
description={run.scope.replaceAll("_", " ")}
onClose={onClose}
closeLabel="Close sensitive suggestion history"
bodyClassName="thot-catalog-drawer__history-layout"
bodyClassName="thot-catalog-drawer__history-layout thot-catalog-drawer__run-layout"
>
{runQuery.isError ? (
<div className="thot-catalog-drawer__run-main">
{runQuery.isError ? (
<div role="alert" className="rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive">
{apiErrorMessage(runQuery.error)}
</div>
) : null}
<section aria-label="Sensitive suggestion counters">
<div className={`grid gap-4 sm:grid-cols-2${runQuery.isError ? " mt-4" : ""}`}>
<section aria-label="AI usage">
<h3 className="thot-label mb-2">AI usage</h3>
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1 text-xs">
<dt>Model</dt><dd className="truncate text-right" title={modelDisplay}>{modelDisplay}</dd>
<dt>Input tokens</dt><dd className="text-right tabular-nums">{(run.inputTokens ?? 0).toLocaleString("en-US")}</dd>
<dt>Cache tokens</dt><dd className="text-right tabular-nums">{(run.cacheReadTokens ?? 0).toLocaleString("en-US")}</dd>
<dt>Output tokens</dt><dd className="text-right tabular-nums">{(run.outputTokens ?? 0).toLocaleString("en-US")}</dd>
</dl>
</section>
<section aria-label="Sensitive suggestion timestamps">
<h3 className="thot-label mb-2">Timestamps</h3>
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1 text-xs">
<dt className="text-muted-foreground">Created</dt><dd className="text-right tabular-nums">{timestamp(run.createdAt)}</dd>
<dt className="text-muted-foreground">Started</dt><dd className="text-right tabular-nums">{timestamp(run.startedAt)}</dd>
<dt className="text-muted-foreground">Updated</dt><dd className="text-right tabular-nums">{timestamp(run.updatedAt)}</dd>
<dt className="text-muted-foreground">Finished</dt><dd className="text-right tabular-nums">{timestamp(run.finishedAt)}</dd>
</dl>
</section>
</div>
<section className="mt-4" aria-label="Sensitive suggestion counters">
<h3 className="thot-label mb-2">Results</h3>
<div className="grid grid-cols-2 gap-2">
{([
["total", "Total columns"],
["suggestedSensitive", "Sensitive"],
["suggestedNonSensitive", "Not sensitive"],
] as const).map(([name, label]) => (
<div key={name} className="rounded-md border border-border bg-muted/25 px-3 py-2">
<p className="text-[11px] font-semibold uppercase tracking-wide text-muted-foreground">{label}</p>
<p data-count={name} className="mt-1 text-lg font-semibold tabular-nums">{run[name]}</p>
</div>
))}
<div className="overflow-x-auto rounded-md border border-border bg-muted/20">
<table className="w-full min-w-[20rem] table-fixed" aria-label="Sensitive suggestion progress">
<thead>
<tr className="border-b border-border bg-muted/45">
{progressCounters.map(([name, label]) => (
<th key={name} scope="col" className="border-l border-border px-2 py-1.5 text-left text-[10px] font-semibold uppercase tracking-wide text-muted-foreground first:border-l-0">
{label}
</th>
))}
</tr>
</thead>
<tbody>
<tr>
{progressCounters.map(([name]) => (
<td key={name} data-count={name} className="border-l border-border px-2 py-1.5 text-base font-semibold tabular-nums first:border-l-0">
{run[name]}
</td>
))}
</tr>
</tbody>
</table>
</div>
</section>
<section className="mt-5" aria-label="Sensitive suggestion timestamps">
<h3 className="thot-label mb-2">Timestamps</h3>
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1.5 text-sm">
<dt className="text-muted-foreground">Created</dt><dd className="text-right">{timestamp(run.createdAt)}</dd>
<dt className="text-muted-foreground">Started</dt><dd className="text-right">{timestamp(run.startedAt)}</dd>
<dt className="text-muted-foreground">Updated</dt><dd className="text-right">{timestamp(run.updatedAt)}</dd>
<dt className="text-muted-foreground">Finished</dt><dd className="text-right">{timestamp(run.finishedAt)}</dd>
</dl>
</section>
{finalError ? (
<div className="mt-5 rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive">
<div className="mt-3 rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive">
{finalError}
</div>
) : null}
<section className="mt-5" aria-label="Sensitive suggestion log">
<section className="thot-catalog-drawer__events mt-3" aria-label="Sensitive suggestion log">
<div className="mb-2 flex items-center justify-between">
<h3 className="thot-label">Events</h3>
<span className="text-xs tabular-nums text-muted-foreground">{events.length}</span>
@@ -190,7 +233,7 @@ export function SensitiveDataSuggestionHistoryDrawer({
<div
role="log"
aria-label="Sensitive suggestion events"
className="max-h-72 overflow-y-auto rounded-md bg-zinc-950 p-3 font-mono text-xs leading-5 text-zinc-200"
className="thot-catalog-drawer__event-log overflow-y-auto rounded-md bg-zinc-950 p-3 font-mono text-xs leading-5 text-zinc-200"
>
{events.length === 0 ? <p className="text-zinc-500">Waiting for events…</p> : events.map((event) => (
<p key={event.sequence} className={event.level === "error" ? "text-red-300" : event.level === "warning" ? "text-amber-300" : undefined}>
@@ -200,8 +243,9 @@ export function SensitiveDataSuggestionHistoryDrawer({
</div>
{eventQuery.isError ? <p role="alert" className="mt-2 text-sm text-destructive">{apiErrorMessage(eventQuery.error)}</p> : null}
</section>
</div>
<section className="thot-catalog-drawer__history mt-5" aria-label="Sensitive suggestion run history">
<section className="thot-catalog-drawer__history" aria-label="Sensitive suggestion run history">
<div className="mb-2 flex items-center justify-between gap-3">
<h3 className="thot-label">Recent runs</h3>
<span className="text-xs tabular-nums text-muted-foreground">
@@ -215,7 +259,7 @@ export function SensitiveDataSuggestionHistoryDrawer({
) : visibleHistory.length === 0 ? (
<p className="text-sm text-muted-foreground">No sensitive suggestion runs yet.</p>
) : (
<div className="max-h-56 divide-y divide-border overflow-y-auto rounded-md border border-border">
<div className="thot-catalog-drawer__history-list divide-y divide-border overflow-y-auto rounded-md border border-border">
{visibleHistory.map((item) => (
<button
key={item.id}
+35 -1
View File
@@ -265,6 +265,36 @@ def _response_content(response: object) -> str:
return content
def _response_usage(response: object) -> dict[str, int] | None:
"""Normalize LiteLLM usage, tolerating providers that omit it."""
try:
usage = _field(response, "usage")
prompt = _field(usage, "prompt_tokens")
output = _field(usage, "completion_tokens")
if type(prompt) is not int or prompt < 0 or type(output) is not int or output < 0:
return None
cached = 0
for source, name in ((usage, "cache_read_input_tokens"), (usage, "cached_tokens")):
try:
value = _field(source, name)
except _InvalidResponse:
continue
if type(value) is int and value >= 0:
cached = value
break
try:
details = _field(usage, "prompt_tokens_details")
value = _field(details, "cached_tokens")
if type(value) is int and value >= 0:
cached = value
except _InvalidResponse:
pass
cached = min(cached, prompt)
return {"input": prompt - cached, "cacheRead": cached, "output": output}
except _InvalidResponse:
return None
def handle_request(request: object, *, completion: Completion | None = None) -> Result:
"""Validate one request and normalize one injected or LiteLLM completion."""
try:
@@ -284,7 +314,11 @@ def handle_request(request: object, *, completion: Completion | None = None) ->
content = _response_content(response)
except Exception: # noqa: BLE001 - arbitrary provider objects are untrusted response data
return {"ok": False, "error": "invalid_response"}
return {"ok": True, "content": content}
result: dict[str, object] = {"ok": True, "content": content}
usage = _response_usage(response)
if usage is not None:
result["usage"] = usage
return result
def _write_json(payload: Mapping[str, object]) -> None:
@@ -75,6 +75,10 @@ func TestLoadAcceptsMixedKeyedAndKeylessMetadataGenerationModels(t *testing.T) {
label: DeepSeek V4 Pro
litellm: {provider: deepseek, model: deepseek-v4-pro}
apiKeyEnv: DEEPSEEK_API_KEY
- id: deepseek-v4-flash
label: DeepSeek V4 Flash
litellm: {provider: deepseek, model: deepseek-v4-flash}
apiKeyEnv: DEEPSEEK_API_KEY
- id: glm-53
label: GLM 5.3
litellm:
@@ -95,10 +99,10 @@ func TestLoadAcceptsMixedKeyedAndKeylessMetadataGenerationModels(t *testing.T) {
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if len(installation.MetadataGeneration.Models) != 3 {
if len(installation.MetadataGeneration.Models) != 4 {
t.Fatalf("metadata models = %#v", installation.MetadataGeneration.Models)
}
qwen := installation.MetadataGeneration.Models[2]
qwen := installation.MetadataGeneration.Models[3]
if qwen.APIKeyEnv != "" || !qwen.LiteLLM.DisableThinking || qwen.LiteLLM.Endpoint == nil {
t.Fatalf("keyless qwen model = %#v", qwen)
}