feat: refine metadata catalog workflows
This commit is contained in:
+3
-4
@@ -108,10 +108,9 @@ and staging only: `?db-ui=legacy` is honored in Vite development or when
|
|||||||
prototype on port `5173` also remains temporary until owner acceptance of the integrated surface,
|
prototype on port `5173` also remains temporary until owner acceptance of the integrated surface,
|
||||||
after which both migration aids can be removed.
|
after which both migration aids can be removed.
|
||||||
|
|
||||||
Schema refresh is one durable asynchronous engine with database-table, database-column,
|
Schema refresh is one durable asynchronous engine with database-table, selected-table-column,
|
||||||
selected-table-column, relationship, and full-database actions. Database-level menus expose the
|
relationship, and full-database actions. Database-level menus expose only the table, relationship,
|
||||||
table, all-column, relationship, and full scopes separately; selecting tables exposes column
|
and full scopes; selecting tables exposes column synchronization plus manual column and relationship cleanup for that subset. Database selections
|
||||||
synchronization plus manual column and relationship cleanup for that subset. Database selections
|
|
||||||
also expose manual table and relationship cleanup. Cleanup selections are atomic and share the
|
also expose manual table and relationship cleanup. Cleanup selections are atomic and share the
|
||||||
one-active-operation-per-database exclusion with synchronization. Runs have leases and
|
one-active-operation-per-database exclusion with synchronization. Runs have leases and
|
||||||
restart recovery, atomic apply, destructive-diff confirmation with re-scan, cancellation before
|
restart recovery, atomic apply, destructive-diff confirmation with re-scan, cancellation before
|
||||||
|
|||||||
+2
-2
@@ -59,7 +59,7 @@ import { DescriptionGenerationWorker } from "./catalog/description-generation-wo
|
|||||||
import { SensitiveDataSuggester } from "./catalog/sensitive-data-suggester.js";
|
import { SensitiveDataSuggester } from "./catalog/sensitive-data-suggester.js";
|
||||||
import { SensitiveDataSuggestionRunner } from "./catalog/sensitive-data-suggestion-runner.js";
|
import { SensitiveDataSuggestionRunner } from "./catalog/sensitive-data-suggestion-runner.js";
|
||||||
import {
|
import {
|
||||||
PostgresDescriptionSourceSampler,
|
ConcreteDescriptionSourceSampler,
|
||||||
type DescriptionSourceSampler,
|
type DescriptionSourceSampler,
|
||||||
} from "./catalog/description-source-sampler.js";
|
} from "./catalog/description-source-sampler.js";
|
||||||
import { catalogDescriptionGenerationRoutes } from "./routes/catalog-description-generation.js";
|
import { catalogDescriptionGenerationRoutes } from "./routes/catalog-description-generation.js";
|
||||||
@@ -177,7 +177,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
|||||||
{ connectTimeoutMs: config.workspaceDiagnosticTimeoutMs },
|
{ connectTimeoutMs: config.workspaceDiagnosticTimeoutMs },
|
||||||
);
|
);
|
||||||
const descriptionSourceSampler = deps?.descriptionSourceSampler
|
const descriptionSourceSampler = deps?.descriptionSourceSampler
|
||||||
?? new PostgresDescriptionSourceSampler(catalogPostgresAccess);
|
?? new ConcreteDescriptionSourceSampler(catalogPostgresAccess, workspaceSecretStore);
|
||||||
const descriptionGenerationWorker = new DescriptionGenerationWorker(
|
const descriptionGenerationWorker = new DescriptionGenerationWorker(
|
||||||
catalogRepository,
|
catalogRepository,
|
||||||
workspaceRegistry,
|
workspaceRegistry,
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
import type { WorkspaceRegistry } from "../workspaces/registry.js";
|
import type { WorkspaceRegistry } from "../workspaces/registry.js";
|
||||||
import type { MetadataGenerationModels, ResolvedMetadataGenerationModel } from "./metadata-generation-models.js";
|
import type { MetadataGenerationModels, ResolvedMetadataGenerationModel } from "./metadata-generation-models.js";
|
||||||
import type { ModelCompleter, ModelCompletionMessage } from "./model-completer.js";
|
import type { ModelCompleter, ModelCompletionMessage, ModelCompletionResult } from "./model-completer.js";
|
||||||
import {
|
import {
|
||||||
ModelCompletionCancelledError,
|
ModelCompletionCancelledError,
|
||||||
ModelCompletionProviderError,
|
ModelCompletionProviderError,
|
||||||
@@ -38,6 +38,7 @@ const MAX_SAMPLE_FIELDS_PER_ROW = 4;
|
|||||||
const MAX_SAMPLE_COLUMNS = 4;
|
const MAX_SAMPLE_COLUMNS = 4;
|
||||||
const MAX_REPRESENTATIVE_VALUES_PER_REQUEST = 5;
|
const MAX_REPRESENTATIVE_VALUES_PER_REQUEST = 5;
|
||||||
const MAX_TARGET_SAMPLE_JSON_BYTES = 8 * 1024;
|
const MAX_TARGET_SAMPLE_JSON_BYTES = 8 * 1024;
|
||||||
|
const MAX_COMPLETION_ATTEMPTS_PER_BATCH = 2;
|
||||||
const generatedOutcomeSchema = z.object({
|
const generatedOutcomeSchema = z.object({
|
||||||
targetId: z.uuid(),
|
targetId: z.uuid(),
|
||||||
outcome: z.literal("generated"),
|
outcome: z.literal("generated"),
|
||||||
@@ -55,7 +56,9 @@ const completionResponseSchema = z.object({
|
|||||||
results: z.array(outcomeSchema).min(1).max(MAX_TARGETS_PER_BATCH),
|
results: z.array(outcomeSchema).min(1).max(MAX_TARGETS_PER_BATCH),
|
||||||
}).strict();
|
}).strict();
|
||||||
|
|
||||||
class InvalidModelOutcomeError extends Error {}
|
class InvalidModelJsonError extends Error {}
|
||||||
|
class InvalidModelSchemaError extends Error {}
|
||||||
|
class MissingModelTargetsError extends Error {}
|
||||||
class DescriptionGenerationBatchError extends Error {
|
class DescriptionGenerationBatchError extends Error {
|
||||||
constructor(
|
constructor(
|
||||||
readonly failure: unknown,
|
readonly failure: unknown,
|
||||||
@@ -143,6 +146,9 @@ interface DescriptionGenerationCounters {
|
|||||||
generated: number;
|
generated: number;
|
||||||
nonGeneratable: number;
|
nonGeneratable: number;
|
||||||
failed: number;
|
failed: number;
|
||||||
|
inputTokens: number;
|
||||||
|
cacheReadTokens: number;
|
||||||
|
outputTokens: number;
|
||||||
consecutiveTechnicalFailures: number;
|
consecutiveTechnicalFailures: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -152,6 +158,9 @@ function persistedCounters(counters: DescriptionGenerationCounters) {
|
|||||||
generated: counters.generated,
|
generated: counters.generated,
|
||||||
nonGeneratable: counters.nonGeneratable,
|
nonGeneratable: counters.nonGeneratable,
|
||||||
failed: counters.failed,
|
failed: counters.failed,
|
||||||
|
inputTokens: counters.inputTokens,
|
||||||
|
cacheReadTokens: counters.cacheReadTokens,
|
||||||
|
outputTokens: counters.outputTokens,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -625,36 +634,47 @@ function messagesFor(
|
|||||||
}
|
}
|
||||||
|
|
||||||
function parseOutcomes(content: string, expectedTargetIds: readonly string[]): Map<string, ParsedOutcome> {
|
function parseOutcomes(content: string, expectedTargetIds: readonly string[]): Map<string, ParsedOutcome> {
|
||||||
|
const trimmed = content.trim();
|
||||||
|
const fenced = /^```(?:json)?[ \t]*\r?\n([\s\S]*?)\r?\n```$/iu.exec(trimmed);
|
||||||
|
let parsed: unknown;
|
||||||
try {
|
try {
|
||||||
const trimmed = content.trim();
|
parsed = JSON.parse(fenced?.[1] ?? trimmed);
|
||||||
const fenced = /^```(?:json)?[ \t]*\r?\n([\s\S]*?)\r?\n```$/iu.exec(trimmed);
|
} catch {
|
||||||
const outcomes = completionResponseSchema.parse(JSON.parse(fenced?.[1] ?? trimmed)).results;
|
throw new InvalidModelJsonError();
|
||||||
const expected = new Set(expectedTargetIds);
|
|
||||||
if (outcomes.length !== expectedTargetIds.length || expected.size !== expectedTargetIds.length) {
|
|
||||||
throw new InvalidModelOutcomeError();
|
|
||||||
}
|
|
||||||
const mapped = new Map<string, ParsedOutcome>();
|
|
||||||
for (const outcome of outcomes) {
|
|
||||||
if (!expected.has(outcome.targetId) || mapped.has(outcome.targetId)) {
|
|
||||||
throw new InvalidModelOutcomeError();
|
|
||||||
}
|
|
||||||
mapped.set(outcome.targetId, outcome.outcome === "generated"
|
|
||||||
? { ...outcome, description: outcome.description.trim() }
|
|
||||||
: outcome);
|
|
||||||
}
|
|
||||||
if (mapped.size !== expected.size) throw new InvalidModelOutcomeError();
|
|
||||||
return mapped;
|
|
||||||
} catch (error) {
|
|
||||||
if (error instanceof InvalidModelOutcomeError) throw error;
|
|
||||||
throw new InvalidModelOutcomeError();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const response = completionResponseSchema.safeParse(parsed);
|
||||||
|
if (!response.success) throw new InvalidModelSchemaError();
|
||||||
|
|
||||||
|
const outcomes = response.data.results;
|
||||||
|
const expected = new Set(expectedTargetIds);
|
||||||
|
if (outcomes.length !== expectedTargetIds.length || expected.size !== expectedTargetIds.length) {
|
||||||
|
throw new MissingModelTargetsError();
|
||||||
|
}
|
||||||
|
const mapped = new Map<string, ParsedOutcome>();
|
||||||
|
for (const outcome of outcomes) {
|
||||||
|
if (!expected.has(outcome.targetId) || mapped.has(outcome.targetId)) {
|
||||||
|
throw new MissingModelTargetsError();
|
||||||
|
}
|
||||||
|
mapped.set(outcome.targetId, outcome.outcome === "generated"
|
||||||
|
? { ...outcome, description: outcome.description.trim() }
|
||||||
|
: outcome);
|
||||||
|
}
|
||||||
|
if (mapped.size !== expected.size) throw new MissingModelTargetsError();
|
||||||
|
return mapped;
|
||||||
}
|
}
|
||||||
|
|
||||||
function safeFailure(error: unknown): string {
|
function safeFailure(error: unknown): string {
|
||||||
if (error instanceof DescriptionGenerationFailureStreakError) return error.message;
|
if (error instanceof DescriptionGenerationFailureStreakError) return error.message;
|
||||||
const failure = error instanceof DescriptionGenerationBatchError ? error.failure : error;
|
const failure = error instanceof DescriptionGenerationBatchError ? error.failure : error;
|
||||||
if (failure instanceof ModelCompletionProviderError) return "The model provider request failed.";
|
if (failure instanceof ModelCompletionProviderError) return "The model provider request failed.";
|
||||||
if (failure instanceof InvalidModelOutcomeError) return "The model response was invalid.";
|
if (failure instanceof InvalidModelJsonError) return "The model response was not valid JSON.";
|
||||||
|
if (failure instanceof InvalidModelSchemaError) {
|
||||||
|
return "The model response did not match the required schema.";
|
||||||
|
}
|
||||||
|
if (failure instanceof MissingModelTargetsError) {
|
||||||
|
return "The model response was missing one or more requested targets.";
|
||||||
|
}
|
||||||
return "Description generation failed.";
|
return "Description generation failed.";
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -907,6 +927,9 @@ export class DescriptionGenerationWorker {
|
|||||||
generated: 0,
|
generated: 0,
|
||||||
nonGeneratable: 0,
|
nonGeneratable: 0,
|
||||||
failed: 0,
|
failed: 0,
|
||||||
|
inputTokens: 0,
|
||||||
|
cacheReadTokens: 0,
|
||||||
|
outputTokens: 0,
|
||||||
consecutiveTechnicalFailures: 0,
|
consecutiveTechnicalFailures: 0,
|
||||||
};
|
};
|
||||||
await this.processTargets(run, database, plan.columnTargets, model, counters, signal);
|
await this.processTargets(run, database, plan.columnTargets, model, counters, signal);
|
||||||
@@ -973,20 +996,41 @@ export class DescriptionGenerationWorker {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
throwIfCancelled(signal);
|
throwIfCancelled(signal);
|
||||||
let outcomes: Map<string, ParsedOutcome>;
|
const expectedTargetIds = batch.map((target) => (
|
||||||
try {
|
target.kind === "column" ? target.column.id : target.table.id
|
||||||
const content = await this.completer.complete({
|
));
|
||||||
model,
|
const messages = messagesFor(database, batch, run.language, sourceSamples);
|
||||||
messages: messagesFor(database, batch, run.language, sourceSamples),
|
let outcomes: Map<string, ParsedOutcome> | undefined;
|
||||||
signal,
|
let terminalFailure: unknown;
|
||||||
});
|
for (let attempt = 1; attempt <= MAX_COMPLETION_ATTEMPTS_PER_BATCH; attempt += 1) {
|
||||||
throwIfCancelled(signal);
|
try {
|
||||||
outcomes = parseOutcomes(content, batch.map((target) => (
|
const completion = await this.completer.complete({ model, messages, signal });
|
||||||
target.kind === "column" ? target.column.id : target.table.id
|
const result: ModelCompletionResult = typeof completion === "string"
|
||||||
)));
|
? { content: completion, usage: { input: 0, cacheRead: 0, output: 0 } }
|
||||||
} catch (error) {
|
: completion;
|
||||||
if (error instanceof ModelCompletionCancelledError) throw error;
|
counters.inputTokens += result.usage.input;
|
||||||
const batchError = new DescriptionGenerationBatchError(error, batch.map(failureTarget));
|
counters.cacheReadTokens += result.usage.cacheRead;
|
||||||
|
counters.outputTokens += result.usage.output;
|
||||||
|
throwIfCancelled(signal);
|
||||||
|
outcomes = parseOutcomes(result.content, expectedTargetIds);
|
||||||
|
break;
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof ModelCompletionCancelledError) throw error;
|
||||||
|
terminalFailure = error;
|
||||||
|
if (attempt < MAX_COMPLETION_ATTEMPTS_PER_BATCH) {
|
||||||
|
await this.appendEvent(
|
||||||
|
run.id,
|
||||||
|
"warning",
|
||||||
|
`${safeFailure(error)} Retrying batch (attempt ${attempt + 1} of ${MAX_COMPLETION_ATTEMPTS_PER_BATCH}).`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!outcomes) {
|
||||||
|
const batchError = new DescriptionGenerationBatchError(
|
||||||
|
terminalFailure,
|
||||||
|
batch.map(failureTarget),
|
||||||
|
);
|
||||||
counters.processed += batch.length;
|
counters.processed += batch.length;
|
||||||
counters.failed += batch.length;
|
counters.failed += batch.length;
|
||||||
counters.consecutiveTechnicalFailures += 1;
|
counters.consecutiveTechnicalFailures += 1;
|
||||||
@@ -1009,7 +1053,10 @@ export class DescriptionGenerationWorker {
|
|||||||
const targetId = target.kind === "column" ? target.column.id : target.table.id;
|
const targetId = target.kind === "column" ? target.column.id : target.table.id;
|
||||||
const outcome = outcomes.get(targetId);
|
const outcome = outcomes.get(targetId);
|
||||||
if (!outcome) {
|
if (!outcome) {
|
||||||
throw new DescriptionGenerationBatchError(new InvalidModelOutcomeError(), [failureTarget(target)]);
|
throw new DescriptionGenerationBatchError(
|
||||||
|
new MissingModelTargetsError(),
|
||||||
|
[failureTarget(target)],
|
||||||
|
);
|
||||||
}
|
}
|
||||||
const generatedDescription = outcome.outcome === "generated"
|
const generatedDescription = outcome.outcome === "generated"
|
||||||
? outcome.description
|
? outcome.description
|
||||||
|
|||||||
@@ -1,5 +1,8 @@
|
|||||||
|
import { readFile } from "node:fs/promises";
|
||||||
|
import type { WorkspaceSecretStore } from "../workspaces/secret-store.js";
|
||||||
import type { CatalogPostgresAccess } from "./postgres-access.js";
|
import type { CatalogPostgresAccess } from "./postgres-access.js";
|
||||||
import type { WorkspaceDatabase } from "./types.js";
|
import { CATALOG_SECRET_IDS } from "./secrets.js";
|
||||||
|
import { CatalogConnectorError, type WorkspaceDatabase } from "./types.js";
|
||||||
|
|
||||||
const MAX_SOURCE_ROWS = 5;
|
const MAX_SOURCE_ROWS = 5;
|
||||||
const MAX_REPRESENTATIVE_VALUES = 5;
|
const MAX_REPRESENTATIVE_VALUES = 5;
|
||||||
@@ -79,15 +82,82 @@ function distinctKey(value: Exclude<DescriptionSourceSampleValue, null>): string
|
|||||||
return `${typeof value}:${String(value)}`;
|
return `${typeof value}:${String(value)}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** PostgreSQL-wire sampler. REST bindings remain unsupported by CatalogPostgresAccess. */
|
function columnsFor(target: DescriptionSourceSamplingTarget): string[] {
|
||||||
export class PostgresDescriptionSourceSampler implements DescriptionSourceSampler {
|
return [...new Set(target.columnNames)].slice(0, MAX_SOURCE_COLUMNS_PER_TARGET);
|
||||||
constructor(private readonly access: CatalogPostgresAccess) {}
|
}
|
||||||
|
|
||||||
|
function normalizedSample(
|
||||||
|
target: DescriptionSourceSamplingTarget,
|
||||||
|
columnNames: readonly string[],
|
||||||
|
sourceRows: readonly Record<string, unknown>[],
|
||||||
|
): DescriptionTargetSourceSample {
|
||||||
|
const rows = sourceRows.slice(0, MAX_SOURCE_ROWS).map((row) => ({
|
||||||
|
fields: columnNames.flatMap((name) => {
|
||||||
|
const value = normalizeValue(row[name]);
|
||||||
|
return value === undefined ? [] : [{ name, value }];
|
||||||
|
}),
|
||||||
|
}));
|
||||||
|
const valuesByColumn = new Map<string, Exclude<DescriptionSourceSampleValue, null>[]>();
|
||||||
|
const seenByColumn = new Map<string, Set<string>>();
|
||||||
|
let representativeValueCount = 0;
|
||||||
|
for (const row of rows) {
|
||||||
|
for (const field of row.fields) {
|
||||||
|
if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break;
|
||||||
|
if (field.value === null) continue;
|
||||||
|
const seen = seenByColumn.get(field.name) ?? new Set<string>();
|
||||||
|
const key = distinctKey(field.value);
|
||||||
|
if (seen.has(key)) continue;
|
||||||
|
seen.add(key);
|
||||||
|
seenByColumn.set(field.name, seen);
|
||||||
|
const values = valuesByColumn.get(field.name) ?? [];
|
||||||
|
values.push(field.value);
|
||||||
|
valuesByColumn.set(field.name, values);
|
||||||
|
representativeValueCount += 1;
|
||||||
|
}
|
||||||
|
if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
targetId: target.targetId,
|
||||||
|
tableName: target.tableName,
|
||||||
|
rows,
|
||||||
|
representativeValues: columnNames.flatMap((column) => {
|
||||||
|
const values = valuesByColumn.get(column);
|
||||||
|
return values && values.length > 0 ? [{ column, values }] : [];
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function samplingSql(
|
||||||
|
database: WorkspaceDatabase,
|
||||||
|
target: DescriptionSourceSamplingTarget,
|
||||||
|
columns: readonly string[],
|
||||||
|
): string {
|
||||||
|
const projections = columns.map((columnName) => {
|
||||||
|
const identifier = quoteIdentifier(columnName);
|
||||||
|
return `LEFT((${identifier})::text, ${MAX_SOURCE_VALUE_BYTES}) AS ${identifier}`;
|
||||||
|
});
|
||||||
|
return [
|
||||||
|
`SELECT ${projections.join(", ")}`,
|
||||||
|
`FROM ${quoteIdentifier(database.schema)}.${quoteIdentifier(target.tableName)}`,
|
||||||
|
`LIMIT ${MAX_SOURCE_ROWS}`,
|
||||||
|
].join(" ");
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Bounded source sampler that follows the database's PostgreSQL-wire or REST binding. */
|
||||||
|
export class ConcreteDescriptionSourceSampler implements DescriptionSourceSampler {
|
||||||
|
constructor(
|
||||||
|
private readonly access: CatalogPostgresAccess,
|
||||||
|
private readonly secretStore?: Pick<WorkspaceSecretStore, "materialize">,
|
||||||
|
) {}
|
||||||
|
|
||||||
async sample(
|
async sample(
|
||||||
database: WorkspaceDatabase,
|
database: WorkspaceDatabase,
|
||||||
targets: readonly DescriptionSourceSamplingTarget[],
|
targets: readonly DescriptionSourceSamplingTarget[],
|
||||||
signal: AbortSignal,
|
signal: AbortSignal,
|
||||||
): Promise<readonly DescriptionTargetSourceSample[]> {
|
): Promise<readonly DescriptionTargetSourceSample[]> {
|
||||||
|
if (database.binding.transport === "rest_api") {
|
||||||
|
return await this.sampleRest(database, targets, signal);
|
||||||
|
}
|
||||||
const client = await this.access.connect(database, signal);
|
const client = await this.access.connect(database, signal);
|
||||||
let transactionOpen = false;
|
let transactionOpen = false;
|
||||||
try {
|
try {
|
||||||
@@ -95,7 +165,7 @@ export class PostgresDescriptionSourceSampler implements DescriptionSourceSample
|
|||||||
transactionOpen = true;
|
transactionOpen = true;
|
||||||
const samples: DescriptionTargetSourceSample[] = [];
|
const samples: DescriptionTargetSourceSample[] = [];
|
||||||
for (const target of targets) {
|
for (const target of targets) {
|
||||||
const columnNames = [...new Set(target.columnNames)].slice(0, MAX_SOURCE_COLUMNS_PER_TARGET);
|
const columnNames = columnsFor(target);
|
||||||
if (columnNames.length === 0) {
|
if (columnNames.length === 0) {
|
||||||
samples.push({
|
samples.push({
|
||||||
targetId: target.targetId,
|
targetId: target.targetId,
|
||||||
@@ -115,44 +185,7 @@ export class PostgresDescriptionSourceSampler implements DescriptionSourceSample
|
|||||||
"LIMIT $2",
|
"LIMIT $2",
|
||||||
].join(" ");
|
].join(" ");
|
||||||
const result = await client.query(sql, [MAX_SOURCE_VALUE_BYTES, MAX_SOURCE_ROWS]);
|
const result = await client.query(sql, [MAX_SOURCE_VALUE_BYTES, MAX_SOURCE_ROWS]);
|
||||||
const rows = result.rows.slice(0, MAX_SOURCE_ROWS).map((row) => ({
|
samples.push(normalizedSample(target, columnNames, result.rows));
|
||||||
fields: columnNames.flatMap((name) => {
|
|
||||||
const value = normalizeValue(row[name]);
|
|
||||||
return value === undefined ? [] : [{ name, value }];
|
|
||||||
}),
|
|
||||||
}));
|
|
||||||
const valuesByColumn = new Map<
|
|
||||||
string,
|
|
||||||
Exclude<DescriptionSourceSampleValue, null>[]
|
|
||||||
>();
|
|
||||||
const seenByColumn = new Map<string, Set<string>>();
|
|
||||||
let representativeValueCount = 0;
|
|
||||||
for (const row of rows) {
|
|
||||||
for (const field of row.fields) {
|
|
||||||
if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break;
|
|
||||||
if (field.value === null) continue;
|
|
||||||
const seen = seenByColumn.get(field.name) ?? new Set<string>();
|
|
||||||
const key = distinctKey(field.value);
|
|
||||||
if (seen.has(key)) continue;
|
|
||||||
seen.add(key);
|
|
||||||
seenByColumn.set(field.name, seen);
|
|
||||||
const values = valuesByColumn.get(field.name) ?? [];
|
|
||||||
values.push(field.value);
|
|
||||||
valuesByColumn.set(field.name, values);
|
|
||||||
representativeValueCount += 1;
|
|
||||||
}
|
|
||||||
if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break;
|
|
||||||
}
|
|
||||||
const representativeValues = columnNames.flatMap((column) => {
|
|
||||||
const values = valuesByColumn.get(column);
|
|
||||||
return values && values.length > 0 ? [{ column, values }] : [];
|
|
||||||
});
|
|
||||||
samples.push({
|
|
||||||
targetId: target.targetId,
|
|
||||||
tableName: target.tableName,
|
|
||||||
rows,
|
|
||||||
representativeValues,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
return samples;
|
return samples;
|
||||||
} finally {
|
} finally {
|
||||||
@@ -160,4 +193,60 @@ export class PostgresDescriptionSourceSampler implements DescriptionSourceSample
|
|||||||
await client.end().catch(() => undefined);
|
await client.end().catch(() => undefined);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async sampleRest(
|
||||||
|
database: WorkspaceDatabase,
|
||||||
|
targets: readonly DescriptionSourceSamplingTarget[],
|
||||||
|
signal: AbortSignal,
|
||||||
|
): Promise<readonly DescriptionTargetSourceSample[]> {
|
||||||
|
if (!this.secretStore) throw new CatalogConnectorError("REST source sampling is not configured");
|
||||||
|
const auth = database.binding.restAuth ?? "bearer";
|
||||||
|
const materialized = this.secretStore.materialize(
|
||||||
|
database.workspaceId,
|
||||||
|
auth === "none" ? [] : [CATALOG_SECRET_IDS.apiKey],
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
const headers: Record<string, string> = { "content-type": "application/json" };
|
||||||
|
if (auth !== "none") {
|
||||||
|
const credentialFile = materialized.files.get(CATALOG_SECRET_IDS.apiKey);
|
||||||
|
if (!credentialFile) throw new CatalogConnectorError("REST API key is not configured");
|
||||||
|
const credential = (await readFile(credentialFile, "utf8")).trim();
|
||||||
|
if (auth === "bearer") headers.authorization = `Bearer ${credential}`;
|
||||||
|
else headers["x-api-key"] = credential;
|
||||||
|
}
|
||||||
|
const baseUrl = database.binding.baseUrl?.replace(/\/+$/, "");
|
||||||
|
if (!baseUrl) throw new CatalogConnectorError("Database binding is incomplete");
|
||||||
|
const samples: DescriptionTargetSourceSample[] = [];
|
||||||
|
for (const target of targets) {
|
||||||
|
const columnNames = columnsFor(target);
|
||||||
|
if (columnNames.length === 0) {
|
||||||
|
samples.push(normalizedSample(target, columnNames, []));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const response = await fetch(`${baseUrl}/rpc/run_query`, {
|
||||||
|
method: "POST",
|
||||||
|
headers,
|
||||||
|
body: JSON.stringify({ query_text: samplingSql(database, target, columnNames) }),
|
||||||
|
signal,
|
||||||
|
});
|
||||||
|
if (!response.ok) throw new CatalogConnectorError("REST source sampling failed");
|
||||||
|
const body: unknown = await response.json();
|
||||||
|
if (!Array.isArray(body)
|
||||||
|
|| body.some((row) => !row || typeof row !== "object" || Array.isArray(row))) {
|
||||||
|
throw new CatalogConnectorError("REST source sampling response is invalid");
|
||||||
|
}
|
||||||
|
samples.push(normalizedSample(
|
||||||
|
target,
|
||||||
|
columnNames,
|
||||||
|
body as Array<Record<string, unknown>>,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
return samples;
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof CatalogConnectorError) throw error;
|
||||||
|
throw new CatalogConnectorError("REST source sampling failed");
|
||||||
|
} finally {
|
||||||
|
materialized.release();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -335,7 +335,10 @@ export class MemoryCatalogRepository implements CatalogRepository {
|
|||||||
processed: 0,
|
processed: 0,
|
||||||
generated: 0,
|
generated: 0,
|
||||||
nonGeneratable: 0,
|
nonGeneratable: 0,
|
||||||
failed: 0,
|
failed: 0,
|
||||||
|
inputTokens: 0,
|
||||||
|
cacheReadTokens: 0,
|
||||||
|
outputTokens: 0,
|
||||||
createdAt: now,
|
createdAt: now,
|
||||||
startedAt: null,
|
startedAt: null,
|
||||||
updatedAt: now,
|
updatedAt: now,
|
||||||
@@ -444,7 +447,10 @@ export class MemoryCatalogRepository implements CatalogRepository {
|
|||||||
status: "running",
|
status: "running",
|
||||||
total: 0,
|
total: 0,
|
||||||
suggestedSensitive: 0,
|
suggestedSensitive: 0,
|
||||||
suggestedNonSensitive: 0,
|
suggestedNonSensitive: 0,
|
||||||
|
inputTokens: 0,
|
||||||
|
cacheReadTokens: 0,
|
||||||
|
outputTokens: 0,
|
||||||
createdAt: now,
|
createdAt: now,
|
||||||
startedAt: now,
|
startedAt: now,
|
||||||
updatedAt: now,
|
updatedAt: now,
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import * as descriptionGenerationRunsMigration from "./migrations/005_descriptio
|
|||||||
import * as sensitiveDataFlagMigration from "./migrations/006_sensitive_data_flag.js";
|
import * as sensitiveDataFlagMigration from "./migrations/006_sensitive_data_flag.js";
|
||||||
import * as sensitiveDataSuggestionRunsMigration from "./migrations/007_sensitive_data_suggestion_runs.js";
|
import * as sensitiveDataSuggestionRunsMigration from "./migrations/007_sensitive_data_suggestion_runs.js";
|
||||||
import * as catalogLogicalRelationshipsMigration from "./migrations/008_catalog_logical_relationships.js";
|
import * as catalogLogicalRelationshipsMigration from "./migrations/008_catalog_logical_relationships.js";
|
||||||
|
import * as aiTokenUsageMigration from "./migrations/009_ai_token_usage.js";
|
||||||
|
|
||||||
const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL;
|
const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL;
|
||||||
const host = process.env.THT_CATALOG_DB_HOST;
|
const host = process.env.THT_CATALOG_DB_HOST;
|
||||||
@@ -44,6 +45,7 @@ const provider: MigrationProvider = {
|
|||||||
"006_sensitive_data_flag": sensitiveDataFlagMigration,
|
"006_sensitive_data_flag": sensitiveDataFlagMigration,
|
||||||
"007_sensitive_data_suggestion_runs": sensitiveDataSuggestionRunsMigration,
|
"007_sensitive_data_suggestion_runs": sensitiveDataSuggestionRunsMigration,
|
||||||
"008_catalog_logical_relationships": catalogLogicalRelationshipsMigration,
|
"008_catalog_logical_relationships": catalogLogicalRelationshipsMigration,
|
||||||
|
"009_ai_token_usage": aiTokenUsageMigration,
|
||||||
};
|
};
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
import type { Kysely } from "kysely";
|
||||||
|
import type { CatalogDatabase } from "../repository.js";
|
||||||
|
|
||||||
|
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||||
|
for (const table of ["description_generation_runs", "sensitive_data_suggestion_runs"] as const) {
|
||||||
|
await db.schema.alterTable(table)
|
||||||
|
.addColumn("input_tokens", "integer", (col) => col.notNull().defaultTo(0))
|
||||||
|
.addColumn("cache_read_tokens", "integer", (col) => col.notNull().defaultTo(0))
|
||||||
|
.addColumn("output_tokens", "integer", (col) => col.notNull().defaultTo(0))
|
||||||
|
.execute();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||||
|
for (const table of ["sensitive_data_suggestion_runs", "description_generation_runs"] as const) {
|
||||||
|
await db.schema.alterTable(table)
|
||||||
|
.dropColumn("input_tokens").dropColumn("cache_read_tokens").dropColumn("output_tokens")
|
||||||
|
.execute();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,7 +4,7 @@ import type { ResolvedMetadataGenerationModel } from "./metadata-generation-mode
|
|||||||
|
|
||||||
const MAX_HELPER_OUTPUT_BYTES = 64 * 1024;
|
const MAX_HELPER_OUTPUT_BYTES = 64 * 1024;
|
||||||
const helperOutputSchema = z.discriminatedUnion("ok", [
|
const helperOutputSchema = z.discriminatedUnion("ok", [
|
||||||
z.object({ ok: z.literal(true), content: z.string() }).strict(),
|
z.object({ ok: z.literal(true), content: z.string(), usage: z.object({ input: z.number().int().nonnegative(), cacheRead: z.number().int().nonnegative(), output: z.number().int().nonnegative() }).strict().optional() }).strict(),
|
||||||
z.object({ ok: z.literal(false), error: z.literal("provider_failure") }).strict(),
|
z.object({ ok: z.literal(false), error: z.literal("provider_failure") }).strict(),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
@@ -18,10 +18,12 @@ export interface ModelCompletionRequest {
|
|||||||
messages: readonly ModelCompletionMessage[];
|
messages: readonly ModelCompletionMessage[];
|
||||||
signal: AbortSignal;
|
signal: AbortSignal;
|
||||||
}
|
}
|
||||||
|
export interface ModelCompletionUsage { input: number; cacheRead: number; output: number; }
|
||||||
|
export interface ModelCompletionResult { content: string; usage: ModelCompletionUsage; }
|
||||||
|
|
||||||
/** The provider boundary used by Description Generation. */
|
/** The provider boundary used by Description Generation. */
|
||||||
export interface ModelCompleter {
|
export interface ModelCompleter {
|
||||||
complete(request: ModelCompletionRequest): Promise<string>;
|
complete(request: ModelCompletionRequest): Promise<string | ModelCompletionResult>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class ModelCompletionProviderError extends Error {
|
export class ModelCompletionProviderError extends Error {
|
||||||
@@ -47,7 +49,7 @@ export class PythonModelCompleter implements ModelCompleter {
|
|||||||
terminationGraceMs?: number;
|
terminationGraceMs?: number;
|
||||||
}) {}
|
}) {}
|
||||||
|
|
||||||
async complete(request: ModelCompletionRequest): Promise<string> {
|
async complete(request: ModelCompletionRequest): Promise<ModelCompletionResult> {
|
||||||
if (request.signal.aborted) throw new ModelCompletionCancelledError();
|
if (request.signal.aborted) throw new ModelCompletionCancelledError();
|
||||||
const payload = {
|
const payload = {
|
||||||
model: `${request.model.provider}/${request.model.model}`,
|
model: `${request.model.provider}/${request.model.model}`,
|
||||||
@@ -62,7 +64,7 @@ export class PythonModelCompleter implements ModelCompleter {
|
|||||||
...(request.model.disableThinking === true ? { disable_thinking: true } : {}),
|
...(request.model.disableThinking === true ? { disable_thinking: true } : {}),
|
||||||
};
|
};
|
||||||
|
|
||||||
return await new Promise<string>((resolve, reject) => {
|
return await new Promise<ModelCompletionResult>((resolve, reject) => {
|
||||||
const child = spawn(
|
const child = spawn(
|
||||||
this.options.pythonExecutable,
|
this.options.pythonExecutable,
|
||||||
["-m", this.options.helperModule ?? "tht.internal.litellm_completion"],
|
["-m", this.options.helperModule ?? "tht.internal.litellm_completion"],
|
||||||
@@ -135,7 +137,7 @@ export class PythonModelCompleter implements ModelCompleter {
|
|||||||
if (!output.ok) return fail();
|
if (!output.ok) return fail();
|
||||||
settled = true;
|
settled = true;
|
||||||
cleanup();
|
cleanup();
|
||||||
resolve(output.content);
|
resolve({ content: output.content, usage: output.usage ?? { input: 0, cacheRead: 0, output: 0 } });
|
||||||
} catch {
|
} catch {
|
||||||
fail();
|
fail();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -171,6 +171,9 @@ interface DescriptionGenerationRunTable {
|
|||||||
generated: number;
|
generated: number;
|
||||||
nonGeneratable: number;
|
nonGeneratable: number;
|
||||||
failed: number;
|
failed: number;
|
||||||
|
inputTokens: number;
|
||||||
|
cacheReadTokens: number;
|
||||||
|
outputTokens: number;
|
||||||
createdAt: Timestamp;
|
createdAt: Timestamp;
|
||||||
startedAt: Timestamp | null;
|
startedAt: Timestamp | null;
|
||||||
updatedAt: Timestamp;
|
updatedAt: Timestamp;
|
||||||
@@ -195,6 +198,9 @@ interface SensitiveDataSuggestionRunTable {
|
|||||||
total: number;
|
total: number;
|
||||||
suggestedSensitive: number;
|
suggestedSensitive: number;
|
||||||
suggestedNonSensitive: number;
|
suggestedNonSensitive: number;
|
||||||
|
inputTokens: number;
|
||||||
|
cacheReadTokens: number;
|
||||||
|
outputTokens: number;
|
||||||
createdAt: Timestamp;
|
createdAt: Timestamp;
|
||||||
startedAt: Timestamp;
|
startedAt: Timestamp;
|
||||||
updatedAt: Timestamp;
|
updatedAt: Timestamp;
|
||||||
@@ -816,6 +822,9 @@ export class KyselyCatalogRepository implements CatalogRepository {
|
|||||||
generated: 0,
|
generated: 0,
|
||||||
nonGeneratable: 0,
|
nonGeneratable: 0,
|
||||||
failed: 0,
|
failed: 0,
|
||||||
|
inputTokens: 0,
|
||||||
|
cacheReadTokens: 0,
|
||||||
|
outputTokens: 0,
|
||||||
startedAt: null,
|
startedAt: null,
|
||||||
finishedAt: null,
|
finishedAt: null,
|
||||||
errorSummary: null,
|
errorSummary: null,
|
||||||
@@ -943,6 +952,9 @@ export class KyselyCatalogRepository implements CatalogRepository {
|
|||||||
total: 0,
|
total: 0,
|
||||||
suggestedSensitive: 0,
|
suggestedSensitive: 0,
|
||||||
suggestedNonSensitive: 0,
|
suggestedNonSensitive: 0,
|
||||||
|
inputTokens: 0,
|
||||||
|
cacheReadTokens: 0,
|
||||||
|
outputTokens: 0,
|
||||||
finishedAt: null,
|
finishedAt: null,
|
||||||
errorSummary: null,
|
errorSummary: null,
|
||||||
}).returningAll().executeTakeFirstOrThrow();
|
}).returningAll().executeTakeFirstOrThrow();
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
import type { MetadataGenerationModels } from "./metadata-generation-models.js";
|
import type { MetadataGenerationModels } from "./metadata-generation-models.js";
|
||||||
import type { ModelCompleter, ModelCompletionMessage } from "./model-completer.js";
|
import type { ModelCompleter, ModelCompletionMessage, ModelCompletionResult, ModelCompletionUsage } from "./model-completer.js";
|
||||||
import type {
|
import type {
|
||||||
CatalogColumn,
|
CatalogColumn,
|
||||||
CatalogRepository,
|
CatalogRepository,
|
||||||
@@ -201,6 +201,8 @@ export class SensitiveDataSuggester {
|
|||||||
targetIds: readonly string[],
|
targetIds: readonly string[],
|
||||||
signal: AbortSignal,
|
signal: AbortSignal,
|
||||||
onPrepared?: (total: number) => void | Promise<void>,
|
onPrepared?: (total: number) => void | Promise<void>,
|
||||||
|
onProgress?: (processed: number, suggestions: readonly SensitiveDataSuggestion[]) => void | Promise<void>,
|
||||||
|
onUsage?: (usage: ModelCompletionUsage) => void | Promise<void>,
|
||||||
): Promise<readonly SensitiveDataSuggestion[]> {
|
): Promise<readonly SensitiveDataSuggestion[]> {
|
||||||
const database = await this.repository.get(databaseId);
|
const database = await this.repository.get(databaseId);
|
||||||
if (!database) throw new SensitiveDataSuggestionTargetNotFoundError("database");
|
if (!database) throw new SensitiveDataSuggestionTargetNotFoundError("database");
|
||||||
@@ -212,11 +214,16 @@ export class SensitiveDataSuggester {
|
|||||||
for (const batch of batchesFor(database, columns)) {
|
for (const batch of batchesFor(database, columns)) {
|
||||||
let received: Map<string, { columnId: string; sensitive: boolean }> | undefined;
|
let received: Map<string, { columnId: string; sensitive: boolean }> | undefined;
|
||||||
for (let attempt = 0; attempt < 2 && !received; attempt += 1) {
|
for (let attempt = 0; attempt < 2 && !received; attempt += 1) {
|
||||||
const content = await this.completer.complete({
|
const completion = await this.completer.complete({
|
||||||
model,
|
model,
|
||||||
signal,
|
signal,
|
||||||
messages: [systemMessage, { role: "user", content: userContent(database, batch) }],
|
messages: [systemMessage, { role: "user", content: userContent(database, batch) }],
|
||||||
});
|
});
|
||||||
|
const result: ModelCompletionResult = typeof completion === "string"
|
||||||
|
? { content: completion, usage: { input: 0, cacheRead: 0, output: 0 } }
|
||||||
|
: completion;
|
||||||
|
await onUsage?.(result.usage);
|
||||||
|
const content = result.content;
|
||||||
try {
|
try {
|
||||||
const parsed = responseSchema.parse(JSON.parse(content));
|
const parsed = responseSchema.parse(JSON.parse(content));
|
||||||
const expected = new Set(batch.map((column) => column.columnId));
|
const expected = new Set(batch.map((column) => column.columnId));
|
||||||
@@ -240,6 +247,7 @@ export class SensitiveDataSuggester {
|
|||||||
currentSensitive: column.currentSensitive,
|
currentSensitive: column.currentSensitive,
|
||||||
sensitive: received!.get(column.columnId)!.sensitive,
|
sensitive: received!.get(column.columnId)!.sensitive,
|
||||||
})));
|
})));
|
||||||
|
await onProgress?.(suggestions.length, suggestions.slice(-batch.length));
|
||||||
}
|
}
|
||||||
return suggestions;
|
return suggestions;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import type {
|
|||||||
SensitiveDataSuggestionRun,
|
SensitiveDataSuggestionRun,
|
||||||
SensitiveDataSuggestionScope,
|
SensitiveDataSuggestionScope,
|
||||||
} from "./types.js";
|
} from "./types.js";
|
||||||
|
import type { ModelCompletionUsage } from "./model-completer.js";
|
||||||
|
|
||||||
const interruptedMessage = "Sensitive-field suggestion generation was interrupted by backend restart.";
|
const interruptedMessage = "Sensitive-field suggestion generation was interrupted by backend restart.";
|
||||||
const failedMessage = "Sensitive-field suggestion generation failed.";
|
const failedMessage = "Sensitive-field suggestion generation failed.";
|
||||||
@@ -73,6 +74,31 @@ export class SensitiveDataSuggestionRunner {
|
|||||||
});
|
});
|
||||||
if (!prepared) throw new Error("Sensitive Data Suggestion Run disappeared");
|
if (!prepared) throw new Error("Sensitive Data Suggestion Run disappeared");
|
||||||
},
|
},
|
||||||
|
async (processed, batch) => {
|
||||||
|
const suggestedSensitive = batch.filter((suggestion) => suggestion.sensitive).length;
|
||||||
|
const suggestedNonSensitive = batch.length - suggestedSensitive;
|
||||||
|
const current = await this.repository.getSensitiveDataSuggestionRun(started.id);
|
||||||
|
if (!current) throw new Error("Sensitive Data Suggestion Run disappeared");
|
||||||
|
const progress = await this.repository.updateSensitiveDataSuggestionRun(started.id, {
|
||||||
|
suggestedSensitive: current.suggestedSensitive + suggestedSensitive,
|
||||||
|
suggestedNonSensitive: current.suggestedNonSensitive + suggestedNonSensitive,
|
||||||
|
});
|
||||||
|
if (!progress) throw new Error("Sensitive Data Suggestion Run disappeared");
|
||||||
|
await this.repository.appendSensitiveDataSuggestionEvent(
|
||||||
|
started.id,
|
||||||
|
"info",
|
||||||
|
`Classified ${processed} of ${progress.total} columns.`,
|
||||||
|
);
|
||||||
|
},
|
||||||
|
async (usage: ModelCompletionUsage) => {
|
||||||
|
const current = await this.repository.getSensitiveDataSuggestionRun(started.id);
|
||||||
|
if (!current) throw new Error("Sensitive Data Suggestion Run disappeared");
|
||||||
|
await this.repository.updateSensitiveDataSuggestionRun(started.id, {
|
||||||
|
inputTokens: current.inputTokens + usage.input,
|
||||||
|
cacheReadTokens: current.cacheReadTokens + usage.cacheRead,
|
||||||
|
outputTokens: current.outputTokens + usage.output,
|
||||||
|
});
|
||||||
|
},
|
||||||
);
|
);
|
||||||
const suggestedSensitive = suggestions.filter((suggestion) => suggestion.sensitive).length;
|
const suggestedSensitive = suggestions.filter((suggestion) => suggestion.sensitive).length;
|
||||||
const suggestedNonSensitive = suggestions.length - suggestedSensitive;
|
const suggestedNonSensitive = suggestions.length - suggestedSensitive;
|
||||||
|
|||||||
@@ -234,6 +234,9 @@ export interface DescriptionGenerationRun {
|
|||||||
generated: number;
|
generated: number;
|
||||||
nonGeneratable: number;
|
nonGeneratable: number;
|
||||||
failed: number;
|
failed: number;
|
||||||
|
inputTokens: number;
|
||||||
|
cacheReadTokens: number;
|
||||||
|
outputTokens: number;
|
||||||
createdAt: string;
|
createdAt: string;
|
||||||
startedAt: string | null;
|
startedAt: string | null;
|
||||||
updatedAt: string;
|
updatedAt: string;
|
||||||
@@ -250,6 +253,9 @@ export interface DescriptionGenerationRunUpdate {
|
|||||||
startedAt?: string | null;
|
startedAt?: string | null;
|
||||||
finishedAt?: string | null;
|
finishedAt?: string | null;
|
||||||
errorSummary?: string | null;
|
errorSummary?: string | null;
|
||||||
|
inputTokens?: number;
|
||||||
|
cacheReadTokens?: number;
|
||||||
|
outputTokens?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface DescriptionGenerationEvent {
|
export interface DescriptionGenerationEvent {
|
||||||
@@ -272,6 +278,9 @@ export interface SensitiveDataSuggestionRun {
|
|||||||
total: number;
|
total: number;
|
||||||
suggestedSensitive: number;
|
suggestedSensitive: number;
|
||||||
suggestedNonSensitive: number;
|
suggestedNonSensitive: number;
|
||||||
|
inputTokens: number;
|
||||||
|
cacheReadTokens: number;
|
||||||
|
outputTokens: number;
|
||||||
createdAt: string;
|
createdAt: string;
|
||||||
startedAt: string;
|
startedAt: string;
|
||||||
updatedAt: string;
|
updatedAt: string;
|
||||||
@@ -286,6 +295,9 @@ export interface SensitiveDataSuggestionRunUpdate {
|
|||||||
suggestedNonSensitive?: number;
|
suggestedNonSensitive?: number;
|
||||||
finishedAt?: string | null;
|
finishedAt?: string | null;
|
||||||
errorSummary?: string | null;
|
errorSummary?: string | null;
|
||||||
|
inputTokens?: number;
|
||||||
|
cacheReadTokens?: number;
|
||||||
|
outputTokens?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SensitiveDataSuggestionEvent {
|
export interface SensitiveDataSuggestionEvent {
|
||||||
|
|||||||
@@ -101,6 +101,9 @@ function publicRun(run: DescriptionGenerationRun) {
|
|||||||
generated: run.generated,
|
generated: run.generated,
|
||||||
nonGeneratable: run.nonGeneratable,
|
nonGeneratable: run.nonGeneratable,
|
||||||
failed: run.failed,
|
failed: run.failed,
|
||||||
|
inputTokens: run.inputTokens,
|
||||||
|
cacheReadTokens: run.cacheReadTokens,
|
||||||
|
outputTokens: run.outputTokens,
|
||||||
createdAt: run.createdAt,
|
createdAt: run.createdAt,
|
||||||
startedAt: run.startedAt,
|
startedAt: run.startedAt,
|
||||||
updatedAt: run.updatedAt,
|
updatedAt: run.updatedAt,
|
||||||
@@ -129,6 +132,9 @@ function publicSensitiveDataSuggestionRun(run: SensitiveDataSuggestionRun) {
|
|||||||
total: run.total,
|
total: run.total,
|
||||||
suggestedSensitive: run.suggestedSensitive,
|
suggestedSensitive: run.suggestedSensitive,
|
||||||
suggestedNonSensitive: run.suggestedNonSensitive,
|
suggestedNonSensitive: run.suggestedNonSensitive,
|
||||||
|
inputTokens: run.inputTokens,
|
||||||
|
cacheReadTokens: run.cacheReadTokens,
|
||||||
|
outputTokens: run.outputTokens,
|
||||||
createdAt: run.createdAt,
|
createdAt: run.createdAt,
|
||||||
startedAt: run.startedAt,
|
startedAt: run.startedAt,
|
||||||
updatedAt: run.updatedAt,
|
updatedAt: run.updatedAt,
|
||||||
|
|||||||
@@ -218,6 +218,12 @@ test("suggests sensitive flags from structural metadata without persisting them"
|
|||||||
runId: responseBody.run.id,
|
runId: responseBody.run.id,
|
||||||
sequence: 2,
|
sequence: 2,
|
||||||
level: "info",
|
level: "info",
|
||||||
|
message: "Classified 1 of 1 columns.",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
runId: responseBody.run.id,
|
||||||
|
sequence: 3,
|
||||||
|
level: "info",
|
||||||
message: "Sensitive-field suggestion generation completed for 1 column.",
|
message: "Sensitive-field suggestion generation completed for 1 column.",
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
@@ -671,6 +677,7 @@ test("generates one selected Catalog Column from a single JSON code fence", asyn
|
|||||||
errorSummary: null,
|
errorSummary: null,
|
||||||
});
|
});
|
||||||
expect(Object.keys(start.json()).sort()).toEqual([
|
expect(Object.keys(start.json()).sort()).toEqual([
|
||||||
|
"cacheReadTokens",
|
||||||
"createdAt",
|
"createdAt",
|
||||||
"databaseId",
|
"databaseId",
|
||||||
"errorSummary",
|
"errorSummary",
|
||||||
@@ -678,9 +685,11 @@ test("generates one selected Catalog Column from a single JSON code fence", asyn
|
|||||||
"finishedAt",
|
"finishedAt",
|
||||||
"generated",
|
"generated",
|
||||||
"id",
|
"id",
|
||||||
|
"inputTokens",
|
||||||
"language",
|
"language",
|
||||||
"modelId",
|
"modelId",
|
||||||
"nonGeneratable",
|
"nonGeneratable",
|
||||||
|
"outputTokens",
|
||||||
"processed",
|
"processed",
|
||||||
"scope",
|
"scope",
|
||||||
"startedAt",
|
"startedAt",
|
||||||
@@ -1272,7 +1281,7 @@ test("Stop aborts source sampling before any model request", async () => {
|
|||||||
test("an isolated exhausted technical batch failure allows completion with errors", async () => {
|
test("an isolated exhausted technical batch failure allows completion with errors", async () => {
|
||||||
const modelCompleter: ModelCompleter = {
|
const modelCompleter: ModelCompleter = {
|
||||||
complete: vi.fn(async (request) => {
|
complete: vi.fn(async (request) => {
|
||||||
if (vi.mocked(modelCompleter.complete).mock.calls.length === 1) {
|
if (vi.mocked(modelCompleter.complete).mock.calls.length <= 2) {
|
||||||
throw new ModelCompletionProviderError();
|
throw new ModelCompletionProviderError();
|
||||||
}
|
}
|
||||||
const context = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n"));
|
const context = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n"));
|
||||||
@@ -1320,7 +1329,7 @@ test("an isolated exhausted technical batch failure allows completion with error
|
|||||||
failed: 10,
|
failed: 10,
|
||||||
errorSummary: "Description generation completed with errors.",
|
errorSummary: "Description generation completed with errors.",
|
||||||
});
|
});
|
||||||
expect(modelCompleter.complete).toHaveBeenCalledTimes(2);
|
expect(modelCompleter.complete).toHaveBeenCalledTimes(3);
|
||||||
const updated = new Map(
|
const updated = new Map(
|
||||||
(await repository.listColumns(database.id, table.id)).map((column) => [column.id, column]),
|
(await repository.listColumns(database.id, table.id)).map((column) => [column.id, column]),
|
||||||
);
|
);
|
||||||
@@ -1346,10 +1355,10 @@ test("success resets the technical-failure streak and the third later failure st
|
|||||||
const modelCompleter: ModelCompleter = {
|
const modelCompleter: ModelCompleter = {
|
||||||
complete: vi.fn(async (request) => {
|
complete: vi.fn(async (request) => {
|
||||||
const call = vi.mocked(modelCompleter.complete).mock.calls.length;
|
const call = vi.mocked(modelCompleter.complete).mock.calls.length;
|
||||||
if ([1, 2, 4, 5, 6].includes(call)) {
|
if ([1, 2, 4, 5, 6, 7, 8, 9].includes(call)) {
|
||||||
throw Object.assign(new ModelCompletionProviderError(), { message: sensitiveDiagnostic });
|
throw Object.assign(new ModelCompletionProviderError(), { message: sensitiveDiagnostic });
|
||||||
}
|
}
|
||||||
if (call > 6) throw new Error("a later batch must not start");
|
if (call > 9) throw new Error("a later batch must not start");
|
||||||
const context = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n"));
|
const context = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n"));
|
||||||
return JSON.stringify({
|
return JSON.stringify({
|
||||||
results: context.targets.map((target: { targetId: string }) => ({
|
results: context.targets.map((target: { targetId: string }) => ({
|
||||||
@@ -1389,25 +1398,25 @@ test("success resets the technical-failure streak and the third later failure st
|
|||||||
expect(run).toMatchObject({
|
expect(run).toMatchObject({
|
||||||
status: "failed",
|
status: "failed",
|
||||||
total: 61,
|
total: 61,
|
||||||
processed: 60,
|
processed: 50,
|
||||||
generated: 10,
|
generated: 10,
|
||||||
nonGeneratable: 0,
|
nonGeneratable: 0,
|
||||||
failed: 50,
|
failed: 40,
|
||||||
errorSummary: "Description generation stopped after three consecutive technical batch failures.",
|
errorSummary: "Description generation stopped after three consecutive technical batch failures.",
|
||||||
});
|
});
|
||||||
expect(modelCompleter.complete).toHaveBeenCalledTimes(6);
|
expect(modelCompleter.complete).toHaveBeenCalledTimes(9);
|
||||||
const requests = vi.mocked(modelCompleter.complete).mock.calls.map(([request]) => request);
|
const requests = vi.mocked(modelCompleter.complete).mock.calls.map(([request]) => request);
|
||||||
expect(new Set(requests.map((request) => request.signal)).size).toBe(1);
|
expect(new Set(requests.map((request) => request.signal)).size).toBe(1);
|
||||||
expect(new Set(requests.map((request) => request.model.id))).toEqual(new Set([configuredModel.id]));
|
expect(new Set(requests.map((request) => request.model.id))).toEqual(new Set([configuredModel.id]));
|
||||||
const updated = new Map(
|
const updated = new Map(
|
||||||
(await repository.listColumns(database.id, table.id)).map((column) => [column.id, column]),
|
(await repository.listColumns(database.id, table.id)).map((column) => [column.id, column]),
|
||||||
);
|
);
|
||||||
targetIds.slice(20, 30).forEach((targetId) => {
|
targetIds.slice(10, 20).forEach((targetId) => {
|
||||||
expect(updated.get(targetId)?.generatedDescription).toBe("Successful reset batch.");
|
expect(updated.get(targetId)?.generatedDescription).toBe("Successful reset batch.");
|
||||||
});
|
});
|
||||||
expect(updated.get(targetIds[60]!)?.generatedDescription).toBeNull();
|
expect(updated.get(targetIds[50]!)?.generatedDescription).toBeNull();
|
||||||
const events = await repository.listDescriptionGenerationEvents(run.id);
|
const events = await repository.listDescriptionGenerationEvents(run.id);
|
||||||
expect(events.filter((event) => event.message.includes("model provider request failed"))).toHaveLength(5);
|
expect(events.filter((event) => event.message.includes("model provider request failed"))).toHaveLength(8);
|
||||||
expect(events.at(-1)).toEqual(expect.objectContaining({
|
expect(events.at(-1)).toEqual(expect.objectContaining({
|
||||||
level: "error",
|
level: "error",
|
||||||
message: "Description generation stopped after three consecutive technical batch failures.",
|
message: "Description generation stopped after three consecutive technical batch failures.",
|
||||||
@@ -1660,6 +1669,7 @@ test("Description Generation history is newest-first, bounded, and exposes only
|
|||||||
expect(response.statusCode).toBe(200);
|
expect(response.statusCode).toBe(200);
|
||||||
expect(response.json().map((run: { id: string }) => run.id)).toEqual(ids.slice(1).reverse());
|
expect(response.json().map((run: { id: string }) => run.id)).toEqual(ids.slice(1).reverse());
|
||||||
expect(Object.keys(response.json()[0]).sort()).toEqual([
|
expect(Object.keys(response.json()[0]).sort()).toEqual([
|
||||||
|
"cacheReadTokens",
|
||||||
"createdAt",
|
"createdAt",
|
||||||
"databaseId",
|
"databaseId",
|
||||||
"errorSummary",
|
"errorSummary",
|
||||||
@@ -1667,9 +1677,11 @@ test("Description Generation history is newest-first, bounded, and exposes only
|
|||||||
"finishedAt",
|
"finishedAt",
|
||||||
"generated",
|
"generated",
|
||||||
"id",
|
"id",
|
||||||
|
"inputTokens",
|
||||||
"language",
|
"language",
|
||||||
"modelId",
|
"modelId",
|
||||||
"nonGeneratable",
|
"nonGeneratable",
|
||||||
|
"outputTokens",
|
||||||
"processed",
|
"processed",
|
||||||
"scope",
|
"scope",
|
||||||
"startedAt",
|
"startedAt",
|
||||||
@@ -1945,7 +1957,7 @@ test("retains completed batch writes when a later batch response is malformed",
|
|||||||
});
|
});
|
||||||
const { run } = await waitForTerminalRun(app, start.json().id);
|
const { run } = await waitForTerminalRun(app, start.json().id);
|
||||||
|
|
||||||
expect(modelCompleter.complete).toHaveBeenCalledTimes(2);
|
expect(modelCompleter.complete).toHaveBeenCalledTimes(3);
|
||||||
expect(run).toMatchObject({
|
expect(run).toMatchObject({
|
||||||
status: "completed_with_errors",
|
status: "completed_with_errors",
|
||||||
total: 11,
|
total: 11,
|
||||||
@@ -1972,9 +1984,14 @@ test("retains completed batch writes when a later batch response is malformed",
|
|||||||
expect(events.slice(2, 12).map((event) => event.message)).toEqual(
|
expect(events.slice(2, 12).map((event) => event.message)).toEqual(
|
||||||
orderedIds.slice(0, 10).map((targetId) => `Generated description for Catalog Column ${targetId}.`),
|
orderedIds.slice(0, 10).map((targetId) => `Generated description for Catalog Column ${targetId}.`),
|
||||||
);
|
);
|
||||||
|
expect(events.find((event) => event.level === "warning" && event.message.includes("Retrying batch"))).toEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
message: "The model response did not match the required schema. Retrying batch (attempt 2 of 2).",
|
||||||
|
}),
|
||||||
|
);
|
||||||
expect(events.find((event) => event.level === "error")).toEqual(expect.objectContaining({
|
expect(events.find((event) => event.level === "error")).toEqual(expect.objectContaining({
|
||||||
level: "error",
|
level: "error",
|
||||||
message: `The model response was invalid. Affected Catalog Column target: ${orderedIds[10]}.`,
|
message: `The model response did not match the required schema. Affected Catalog Column target: ${orderedIds[10]}.`,
|
||||||
}));
|
}));
|
||||||
} finally {
|
} finally {
|
||||||
await app.close();
|
await app.close();
|
||||||
@@ -2186,7 +2203,7 @@ test("Generate Missing skips prior partial results and includes null, empty, and
|
|||||||
const metadata = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n"));
|
const metadata = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n"));
|
||||||
if (mode === "partial") {
|
if (mode === "partial") {
|
||||||
partialCall += 1;
|
partialCall += 1;
|
||||||
if (partialCall === 2) throw new ModelCompletionProviderError();
|
if (partialCall === 2 || partialCall === 3) throw new ModelCompletionProviderError();
|
||||||
return JSON.stringify({
|
return JSON.stringify({
|
||||||
results: metadata.targets.map((target: { targetId: string }) => ({
|
results: metadata.targets.map((target: { targetId: string }) => ({
|
||||||
targetId: target.targetId,
|
targetId: target.targetId,
|
||||||
@@ -2512,6 +2529,58 @@ test("localizes valid non-generatable Catalog Column results in English", async
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("retries invalid JSON once and completes the batch when the second response is valid", async () => {
|
||||||
|
const modelCompleter: ModelCompleter = {
|
||||||
|
complete: vi.fn(async (request) => {
|
||||||
|
if (vi.mocked(modelCompleter.complete).mock.calls.length === 1) {
|
||||||
|
return { content: "not-json", usage: { input: 11, cacheRead: 3, output: 2 } };
|
||||||
|
}
|
||||||
|
const context = JSON.parse(request.messages[1]!.content.split("\n").slice(1).join("\n"));
|
||||||
|
return {
|
||||||
|
content: JSON.stringify({
|
||||||
|
results: context.targets.map((target: { targetId: string }) => ({
|
||||||
|
targetId: target.targetId,
|
||||||
|
outcome: "generated",
|
||||||
|
description: "Generated after the application retry.",
|
||||||
|
})),
|
||||||
|
}),
|
||||||
|
usage: { input: 7, cacheRead: 1, output: 5 },
|
||||||
|
};
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
const { app, repository, database, table, column } = await setup(modelCompleter);
|
||||||
|
try {
|
||||||
|
const start = await app.inject({
|
||||||
|
method: "POST",
|
||||||
|
url: `/catalog/databases/${database.id}/description-generation-runs`,
|
||||||
|
payload: { modelId: configuredModel.id, scope: "selected_columns", targetIds: [column.id] },
|
||||||
|
});
|
||||||
|
const { run } = await waitForTerminalRun(app, start.json().id);
|
||||||
|
|
||||||
|
expect(run).toMatchObject({
|
||||||
|
status: "completed",
|
||||||
|
processed: 1,
|
||||||
|
generated: 1,
|
||||||
|
failed: 0,
|
||||||
|
inputTokens: 18,
|
||||||
|
cacheReadTokens: 4,
|
||||||
|
outputTokens: 7,
|
||||||
|
});
|
||||||
|
expect(modelCompleter.complete).toHaveBeenCalledTimes(2);
|
||||||
|
expect(await repository.getColumn(database.id, table.id, column.id)).toMatchObject({
|
||||||
|
generatedDescription: "Generated after the application retry.",
|
||||||
|
});
|
||||||
|
expect(await repository.listDescriptionGenerationEvents(run.id)).toEqual(expect.arrayContaining([
|
||||||
|
expect.objectContaining({
|
||||||
|
level: "warning",
|
||||||
|
message: "The model response was not valid JSON. Retrying batch (attempt 2 of 2).",
|
||||||
|
}),
|
||||||
|
]));
|
||||||
|
} finally {
|
||||||
|
await app.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test("fails safely when the provider fails and redacts provider diagnostics", async () => {
|
test("fails safely when the provider fails and redacts provider diagnostics", async () => {
|
||||||
const sensitiveDiagnostic = "test-provider-secret private prompt raw provider payload";
|
const sensitiveDiagnostic = "test-provider-secret private prompt raw provider payload";
|
||||||
const modelCompleter: ModelCompleter = {
|
const modelCompleter: ModelCompleter = {
|
||||||
@@ -2528,6 +2597,7 @@ test("fails safely when the provider fails and redacts provider diagnostics", as
|
|||||||
});
|
});
|
||||||
const { run } = await waitForTerminalRun(app, start.json().id);
|
const { run } = await waitForTerminalRun(app, start.json().id);
|
||||||
|
|
||||||
|
expect(modelCompleter.complete).toHaveBeenCalledTimes(2);
|
||||||
expect(run).toMatchObject({
|
expect(run).toMatchObject({
|
||||||
status: "completed_with_errors",
|
status: "completed_with_errors",
|
||||||
processed: 1,
|
processed: 1,
|
||||||
@@ -2565,7 +2635,7 @@ test.each([
|
|||||||
["duplicate mappings", (targetIds: readonly string[]) => ({ results: [
|
["duplicate mappings", (targetIds: readonly string[]) => ({ results: [
|
||||||
{ targetId: targetIds[0], outcome: "generated", description: "First valid value" },
|
{ targetId: targetIds[0], outcome: "generated", description: "First valid value" },
|
||||||
{ targetId: targetIds[0], outcome: "non_generatable" },
|
{ targetId: targetIds[0], outcome: "non_generatable" },
|
||||||
] })],
|
] }), "The model response was missing one or more requested targets."],
|
||||||
["unknown mappings", (targetIds: readonly string[]) => ({ results: [
|
["unknown mappings", (targetIds: readonly string[]) => ({ results: [
|
||||||
{ targetId: targetIds[0], outcome: "non_generatable" },
|
{ targetId: targetIds[0], outcome: "non_generatable" },
|
||||||
{
|
{
|
||||||
@@ -2573,15 +2643,15 @@ test.each([
|
|||||||
outcome: "generated",
|
outcome: "generated",
|
||||||
description: "Unknown target value",
|
description: "Unknown target value",
|
||||||
},
|
},
|
||||||
] })],
|
] }), "The model response was missing one or more requested targets."],
|
||||||
["missing mappings", (targetIds: readonly string[]) => ({ results: [
|
["missing mappings", (targetIds: readonly string[]) => ({ results: [
|
||||||
{ targetId: targetIds[0], outcome: "generated", description: "Only one result" },
|
{ targetId: targetIds[0], outcome: "generated", description: "Only one result" },
|
||||||
] })],
|
] }), "The model response was missing one or more requested targets."],
|
||||||
["malformed mappings", (targetIds: readonly string[]) => ({ results: [
|
["malformed mappings", (targetIds: readonly string[]) => ({ results: [
|
||||||
{ targetId: targetIds[0], outcome: "generated", description: "First valid value" },
|
{ targetId: targetIds[0], outcome: "generated", description: "First valid value" },
|
||||||
{ targetId: targetIds[1], outcome: "generated", description: " " },
|
{ targetId: targetIds[1], outcome: "generated", description: " " },
|
||||||
] })],
|
] }), "The model response did not match the required schema."],
|
||||||
] as const)("rejects %s without applying any result from the batch", async (_name, responseFor) => {
|
] as const)("rejects %s without applying any result from the batch", async (_name, responseFor, failureMessage) => {
|
||||||
let selectedColumnIds: string[] = [];
|
let selectedColumnIds: string[] = [];
|
||||||
const modelCompleter: ModelCompleter = {
|
const modelCompleter: ModelCompleter = {
|
||||||
complete: vi.fn(async () => JSON.stringify(responseFor(selectedColumnIds))),
|
complete: vi.fn(async () => JSON.stringify(responseFor(selectedColumnIds))),
|
||||||
@@ -2625,6 +2695,7 @@ test.each([
|
|||||||
});
|
});
|
||||||
const { run } = await waitForTerminalRun(app, start.json().id);
|
const { run } = await waitForTerminalRun(app, start.json().id);
|
||||||
|
|
||||||
|
expect(modelCompleter.complete).toHaveBeenCalledTimes(2);
|
||||||
expect(run).toMatchObject({
|
expect(run).toMatchObject({
|
||||||
status: "completed_with_errors",
|
status: "completed_with_errors",
|
||||||
processed: 2,
|
processed: 2,
|
||||||
@@ -2644,7 +2715,7 @@ test.each([
|
|||||||
expect((await repository.listDescriptionGenerationEvents(run.id)).find((event) => event.level === "error")).toEqual(
|
expect((await repository.listDescriptionGenerationEvents(run.id)).find((event) => event.level === "error")).toEqual(
|
||||||
expect.objectContaining({
|
expect.objectContaining({
|
||||||
level: "error",
|
level: "error",
|
||||||
message: `The model response was invalid. Affected Catalog Column targets: ${selectedColumnIds.join(", ")}.`,
|
message: `${failureMessage} Affected Catalog Column targets: ${selectedColumnIds.join(", ")}.`,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
} finally {
|
} finally {
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema
|
|||||||
import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js";
|
import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js";
|
||||||
import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js";
|
import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js";
|
||||||
import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js";
|
import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js";
|
||||||
|
import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js";
|
||||||
import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js";
|
import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js";
|
||||||
import { loadConfig } from "../src/config.js";
|
import { loadConfig } from "../src/config.js";
|
||||||
import type { WorkspaceRegistry } from "../src/workspaces/registry.js";
|
import type { WorkspaceRegistry } from "../src/workspaces/registry.js";
|
||||||
@@ -48,6 +49,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
|
|||||||
await upSensitiveDataFlag(db);
|
await upSensitiveDataFlag(db);
|
||||||
await upDescriptionGeneration(db);
|
await upDescriptionGeneration(db);
|
||||||
await upSensitiveSuggestionRuns(db);
|
await upSensitiveSuggestionRuns(db);
|
||||||
|
await upAiTokenUsage(db);
|
||||||
const repository = new KyselyCatalogRepository(db);
|
const repository = new KyselyCatalogRepository(db);
|
||||||
const database = await repository.create({
|
const database = await repository.create({
|
||||||
workspaceId: "psd-clinical",
|
workspaceId: "psd-clinical",
|
||||||
@@ -124,7 +126,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
|
|||||||
description: "Elenco dei pazienti e dei loro dati clinici.",
|
description: "Elenco dei pazienti e dei loro dati clinici.",
|
||||||
}] });
|
}] });
|
||||||
}
|
}
|
||||||
if (call === 4) {
|
if (call === 5) {
|
||||||
return JSON.stringify({ results: [
|
return JSON.stringify({ results: [
|
||||||
{
|
{
|
||||||
targetId: birthDate.id,
|
targetId: birthDate.id,
|
||||||
@@ -138,14 +140,14 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
|
|||||||
},
|
},
|
||||||
] });
|
] });
|
||||||
}
|
}
|
||||||
if (call === 5) {
|
if (call === 6) {
|
||||||
return JSON.stringify({ results: [{
|
return JSON.stringify({ results: [{
|
||||||
targetId: table.id,
|
targetId: table.id,
|
||||||
outcome: "generated",
|
outcome: "generated",
|
||||||
description: "Descrizione rigenerata della tabella pazienti.",
|
description: "Descrizione rigenerata della tabella pazienti.",
|
||||||
}] });
|
}] });
|
||||||
}
|
}
|
||||||
if (call === 6) {
|
if (call === 7) {
|
||||||
return JSON.stringify({ results: [{
|
return JSON.stringify({ results: [{
|
||||||
targetId: birthDate.id,
|
targetId: birthDate.id,
|
||||||
outcome: "generated",
|
outcome: "generated",
|
||||||
@@ -340,7 +342,7 @@ test.skipIf(!dockerAvailable)("Fastify persists Description Generation success a
|
|||||||
expect(await repository.getColumn(database.id, table.id, birthDate.id)).toMatchObject({
|
expect(await repository.getColumn(database.id, table.id, birthDate.id)).toMatchObject({
|
||||||
generatedDescription: "Descrizione recuperata della data di nascita.",
|
generatedDescription: "Descrizione recuperata della data di nascita.",
|
||||||
});
|
});
|
||||||
expect(modelCompleter.complete).toHaveBeenCalledTimes(6);
|
expect(modelCompleter.complete).toHaveBeenCalledTimes(7);
|
||||||
expect(JSON.stringify(vi.mocked(modelCompleter.complete).mock.calls)).toContain(persistedSampleSecret);
|
expect(JSON.stringify(vi.mocked(modelCompleter.complete).mock.calls)).toContain(persistedSampleSecret);
|
||||||
|
|
||||||
const runIds = [
|
const runIds = [
|
||||||
|
|||||||
@@ -1,6 +1,9 @@
|
|||||||
import { expect, test, vi } from "vitest";
|
import { expect, test, vi } from "vitest";
|
||||||
|
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
import {
|
import {
|
||||||
PostgresDescriptionSourceSampler,
|
ConcreteDescriptionSourceSampler,
|
||||||
type DescriptionSourceSamplingTarget,
|
type DescriptionSourceSamplingTarget,
|
||||||
} from "../src/catalog/description-source-sampler.js";
|
} from "../src/catalog/description-source-sampler.js";
|
||||||
import type {
|
import type {
|
||||||
@@ -8,6 +11,8 @@ import type {
|
|||||||
CatalogPostgresAccess,
|
CatalogPostgresAccess,
|
||||||
} from "../src/catalog/postgres-access.js";
|
} from "../src/catalog/postgres-access.js";
|
||||||
import type { WorkspaceDatabase } from "../src/catalog/types.js";
|
import type { WorkspaceDatabase } from "../src/catalog/types.js";
|
||||||
|
import type { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||||
|
import { CATALOG_SECRET_IDS } from "../src/catalog/secrets.js";
|
||||||
|
|
||||||
const database: WorkspaceDatabase = {
|
const database: WorkspaceDatabase = {
|
||||||
id: "11111111-1111-4111-8111-111111111111",
|
id: "11111111-1111-4111-8111-111111111111",
|
||||||
@@ -51,7 +56,7 @@ test("samples at most five source rows and five distinct non-null examples in a
|
|||||||
const access: CatalogPostgresAccess = {
|
const access: CatalogPostgresAccess = {
|
||||||
connect: vi.fn(async () => ({ query, end }) as CatalogDatabaseClient),
|
connect: vi.fn(async () => ({ query, end }) as CatalogDatabaseClient),
|
||||||
};
|
};
|
||||||
const sampler = new PostgresDescriptionSourceSampler(access);
|
const sampler = new ConcreteDescriptionSourceSampler(access);
|
||||||
const controller = new AbortController();
|
const controller = new AbortController();
|
||||||
|
|
||||||
const samples = await sampler.sample(database, [target], controller.signal);
|
const samples = await sampler.sample(database, [target], controller.signal);
|
||||||
@@ -92,13 +97,71 @@ test("samples at most five source rows and five distinct non-null examples in a
|
|||||||
expect(end).toHaveBeenCalledOnce();
|
expect(end).toHaveBeenCalledOnce();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("samples source rows through the configured REST run_query binding", async () => {
|
||||||
|
const root = mkdtempSync(join(tmpdir(), "tht-source-rest-"));
|
||||||
|
const credentialFile = join(root, "api-key");
|
||||||
|
writeFileSync(credentialFile, "test-api-key\n", { mode: 0o600 });
|
||||||
|
const release = vi.fn();
|
||||||
|
const secretStore = {
|
||||||
|
materialize: vi.fn(() => ({
|
||||||
|
files: new Map([[CATALOG_SECRET_IDS.apiKey, credentialFile]]),
|
||||||
|
release,
|
||||||
|
})),
|
||||||
|
} as unknown as WorkspaceSecretStore;
|
||||||
|
const fetchMock = vi.fn(async () => new Response(JSON.stringify([
|
||||||
|
{ 'status"code': "active", ward: null },
|
||||||
|
{ 'status"code': "pending", ward: "A" },
|
||||||
|
]), { status: 200, headers: { "content-type": "application/json" } }));
|
||||||
|
vi.stubGlobal("fetch", fetchMock);
|
||||||
|
const access: CatalogPostgresAccess = {
|
||||||
|
connect: vi.fn(async () => { throw new Error("PostgreSQL access must not be used"); }),
|
||||||
|
};
|
||||||
|
const sampler = new ConcreteDescriptionSourceSampler(access, secretStore);
|
||||||
|
const restDatabase: WorkspaceDatabase = {
|
||||||
|
...database,
|
||||||
|
binding: {
|
||||||
|
transport: "rest_api",
|
||||||
|
baseUrl: "https://dwh.example.test/root/",
|
||||||
|
restPath: "/health",
|
||||||
|
restAuth: "x-api-key",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
await expect(sampler.sample(restDatabase, [target], new AbortController().signal)).resolves.toEqual([{
|
||||||
|
targetId: target.targetId,
|
||||||
|
tableName: target.tableName,
|
||||||
|
rows: [
|
||||||
|
{ fields: [{ name: 'status"code', value: "active" }, { name: "ward", value: null }] },
|
||||||
|
{ fields: [{ name: 'status"code', value: "pending" }, { name: "ward", value: "A" }] },
|
||||||
|
],
|
||||||
|
representativeValues: [
|
||||||
|
{ column: 'status"code', values: ["active", "pending"] },
|
||||||
|
{ column: "ward", values: ["A"] },
|
||||||
|
],
|
||||||
|
}]);
|
||||||
|
expect(access.connect).not.toHaveBeenCalled();
|
||||||
|
expect(fetchMock).toHaveBeenCalledWith("https://dwh.example.test/root/rpc/run_query", expect.objectContaining({
|
||||||
|
method: "POST",
|
||||||
|
headers: { "content-type": "application/json", "x-api-key": "test-api-key" },
|
||||||
|
body: JSON.stringify({
|
||||||
|
query_text: 'SELECT LEFT(("status""code")::text, 256) AS "status""code", LEFT(("ward")::text, 256) AS "ward" FROM "clinical""data"."patient""facts" LIMIT 5',
|
||||||
|
}),
|
||||||
|
}));
|
||||||
|
expect(release).toHaveBeenCalledOnce();
|
||||||
|
} finally {
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test("does not issue a SELECT when a protected target has no source columns", async () => {
|
test("does not issue a SELECT when a protected target has no source columns", async () => {
|
||||||
const query = vi.fn(async () => ({ rows: [] }));
|
const query = vi.fn(async () => ({ rows: [] }));
|
||||||
const end = vi.fn(async () => undefined);
|
const end = vi.fn(async () => undefined);
|
||||||
const access: CatalogPostgresAccess = {
|
const access: CatalogPostgresAccess = {
|
||||||
connect: vi.fn(async () => ({ query, end }) as CatalogDatabaseClient),
|
connect: vi.fn(async () => ({ query, end }) as CatalogDatabaseClient),
|
||||||
};
|
};
|
||||||
const sampler = new PostgresDescriptionSourceSampler(access);
|
const sampler = new ConcreteDescriptionSourceSampler(access);
|
||||||
|
|
||||||
const samples = await sampler.sample(database, [{
|
const samples = await sampler.sample(database, [{
|
||||||
targetId: target.targetId,
|
targetId: target.targetId,
|
||||||
@@ -129,7 +192,7 @@ test("rolls back and closes the source connection when sampling fails", async ()
|
|||||||
const access: CatalogPostgresAccess = {
|
const access: CatalogPostgresAccess = {
|
||||||
connect: vi.fn(async () => ({ query, end }) as CatalogDatabaseClient),
|
connect: vi.fn(async () => ({ query, end }) as CatalogDatabaseClient),
|
||||||
};
|
};
|
||||||
const sampler = new PostgresDescriptionSourceSampler(access);
|
const sampler = new ConcreteDescriptionSourceSampler(access);
|
||||||
const controller = new AbortController();
|
const controller = new AbortController();
|
||||||
|
|
||||||
await expect(sampler.sample(database, [target], controller.signal)).rejects.toThrow();
|
await expect(sampler.sample(database, [target], controller.signal)).rejects.toThrow();
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_des
|
|||||||
import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js";
|
import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js";
|
||||||
import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js";
|
import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js";
|
||||||
import { up as upLogicalRelationships } from "../src/catalog/migrations/008_catalog_logical_relationships.js";
|
import { up as upLogicalRelationships } from "../src/catalog/migrations/008_catalog_logical_relationships.js";
|
||||||
|
import { up as upAiTokenUsage } from "../src/catalog/migrations/009_ai_token_usage.js";
|
||||||
|
|
||||||
const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
|
const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
|
||||||
|
|
||||||
@@ -28,6 +29,9 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
|
|||||||
await upSchemaSync(db);
|
await upSchemaSync(db);
|
||||||
await upSensitiveDataFlag(db);
|
await upSensitiveDataFlag(db);
|
||||||
await upLogicalRelationships(db);
|
await upLogicalRelationships(db);
|
||||||
|
await upDescriptionGeneration(db);
|
||||||
|
await upSensitiveSuggestionRuns(db);
|
||||||
|
await upAiTokenUsage(db);
|
||||||
await sql`CREATE ROLE thothii_catalog_runtime`.execute(db);
|
await sql`CREATE ROLE thothii_catalog_runtime`.execute(db);
|
||||||
await upRuntimeSequencePrivileges(db);
|
await upRuntimeSequencePrivileges(db);
|
||||||
const sequencePrivilege = await sql<{ allowed: boolean }>`
|
const sequencePrivilege = await sql<{ allowed: boolean }>`
|
||||||
@@ -386,6 +390,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
|
|||||||
await upLogicalRelationships(db);
|
await upLogicalRelationships(db);
|
||||||
await upDescriptionGeneration(db);
|
await upDescriptionGeneration(db);
|
||||||
await upSensitiveSuggestionRuns(db);
|
await upSensitiveSuggestionRuns(db);
|
||||||
|
await upAiTokenUsage(db);
|
||||||
const repository = new KyselyCatalogRepository(db);
|
const repository = new KyselyCatalogRepository(db);
|
||||||
const firstDatabase = await repository.create({
|
const firstDatabase = await repository.create({
|
||||||
workspaceId: "generation-one",
|
workspaceId: "generation-one",
|
||||||
|
|||||||
@@ -127,7 +127,7 @@ test("resolves only a configured selection for the later generation boundary", (
|
|||||||
expect(() => models.resolve("unknown-model")).toThrow(MetadataGenerationModelUnavailableError);
|
expect(() => models.resolve("unknown-model")).toThrow(MetadataGenerationModelUnavailableError);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("loads DeepSeek, GLM, and an explicit keyless Qwen endpoint from installation setup", () => {
|
test("loads DeepSeek models, GLM, and an explicit keyless Qwen endpoint from installation setup", () => {
|
||||||
const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration:
|
const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration:
|
||||||
default: glm-53
|
default: glm-53
|
||||||
models:
|
models:
|
||||||
@@ -135,6 +135,10 @@ test("loads DeepSeek, GLM, and an explicit keyless Qwen endpoint from installati
|
|||||||
label: DeepSeek V4 Pro
|
label: DeepSeek V4 Pro
|
||||||
litellm: {provider: deepseek, model: deepseek-v4-pro}
|
litellm: {provider: deepseek, model: deepseek-v4-pro}
|
||||||
apiKeyEnv: DEEPSEEK_API_KEY
|
apiKeyEnv: DEEPSEEK_API_KEY
|
||||||
|
- id: deepseek-v4-flash
|
||||||
|
label: DeepSeek V4 Flash
|
||||||
|
litellm: {provider: deepseek, model: deepseek-v4-flash}
|
||||||
|
apiKeyEnv: DEEPSEEK_API_KEY
|
||||||
- id: glm-53
|
- id: glm-53
|
||||||
label: GLM 5.3
|
label: GLM 5.3
|
||||||
litellm:
|
litellm:
|
||||||
@@ -156,6 +160,7 @@ test("loads DeepSeek, GLM, and an explicit keyless Qwen endpoint from installati
|
|||||||
expect(models.catalog()).toEqual({
|
expect(models.catalog()).toEqual({
|
||||||
models: [
|
models: [
|
||||||
{ id: "deepseek-v4-pro", label: "DeepSeek V4 Pro" },
|
{ id: "deepseek-v4-pro", label: "DeepSeek V4 Pro" },
|
||||||
|
{ id: "deepseek-v4-flash", label: "DeepSeek V4 Flash" },
|
||||||
{ id: "glm-53", label: "GLM 5.3" },
|
{ id: "glm-53", label: "GLM 5.3" },
|
||||||
{ id: "qwen-36", label: "Qwen 3.6" },
|
{ id: "qwen-36", label: "Qwen 3.6" },
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -64,7 +64,7 @@ sys.stdout.write(json.dumps({"ok": True, "content": "Descrizione italiana"}))
|
|||||||
signal: new AbortController().signal,
|
signal: new AbortController().signal,
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(content).toBe("Descrizione italiana");
|
expect(content).toEqual({ content: "Descrizione italiana", usage: { input: 0, cacheRead: 0, output: 0 } });
|
||||||
const captured = JSON.parse(readFileSync(join(roots[0]!, "request.json"), "utf8"));
|
const captured = JSON.parse(readFileSync(join(roots[0]!, "request.json"), "utf8"));
|
||||||
expect(captured.request).toEqual({
|
expect(captured.request).toEqual({
|
||||||
model: "openai/gpt-4.1-mini",
|
model: "openai/gpt-4.1-mini",
|
||||||
@@ -100,7 +100,7 @@ sys.stdout.write(json.dumps({"ok": True, "content": "Descrizione Qwen"}))
|
|||||||
},
|
},
|
||||||
messages: [{ role: "user", content: "Describe invented metadata." }],
|
messages: [{ role: "user", content: "Describe invented metadata." }],
|
||||||
signal: new AbortController().signal,
|
signal: new AbortController().signal,
|
||||||
})).resolves.toBe("Descrizione Qwen");
|
})).resolves.toEqual({ content: "Descrizione Qwen", usage: { input: 0, cacheRead: 0, output: 0 } });
|
||||||
|
|
||||||
expect(JSON.parse(readFileSync(join(roots[0]!, "request.json"), "utf8"))).toEqual({
|
expect(JSON.parse(readFileSync(join(roots[0]!, "request.json"), "utf8"))).toEqual({
|
||||||
model: "openai/qwen3.6-35b-a3b",
|
model: "openai/qwen3.6-35b-a3b",
|
||||||
|
|||||||
@@ -17,6 +17,12 @@ metadataGeneration:
|
|||||||
provider: deepseek
|
provider: deepseek
|
||||||
model: deepseek-v4-pro
|
model: deepseek-v4-pro
|
||||||
apiKeyEnv: DEEPSEEK_API_KEY
|
apiKeyEnv: DEEPSEEK_API_KEY
|
||||||
|
- id: deepseek-v4-flash
|
||||||
|
label: DeepSeek V4 Flash
|
||||||
|
litellm:
|
||||||
|
provider: deepseek
|
||||||
|
model: deepseek-v4-flash
|
||||||
|
apiKeyEnv: DEEPSEEK_API_KEY
|
||||||
- id: glm-53
|
- id: glm-53
|
||||||
label: GLM 5.3
|
label: GLM 5.3
|
||||||
litellm:
|
litellm:
|
||||||
|
|||||||
@@ -108,6 +108,7 @@ not supported by this installation until the declarative contract is extended ge
|
|||||||
| Model | Level | Why | Visibility |
|
| Model | Level | Why | Visibility |
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| `deepseek/deepseek-v4-pro` | Built-in Pi | Known public API, already in the build | All projects |
|
| `deepseek/deepseek-v4-pro` | Built-in Pi | Known public API, already in the build | All projects |
|
||||||
|
| `deepseek/deepseek-v4-flash` | Built-in Pi | Known public API, already in the build | All projects |
|
||||||
| `zai/glm-5.3` | `deploy/pi/models.json` | Custom OpenAI-compatible endpoint | ThothII installation |
|
| `zai/glm-5.3` | `deploy/pi/models.json` | Custom OpenAI-compatible endpoint | ThothII installation |
|
||||||
| `local-qwen/qwen3.6-35b-a3b` | `deploy/pi/models.json` | Locally configured OpenAI-compatible endpoint | ThothII installation |
|
| `local-qwen/qwen3.6-35b-a3b` | `deploy/pi/models.json` | Locally configured OpenAI-compatible endpoint | ThothII installation |
|
||||||
|
|
||||||
|
|||||||
@@ -179,12 +179,12 @@ canonico; i percorsi e i comportamenti descrivono il sorgente disponibile il 202
|
|||||||
e Schema Sync. Il renderer e il runtime delle sessioni NL→SQL restano fuori scope e continuano a
|
e Schema Sync. Il renderer e il runtime delle sessioni NL→SQL restano fuori scope e continuano a
|
||||||
rifiutarla finché non verrà deciso il relativo cutover.
|
rifiutarla finché non verrà deciso il relativo cutover.
|
||||||
55. I menu di azione a livello Workspace Database espongono separatamente `Synchronize tables`,
|
55. I menu di azione a livello Workspace Database espongono separatamente `Synchronize tables`,
|
||||||
`Synchronize all columns`, `Synchronize relationships` e `Synchronize all`. Su una selezione di
|
`Synchronize relationships` e `Synchronize all`. Su una selezione di
|
||||||
database lo scope scelto viene avviato per ogni database idoneo; non viene sostituito
|
database lo scope scelto viene avviato per ogni database idoneo; non viene sostituito
|
||||||
implicitamente con una sincronizzazione completa.
|
implicitamente con una sincronizzazione completa.
|
||||||
56. Per lo scope Columns, `tableIds` vuoto significa tutte le Catalog Table correnti del Workspace
|
56. Lo scope Columns è disponibile dalla grid Tables e limita la riconciliazione alle tabelle
|
||||||
Database; `tableIds` valorizzato limita invece la riconciliazione alle tabelle indicate. La grid
|
selezionate; la pagina Columns non espone azioni di sincronizzazione. La grid Tables espone
|
||||||
Tables espone `Synchronize columns` sulle tabelle selezionate.
|
`Synchronize columns` sulle tabelle selezionate.
|
||||||
|
|
||||||
## Correzione del modello mentale corrente
|
## Correzione del modello mentale corrente
|
||||||
|
|
||||||
@@ -570,9 +570,9 @@ Le griglie che dispongono di azioni massive usano checkbox e una toolbar contest
|
|||||||
menu `Actions` e cancellazione della selezione. La selezione identifica ID espliciti, può essere
|
menu `Actions` e cancellazione della selezione. La selezione identifica ID espliciti, può essere
|
||||||
accumulata attraverso i filtri e viene azzerata dopo successo, nuova sincronizzazione o uscita
|
accumulata attraverso i filtri e viene azzerata dopo successo, nuova sincronizzazione o uscita
|
||||||
dalla pagina; un'azione è all-or-nothing se un elemento non è idoneo. I menu a livello database
|
dalla pagina; un'azione è all-or-nothing se un elemento non è idoneo. I menu a livello database
|
||||||
espongono gli scope fisici come azioni distinte: `Synchronize tables`, `Synchronize all columns`,
|
espongono gli scope fisici come azioni distinte: `Synchronize tables`, `Synchronize relationships`
|
||||||
`Synchronize relationships` e `Synchronize all`. La grid Tables espone invece `Synchronize
|
e `Synchronize all`. La grid Tables espone invece `Synchronize columns` per le tabelle selezionate;
|
||||||
columns` per le tabelle selezionate. Le selezioni database aggiungono `Delete all tables` e
|
la pagina Columns non espone sincronizzazione. Le selezioni database aggiungono `Delete all tables` e
|
||||||
`Delete all relationships`; le selezioni tabelle aggiungono `Delete all columns` e `Delete all
|
`Delete all relationships`; le selezioni tabelle aggiungono `Delete all columns` e `Delete all
|
||||||
relationships`. Queste operazioni sono atomiche, richiedono conferma e non modificano database
|
relationships`. Queste operazioni sono atomiche, richiedono conferma e non modificano database
|
||||||
esterno, binding, configurazione o segreti. Test connection resta un'azione distinta; griglie senza
|
esterno, binding, configurazione o segreti. Test connection resta un'azione distinta; griglie senza
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ The installation-level model catalog contained these entries, with `glm-53` as t
|
|||||||
| Selection ID | LiteLLM route | Authentication |
|
| Selection ID | LiteLLM route | Authentication |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `deepseek-v4-pro` | `deepseek/deepseek-v4-pro` | Protected `DEEPSEEK_API_KEY` value already used by core |
|
| `deepseek-v4-pro` | `deepseek/deepseek-v4-pro` | Protected `DEEPSEEK_API_KEY` value already used by core |
|
||||||
|
| `deepseek-v4-flash` | `deepseek/deepseek-v4-flash` | Protected `DEEPSEEK_API_KEY` value already used by core |
|
||||||
| `glm-53` | `openai/glm-5.3` on the Z.AI coding endpoint | Protected `ZAI_API_KEY` value already used by core |
|
| `glm-53` | `openai/glm-5.3` on the Z.AI coding endpoint | Protected `ZAI_API_KEY` value already used by core |
|
||||||
| `qwen-36` | `openai/qwen3.6-35b-a3b` on the VPN-only AritmoLab endpoint | No operator API key |
|
| `qwen-36` | `openai/qwen3.6-35b-a3b` on the VPN-only AritmoLab endpoint | No operator API key |
|
||||||
|
|
||||||
|
|||||||
@@ -72,6 +72,28 @@ const table = {
|
|||||||
updatedAt: "2026-08-27T09:00:00Z",
|
updatedAt: "2026-08-27T09:00:00Z",
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const descriptionRun = {
|
||||||
|
id: "77777777-7777-4777-8777-777777777777",
|
||||||
|
databaseId,
|
||||||
|
scope: "selected_columns",
|
||||||
|
modelId: "glm-53",
|
||||||
|
language: "it",
|
||||||
|
status: "completed",
|
||||||
|
total: 5,
|
||||||
|
processed: 5,
|
||||||
|
generated: 4,
|
||||||
|
nonGeneratable: 1,
|
||||||
|
failed: 0,
|
||||||
|
inputTokens: 1200,
|
||||||
|
cacheReadTokens: 80,
|
||||||
|
outputTokens: 420,
|
||||||
|
createdAt: "2026-08-31T09:00:00Z",
|
||||||
|
startedAt: "2026-08-31T09:00:01Z",
|
||||||
|
updatedAt: "2026-08-31T09:00:04Z",
|
||||||
|
finishedAt: "2026-08-31T09:00:04Z",
|
||||||
|
errorSummary: null,
|
||||||
|
};
|
||||||
|
|
||||||
function metrics(requestedDatabaseId: string | null) {
|
function metrics(requestedDatabaseId: string | null) {
|
||||||
return {
|
return {
|
||||||
scope: requestedDatabaseId ? "database" : "global",
|
scope: requestedDatabaseId ? "database" : "global",
|
||||||
@@ -133,7 +155,8 @@ async function expectContextPanelGeometry(page: Page, accessibleName: string) {
|
|||||||
const managerCenter = managerBox.x + managerBox.width / 2;
|
const managerCenter = managerBox.x + managerBox.width / 2;
|
||||||
const panelCenter = panelBox.x + panelBox.width / 2;
|
const panelCenter = panelBox.x + panelBox.width / 2;
|
||||||
expect(Math.abs(panelCenter - managerCenter)).toBeLessThanOrEqual(1);
|
expect(Math.abs(panelCenter - managerCenter)).toBeLessThanOrEqual(1);
|
||||||
expect(Math.abs(panelBox.width / managerBox.width - 0.6)).toBeLessThanOrEqual(0.005);
|
const expectedPanelWidth = Math.min(1200, managerBox.width - (managerBox.width <= 768 ? 24 : 32));
|
||||||
|
expect(Math.abs(panelBox.width - expectedPanelWidth)).toBeLessThanOrEqual(1);
|
||||||
expect(panelBox.x).toBeGreaterThanOrEqual(managerBox.x - 1);
|
expect(panelBox.x).toBeGreaterThanOrEqual(managerBox.x - 1);
|
||||||
expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(managerBox.x + managerBox.width + 1);
|
expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(managerBox.x + managerBox.width + 1);
|
||||||
expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(sessionRailBox.x + 1);
|
expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(sessionRailBox.x + 1);
|
||||||
@@ -297,7 +320,8 @@ async function expectWorkAreaPanelGeometry(page: Page, accessibleName: string) {
|
|||||||
expect(Math.abs(
|
expect(Math.abs(
|
||||||
panelBox.x + panelBox.width / 2 - (workAreaBox.x + workAreaBox.width / 2),
|
panelBox.x + panelBox.width / 2 - (workAreaBox.x + workAreaBox.width / 2),
|
||||||
)).toBeLessThanOrEqual(1);
|
)).toBeLessThanOrEqual(1);
|
||||||
expect(Math.abs(panelBox.width / workAreaBox.width - 0.6)).toBeLessThanOrEqual(0.005);
|
const expectedPanelWidth = Math.min(1200, workAreaBox.width - (workAreaBox.width <= 768 ? 24 : 32));
|
||||||
|
expect(Math.abs(panelBox.width - expectedPanelWidth)).toBeLessThanOrEqual(1);
|
||||||
expect(panelBox.x).toBeGreaterThanOrEqual(workAreaBox.x - 1);
|
expect(panelBox.x).toBeGreaterThanOrEqual(workAreaBox.x - 1);
|
||||||
expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(workAreaBox.x + workAreaBox.width + 1);
|
expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(workAreaBox.x + workAreaBox.width + 1);
|
||||||
expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(sessionRailBox.x + 1);
|
expect(panelBox.x + panelBox.width).toBeLessThanOrEqual(sessionRailBox.x + 1);
|
||||||
@@ -344,12 +368,20 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb
|
|||||||
"GET /api/catalog/metadata-generation/models": {
|
"GET /api/catalog/metadata-generation/models": {
|
||||||
models: [
|
models: [
|
||||||
{ id: "deepseek-v4-pro", label: "DeepSeek V4 Pro" },
|
{ id: "deepseek-v4-pro", label: "DeepSeek V4 Pro" },
|
||||||
|
{ id: "deepseek-v4-flash", label: "DeepSeek V4 Flash" },
|
||||||
{ id: "glm-53", label: "GLM 5.3" },
|
{ id: "glm-53", label: "GLM 5.3" },
|
||||||
{ id: "qwen-36", label: "AritmoLab Qwen 3.6 35B A3B" },
|
{ id: "qwen-36", label: "AritmoLab Qwen 3.6 35B A3B" },
|
||||||
],
|
],
|
||||||
default: "glm-53",
|
default: "glm-53",
|
||||||
},
|
},
|
||||||
"GET /api/catalog/description-generation-runs?limit=50": [],
|
"GET /api/catalog/description-generation-runs?limit=50": [descriptionRun],
|
||||||
|
[`GET /api/catalog/description-generation-runs/${descriptionRun.id}`]: descriptionRun,
|
||||||
|
[`GET /api/catalog/description-generation-runs/${descriptionRun.id}/events-list?after=0`]: [{
|
||||||
|
sequence: 1,
|
||||||
|
level: "info",
|
||||||
|
message: "Description generation completed.",
|
||||||
|
createdAt: "2026-08-31T09:00:04Z",
|
||||||
|
}],
|
||||||
"GET /api/catalog/sensitive-data-suggestion-runs?limit=50": [],
|
"GET /api/catalog/sensitive-data-suggestion-runs?limit=50": [],
|
||||||
[`GET /api/catalog/databases/${databaseId}/tables`]: [table],
|
[`GET /api/catalog/databases/${databaseId}/tables`]: [table],
|
||||||
};
|
};
|
||||||
@@ -387,6 +419,7 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb
|
|||||||
|
|
||||||
const adminNavigationRail = page.getByRole("complementary", { name: "Session navigation" });
|
const adminNavigationRail = page.getByRole("complementary", { name: "Session navigation" });
|
||||||
const administration = adminNavigationRail.getByRole("button", { name: "Administration", exact: true });
|
const administration = adminNavigationRail.getByRole("button", { name: "Administration", exact: true });
|
||||||
|
await expect(administration).toHaveCSS("font-family", /Manrope/);
|
||||||
await expect(administration).toHaveAttribute("aria-expanded", "false");
|
await expect(administration).toHaveAttribute("aria-expanded", "false");
|
||||||
await expect(adminNavigationRail.getByRole("region", { name: "Administration" })).toHaveCount(0);
|
await expect(adminNavigationRail.getByRole("region", { name: "Administration" })).toHaveCount(0);
|
||||||
await administration.click();
|
await administration.click();
|
||||||
@@ -435,6 +468,7 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb
|
|||||||
await expect(metadataModelSelector).toHaveValue("glm-53");
|
await expect(metadataModelSelector).toHaveValue("glm-53");
|
||||||
await expect(metadataModelSelector.locator("option")).toHaveText([
|
await expect(metadataModelSelector.locator("option")).toHaveText([
|
||||||
"DeepSeek V4 Pro",
|
"DeepSeek V4 Pro",
|
||||||
|
"DeepSeek V4 Flash",
|
||||||
"GLM 5.3",
|
"GLM 5.3",
|
||||||
"AritmoLab Qwen 3.6 35B A3B",
|
"AritmoLab Qwen 3.6 35B A3B",
|
||||||
]);
|
]);
|
||||||
@@ -488,10 +522,55 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb
|
|||||||
.click();
|
.click();
|
||||||
await expectContextPanelGeometry(page, "Description generation");
|
await expectContextPanelGeometry(page, "Description generation");
|
||||||
await expectContextPanelHeaderUsesPrimary(page, "Description generation");
|
await expectContextPanelHeaderUsesPrimary(page, "Description generation");
|
||||||
await page
|
const descriptionPanel = page.getByRole("dialog", { name: "Description generation" });
|
||||||
.getByRole("dialog", { name: "Description generation" })
|
const descriptionBody = descriptionPanel.locator(
|
||||||
.getByRole("button", { name: /close/i })
|
':scope > [data-catalog-panel-region="body"]',
|
||||||
.click();
|
);
|
||||||
|
const progressTable = descriptionPanel.getByRole("table", {
|
||||||
|
name: "Description generation progress",
|
||||||
|
});
|
||||||
|
const eventLog = descriptionPanel.getByRole("log", {
|
||||||
|
name: "Description generation events",
|
||||||
|
});
|
||||||
|
const recentRuns = descriptionPanel.getByRole("region", {
|
||||||
|
name: "Description generation history",
|
||||||
|
});
|
||||||
|
await expect(progressTable.getByRole("columnheader")).toHaveCount(5);
|
||||||
|
await expect(eventLog).toBeVisible();
|
||||||
|
await expect(recentRuns).toBeVisible();
|
||||||
|
|
||||||
|
const [bodyBox, eventLogBox, recentRunsBox] = await Promise.all([
|
||||||
|
descriptionBody.boundingBox(),
|
||||||
|
eventLog.boundingBox(),
|
||||||
|
recentRuns.boundingBox(),
|
||||||
|
]);
|
||||||
|
expect(bodyBox).not.toBeNull();
|
||||||
|
expect(eventLogBox).not.toBeNull();
|
||||||
|
expect(recentRunsBox).not.toBeNull();
|
||||||
|
if (bodyBox && eventLogBox && recentRunsBox) {
|
||||||
|
const eventBottomGutter = bodyBox.y + bodyBox.height - eventLogBox.y - eventLogBox.height;
|
||||||
|
const recentBottomGutter = bodyBox.y + bodyBox.height - recentRunsBox.y - recentRunsBox.height;
|
||||||
|
expect(eventBottomGutter).toBeGreaterThanOrEqual(10);
|
||||||
|
expect(recentBottomGutter).toBeGreaterThanOrEqual(10);
|
||||||
|
expect(eventLogBox.x + eventLogBox.width).toBeLessThan(recentRunsBox.x);
|
||||||
|
}
|
||||||
|
|
||||||
|
await page.setViewportSize({ width: 720, height: 800 });
|
||||||
|
const [compactManagerBox, compactDescriptionPanelBox] = await Promise.all([
|
||||||
|
page.getByTestId("conversation-column").getByRole("main", {
|
||||||
|
name: "Database management",
|
||||||
|
}).boundingBox(),
|
||||||
|
descriptionPanel.boundingBox(),
|
||||||
|
]);
|
||||||
|
expect(compactManagerBox).not.toBeNull();
|
||||||
|
expect(compactDescriptionPanelBox).not.toBeNull();
|
||||||
|
if (compactManagerBox && compactDescriptionPanelBox) {
|
||||||
|
expect(compactManagerBox.width - compactDescriptionPanelBox.width).toBeGreaterThanOrEqual(20);
|
||||||
|
}
|
||||||
|
expect(await descriptionBody.evaluate((element) => getComputedStyle(element).display)).not.toBe("grid");
|
||||||
|
|
||||||
|
await page.setViewportSize({ width: 1910, height: 911 });
|
||||||
|
await descriptionPanel.getByRole("button", { name: /close/i }).click();
|
||||||
|
|
||||||
const applicationBar = page.locator(
|
const applicationBar = page.locator(
|
||||||
'main[aria-label="Database management"] .thot-fleet-ledger__header',
|
'main[aria-label="Database management"] .thot-fleet-ledger__header',
|
||||||
@@ -734,7 +813,7 @@ test("Workspace and Pi management share the centered work-area panel without cov
|
|||||||
expect(compactWorkAreaBox).not.toBeNull();
|
expect(compactWorkAreaBox).not.toBeNull();
|
||||||
expect(compactPanelBox).not.toBeNull();
|
expect(compactPanelBox).not.toBeNull();
|
||||||
if (compactWorkAreaBox && compactPanelBox) {
|
if (compactWorkAreaBox && compactPanelBox) {
|
||||||
expect(Math.abs(compactPanelBox.width - compactWorkAreaBox.width + 16)).toBeLessThanOrEqual(1);
|
expect(Math.abs(compactPanelBox.width - compactWorkAreaBox.width + 24)).toBeLessThanOrEqual(1);
|
||||||
expect(Math.abs(
|
expect(Math.abs(
|
||||||
compactPanelBox.x + compactPanelBox.width / 2
|
compactPanelBox.x + compactPanelBox.width / 2
|
||||||
- (compactWorkAreaBox.x + compactWorkAreaBox.width / 2),
|
- (compactWorkAreaBox.x + compactWorkAreaBox.width / 2),
|
||||||
|
|||||||
@@ -269,7 +269,6 @@ export function FleetSelectionActions({ props, direction = "row" }: { props: Var
|
|||||||
if (state.inventoryLevel === "databases") actions = [
|
if (state.inventoryLevel === "databases") actions = [
|
||||||
{ id: "databases-test", group: "Connection", label: "Test connections", icon: <TestTube2 size={14} />, disabled, onClick: run(`Testing ${count} selected connection${count === 1 ? "" : "s"}`) },
|
{ id: "databases-test", group: "Connection", label: "Test connections", icon: <TestTube2 size={14} />, disabled, onClick: run(`Testing ${count} selected connection${count === 1 ? "" : "s"}`) },
|
||||||
{ id: "databases-sync-tables", group: "Synchronization", label: "Synchronize tables", icon: <RefreshCw size={14} />, disabled, onClick: run("Table synchronization started", "running") },
|
{ id: "databases-sync-tables", group: "Synchronization", label: "Synchronize tables", icon: <RefreshCw size={14} />, disabled, onClick: run("Table synchronization started", "running") },
|
||||||
{ id: "databases-sync-columns", group: "Synchronization", label: "Synchronize all columns", icon: <Columns3 size={14} />, disabled, onClick: run("Column synchronization started", "running") },
|
|
||||||
{ id: "databases-sync-relationships", group: "Synchronization", label: "Synchronize relationships", icon: <Activity size={14} />, disabled, onClick: run("Relationship synchronization started", "running") },
|
{ id: "databases-sync-relationships", group: "Synchronization", label: "Synchronize relationships", icon: <Activity size={14} />, disabled, onClick: run("Relationship synchronization started", "running") },
|
||||||
{ id: "databases-sync-all", group: "Synchronization", label: "Synchronize all", icon: <RefreshCw size={14} />, disabled, onClick: run("Full schema synchronization started", "running") },
|
{ id: "databases-sync-all", group: "Synchronization", label: "Synchronize all", icon: <RefreshCw size={14} />, disabled, onClick: run("Full schema synchronization started", "running") },
|
||||||
{ id: "databases-generate-all", group: "Descriptions", label: "Generate all descriptions", icon: <Sparkles size={14} />, disabled, onClick: run("Generate all confirmation opened") },
|
{ id: "databases-generate-all", group: "Descriptions", label: "Generate all descriptions", icon: <Sparkles size={14} />, disabled, onClick: run("Generate all confirmation opened") },
|
||||||
|
|||||||
@@ -157,6 +157,9 @@ export interface SensitiveDataSuggestionRun {
|
|||||||
total: number;
|
total: number;
|
||||||
suggestedSensitive: number;
|
suggestedSensitive: number;
|
||||||
suggestedNonSensitive: number;
|
suggestedNonSensitive: number;
|
||||||
|
inputTokens?: number;
|
||||||
|
cacheReadTokens?: number;
|
||||||
|
outputTokens?: number;
|
||||||
createdAt: string;
|
createdAt: string;
|
||||||
startedAt: string | null;
|
startedAt: string | null;
|
||||||
updatedAt: string;
|
updatedAt: string;
|
||||||
@@ -297,6 +300,9 @@ export interface DescriptionGenerationRun {
|
|||||||
generated: number;
|
generated: number;
|
||||||
nonGeneratable: number;
|
nonGeneratable: number;
|
||||||
failed: number;
|
failed: number;
|
||||||
|
inputTokens?: number;
|
||||||
|
cacheReadTokens?: number;
|
||||||
|
outputTokens?: number;
|
||||||
createdAt: string;
|
createdAt: string;
|
||||||
startedAt: string | null;
|
startedAt: string | null;
|
||||||
updatedAt: string;
|
updatedAt: string;
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import { cva, type VariantProps } from "class-variance-authority"
|
|||||||
import { cn } from "@/lib/utils"
|
import { cn } from "@/lib/utils"
|
||||||
|
|
||||||
const buttonVariants = cva(
|
const buttonVariants = cva(
|
||||||
"group/button inline-flex shrink-0 items-center justify-center rounded-md border border-transparent bg-clip-padding text-sm font-semibold tracking-[0.005em] whitespace-nowrap transition-all outline-none select-none focus-visible:ring-3 focus-visible:ring-ring/25 disabled:pointer-events-none disabled:opacity-50 aria-invalid:border-destructive aria-invalid:ring-3 aria-invalid:ring-destructive/20 dark:aria-invalid:border-destructive/50 dark:aria-invalid:ring-destructive/40 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
|
"group/button inline-flex shrink-0 items-center justify-center rounded-md border border-transparent bg-clip-padding font-sans text-sm font-semibold tracking-[0.005em] whitespace-nowrap transition-all outline-none select-none focus-visible:ring-3 focus-visible:ring-ring/25 disabled:pointer-events-none disabled:opacity-50 aria-invalid:border-destructive aria-invalid:ring-3 aria-invalid:ring-destructive/20 dark:aria-invalid:border-destructive/50 dark:aria-invalid:ring-destructive/40 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
|
||||||
{
|
{
|
||||||
variants: {
|
variants: {
|
||||||
variant: {
|
variant: {
|
||||||
|
|||||||
@@ -67,6 +67,7 @@ describe("authenticated shell permissions", () => {
|
|||||||
|
|
||||||
const sessionNavigation = screen.getByRole("complementary", { name: "Session navigation" });
|
const sessionNavigation = screen.getByRole("complementary", { name: "Session navigation" });
|
||||||
const trigger = await within(sessionNavigation).findByRole("button", { name: "Administration" });
|
const trigger = await within(sessionNavigation).findByRole("button", { name: "Administration" });
|
||||||
|
expect(trigger).toHaveClass("font-sans");
|
||||||
expect(trigger).toHaveAttribute("aria-expanded", "false");
|
expect(trigger).toHaveAttribute("aria-expanded", "false");
|
||||||
expect(within(sessionNavigation).queryByRole("region", { name: "Administration" })).not.toBeInTheDocument();
|
expect(within(sessionNavigation).queryByRole("region", { name: "Administration" })).not.toBeInTheDocument();
|
||||||
|
|
||||||
|
|||||||
@@ -45,6 +45,23 @@ interface AppShellProps {
|
|||||||
type ActiveSurface = "core" | "database-management";
|
type ActiveSurface = "core" | "database-management";
|
||||||
type ActiveManagementPanel = "workspace" | "pi" | null;
|
type ActiveManagementPanel = "workspace" | "pi" | null;
|
||||||
const SESSION_SCOPES: readonly SessionScope[] = ["mine", "all"];
|
const SESSION_SCOPES: readonly SessionScope[] = ["mine", "all"];
|
||||||
|
const APP_NAVIGATION_STORAGE_KEY = "thothii:app-navigation";
|
||||||
|
|
||||||
|
function readPersistedNavigation(): { surface: ActiveSurface; panel: ActiveManagementPanel } {
|
||||||
|
if (import.meta.env.MODE === "test") return { surface: "core", panel: null };
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(window.localStorage.getItem(APP_NAVIGATION_STORAGE_KEY) ?? "null") as {
|
||||||
|
surface?: unknown;
|
||||||
|
panel?: unknown;
|
||||||
|
} | null;
|
||||||
|
return {
|
||||||
|
surface: parsed?.surface === "database-management" ? "database-management" : "core",
|
||||||
|
panel: parsed?.panel === "workspace" || parsed?.panel === "pi" ? parsed.panel : null,
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
return { surface: "core", panel: null };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function sessionScopeTabClass(selected: boolean): string {
|
function sessionScopeTabClass(selected: boolean): string {
|
||||||
const base = "relative -mb-px flex h-8 w-full cursor-pointer items-center justify-center rounded-t-md border border-b-2 px-2 text-xs font-semibold tracking-[0.005em] outline-none focus-visible:z-10 focus-visible:ring-3 focus-visible:ring-[oklch(var(--nav-active-border)/0.28)]";
|
const base = "relative -mb-px flex h-8 w-full cursor-pointer items-center justify-center rounded-t-md border border-b-2 px-2 text-xs font-semibold tracking-[0.005em] outline-none focus-visible:z-10 focus-visible:ring-3 focus-visible:ring-[oklch(var(--nav-active-border)/0.28)]";
|
||||||
@@ -166,7 +183,8 @@ export function AppShell({ canLogout }: AppShellProps) {
|
|||||||
|
|
||||||
const queryClient = useQueryClient();
|
const queryClient = useQueryClient();
|
||||||
const [showActivity, setShowActivity] = useState(false);
|
const [showActivity, setShowActivity] = useState(false);
|
||||||
const [activeSurface, setActiveSurface] = useState<ActiveSurface>("core");
|
const persistedNavigation = readPersistedNavigation();
|
||||||
|
const [activeSurface, setActiveSurface] = useState<ActiveSurface>(persistedNavigation.surface);
|
||||||
const databaseManagementPresentation = resolveDatabaseManagementPresentation({
|
const databaseManagementPresentation = resolveDatabaseManagementPresentation({
|
||||||
isDevelopment: import.meta.env.DEV,
|
isDevelopment: import.meta.env.DEV,
|
||||||
mode: import.meta.env.MODE,
|
mode: import.meta.env.MODE,
|
||||||
@@ -177,7 +195,7 @@ export function AppShell({ canLogout }: AppShellProps) {
|
|||||||
const updateDatabaseNavigationState = useCallback((state: { dirty: boolean; busy: boolean }) => {
|
const updateDatabaseNavigationState = useCallback((state: { dirty: boolean; busy: boolean }) => {
|
||||||
databaseNavigationRef.current = state;
|
databaseNavigationRef.current = state;
|
||||||
}, []);
|
}, []);
|
||||||
const [activeManagementPanel, setActiveManagementPanel] = useState<ActiveManagementPanel>(null);
|
const [activeManagementPanel, setActiveManagementPanel] = useState<ActiveManagementPanel>(persistedNavigation.panel);
|
||||||
const [adminNavigationValue, setAdminNavigationValue] = useState<string[]>([]);
|
const [adminNavigationValue, setAdminNavigationValue] = useState<string[]>([]);
|
||||||
const [activeOpen, setActiveOpen] = useState(true);
|
const [activeOpen, setActiveOpen] = useState(true);
|
||||||
const [archiveOpen, setArchiveOpen] = useState(false);
|
const [archiveOpen, setArchiveOpen] = useState(false);
|
||||||
@@ -188,6 +206,10 @@ export function AppShell({ canLogout }: AppShellProps) {
|
|||||||
const [collapsedGroups, setCollapsedGroups] = useState<Record<string, boolean>>({});
|
const [collapsedGroups, setCollapsedGroups] = useState<Record<string, boolean>>({});
|
||||||
const [renameGroupTarget, setRenameGroupTarget] = useState<string | null>(null);
|
const [renameGroupTarget, setRenameGroupTarget] = useState<string | null>(null);
|
||||||
const operationEpochRef = useRef(0);
|
const operationEpochRef = useRef(0);
|
||||||
|
useEffect(() => {
|
||||||
|
if (import.meta.env.MODE === "test") return;
|
||||||
|
window.localStorage.setItem(APP_NAVIGATION_STORAGE_KEY, JSON.stringify({ surface: activeSurface, panel: activeManagementPanel }));
|
||||||
|
}, [activeManagementPanel, activeSurface]);
|
||||||
useEffect(() => () => { operationEpochRef.current += 1; }, []);
|
useEffect(() => () => { operationEpochRef.current += 1; }, []);
|
||||||
|
|
||||||
const groups = useMemo(
|
const groups = useMemo(
|
||||||
|
|||||||
@@ -197,7 +197,6 @@ function registerCompletedSyncRun(run: CatalogSyncRun) {
|
|||||||
|
|
||||||
const synchronizationScopes = [
|
const synchronizationScopes = [
|
||||||
{ scope: "tables", label: "Synchronize tables" },
|
{ scope: "tables", label: "Synchronize tables" },
|
||||||
{ scope: "columns", label: "Synchronize all columns" },
|
|
||||||
{ scope: "relationships", label: "Synchronize relationships" },
|
{ scope: "relationships", label: "Synchronize relationships" },
|
||||||
{ scope: "all", label: "Synchronize all" },
|
{ scope: "all", label: "Synchronize all" },
|
||||||
] as const;
|
] as const;
|
||||||
@@ -234,9 +233,7 @@ test("renders Fleet Ledger with real metrics, conceptual row tooltips, and persi
|
|||||||
expect(screen.queryByText("Inspect physical metadata, curate descriptions and control catalog operations.")).not.toBeInTheDocument();
|
expect(screen.queryByText("Inspect physical metadata, curate descriptions and control catalog operations.")).not.toBeInTheDocument();
|
||||||
expect(screen.queryByRole("button", { name: "Back to workspace" })).not.toBeInTheDocument();
|
expect(screen.queryByRole("button", { name: "Back to workspace" })).not.toBeInTheDocument();
|
||||||
expect(screen.queryByRole("button", { name: /Add database/i })).not.toBeInTheDocument();
|
expect(screen.queryByRole("button", { name: /Add database/i })).not.toBeInTheDocument();
|
||||||
expect(screen.getByRole("columnheader", { name: /Revision \/ Evidence/ })).toBeVisible();
|
expect(screen.getByRole("columnheader", { name: /Catalog status/ })).toBeVisible();
|
||||||
expect(screen.getByRole("columnheader", { name: /NL→SQL runtime/ })).toBeVisible();
|
|
||||||
expect(screen.getByRole("columnheader", { name: /Metadata Catalog/ })).toBeVisible();
|
|
||||||
const summary = screen.getByRole("region", { name: "Fleet summary" });
|
const summary = screen.getByRole("region", { name: "Fleet summary" });
|
||||||
expect(await within(summary).findByText("2,275")).toBeVisible();
|
expect(await within(summary).findByText("2,275")).toBeVisible();
|
||||||
expect(within(summary).getByText("75%")).toBeVisible();
|
expect(within(summary).getByText("75%")).toBeVisible();
|
||||||
@@ -261,13 +258,14 @@ test("renders Fleet Ledger with real metrics, conceptual row tooltips, and persi
|
|||||||
await user.click(within(restoredDatabaseRow).getByRole("checkbox", { name: /toggle row selection/i }));
|
await user.click(within(restoredDatabaseRow).getByRole("checkbox", { name: /toggle row selection/i }));
|
||||||
const actionPicker = screen.getByRole("combobox", { name: "Batch action" });
|
const actionPicker = screen.getByRole("combobox", { name: "Batch action" });
|
||||||
expect(within(actionPicker).getByRole("option", { name: /^Synchronize all(?:,|$)/ })).toBeVisible();
|
expect(within(actionPicker).getByRole("option", { name: /^Synchronize all(?:,|$)/ })).toBeVisible();
|
||||||
|
expect(within(actionPicker).queryByRole("option", { name: "Synchronize all columns" })).not.toBeInTheDocument();
|
||||||
await user.selectOptions(actionPicker, "generate-missing");
|
await user.selectOptions(actionPicker, "generate-missing");
|
||||||
expect(screen.getAllByRole("combobox", { name: "Metadata-generation LLM model" })).toHaveLength(1);
|
expect(screen.getAllByRole("combobox", { name: "Metadata-generation LLM model" })).toHaveLength(1);
|
||||||
expect(modelSelector).toHaveValue("local-qwen");
|
expect(modelSelector).toHaveValue("local-qwen");
|
||||||
|
|
||||||
});
|
});
|
||||||
|
|
||||||
test("shows repository, NL→SQL runtime, and Metadata Catalog states independently", async () => {
|
test("shows synchronization, description, and sensitive-data states independently", async () => {
|
||||||
const configured = makeDatabase({ connectionStatus: "reachable", testedVersion: 3 });
|
const configured = makeDatabase({ connectionStatus: "reachable", testedVersion: 3 });
|
||||||
const needsRuntimeConfiguration = makeDatabase({
|
const needsRuntimeConfiguration = makeDatabase({
|
||||||
...unconfigured,
|
...unconfigured,
|
||||||
@@ -281,21 +279,58 @@ test("shows repository, NL→SQL runtime, and Metadata Catalog states independen
|
|||||||
renderPage({ rows: [configured, needsRuntimeConfiguration, orphan], presentation: "fleet" });
|
renderPage({ rows: [configured, needsRuntimeConfiguration, orphan], presentation: "fleet" });
|
||||||
|
|
||||||
const configuredRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
|
const configuredRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
|
||||||
expect(within(configuredRow).getByText("Active revision aaaaaaa")).toBeVisible();
|
expect(within(configuredRow).getByText("Not synchronized")).toBeVisible();
|
||||||
expect(within(configuredRow).getByText("Evidence materialized · filesystem")).toBeVisible();
|
expect(within(configuredRow).getByText(/Descriptions ·/)).toBeVisible();
|
||||||
expect(within(configuredRow).getByText("Ready")).toBeVisible();
|
expect(within(configuredRow).getByText(/Sensitive ·/)).toBeVisible();
|
||||||
expect(within(configuredRow).getByText("Configured")).toBeVisible();
|
|
||||||
expect(within(configuredRow).getByText("Connection reachable")).toBeVisible();
|
|
||||||
|
|
||||||
const needsConfigurationRow = screen.getByRole("row", { name: /Research laboratory/ });
|
const needsConfigurationRow = screen.getByRole("row", { name: /Research laboratory/ });
|
||||||
expect(within(needsConfigurationRow).getByText("Evidence credentials required · http")).toBeVisible();
|
expect(within(needsConfigurationRow).getByText("Not synchronized")).toBeVisible();
|
||||||
expect(within(needsConfigurationRow).getByText("Configuration required")).toBeVisible();
|
expect(within(needsConfigurationRow).getByText(/Descriptions ·/)).toBeVisible();
|
||||||
expect(within(needsConfigurationRow).getByText("Not configured")).toBeVisible();
|
|
||||||
|
|
||||||
const orphanRow = screen.getByRole("row", { name: /Retired workspace/ });
|
const orphanRow = screen.getByRole("row", { name: /Retired workspace/ });
|
||||||
expect(within(orphanRow).getByText("Workspace missing")).toBeVisible();
|
expect(within(orphanRow).getByText("Not synchronized")).toBeVisible();
|
||||||
expect(within(orphanRow).getByText("Unavailable")).toBeVisible();
|
});
|
||||||
expect(within(orphanRow).getByText("Orphaned configuration")).toBeVisible();
|
|
||||||
|
test("shows the configured access type and endpoint in catalog status", async () => {
|
||||||
|
const direct = makeDatabase();
|
||||||
|
const rest = makeDatabase({
|
||||||
|
id: "33333333-3333-4333-8333-333333333333",
|
||||||
|
workspaceId: "rest-lab",
|
||||||
|
workspaceName: "REST laboratory",
|
||||||
|
binding: {
|
||||||
|
transport: "rest_api",
|
||||||
|
baseUrl: "https://203.0.113.18/dwh/",
|
||||||
|
restPath: "/health",
|
||||||
|
restAuth: "x-api-key",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const ssh = makeDatabase({
|
||||||
|
id: "44444444-4444-4444-8444-444444444444",
|
||||||
|
workspaceId: "ssh-lab",
|
||||||
|
workspaceName: "SSH laboratory",
|
||||||
|
binding: {
|
||||||
|
transport: "ssh_tunnel",
|
||||||
|
sshHost: "bastion.example.test",
|
||||||
|
sshPort: 2222,
|
||||||
|
sshUsername: "operator",
|
||||||
|
sshTargetHost: "postgres.internal",
|
||||||
|
sshTargetPort: 5432,
|
||||||
|
username: "reader",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
renderPage({ rows: [direct, rest, ssh], presentation: "fleet" });
|
||||||
|
|
||||||
|
const directRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
|
||||||
|
expect(within(directRow).getByText("Direct PostgreSQL")).toBeVisible();
|
||||||
|
expect(within(directRow).getByText("db.internal:5432")).toBeVisible();
|
||||||
|
|
||||||
|
const restRow = screen.getByRole("row", { name: /REST laboratory/ });
|
||||||
|
expect(within(restRow).getByText("REST")).toBeVisible();
|
||||||
|
expect(within(restRow).getByText("203.0.113.18")).toBeVisible();
|
||||||
|
|
||||||
|
const sshRow = screen.getByRole("row", { name: /SSH laboratory/ });
|
||||||
|
expect(within(sshRow).getByText("SSH")).toBeVisible();
|
||||||
|
expect(within(sshRow).getByText("bastion.example.test:2222")).toBeVisible();
|
||||||
});
|
});
|
||||||
|
|
||||||
test("keeps Fleet Ledger visible when a catalog response omits the Evidence projection", async () => {
|
test("keeps Fleet Ledger visible when a catalog response omits the Evidence projection", async () => {
|
||||||
@@ -306,7 +341,7 @@ test("keeps Fleet Ledger visible when a catalog response omits the Evidence proj
|
|||||||
|
|
||||||
expect(await screen.findByRole("heading", { name: "Database management" })).toBeVisible();
|
expect(await screen.findByRole("heading", { name: "Database management" })).toBeVisible();
|
||||||
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
|
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
|
||||||
expect(within(databaseRow).getByText("Evidence state unavailable")).toBeVisible();
|
expect(within(databaseRow).getByText("Not synchronized")).toBeVisible();
|
||||||
});
|
});
|
||||||
|
|
||||||
test("opens the relationship map directly from a Fleet database and restores focus on return", async () => {
|
test("opens the relationship map directly from a Fleet database and restores focus on return", async () => {
|
||||||
@@ -359,7 +394,7 @@ test("offers catalog configuration directly on an unconfigured Fleet workspace",
|
|||||||
expect(within(drawer).getByLabelText("Workspace")).toHaveValue("Research laboratory");
|
expect(within(drawer).getByLabelText("Workspace")).toHaveValue("Research laboratory");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("keeps only the local back action while browsing Fleet Ledger columns", async () => {
|
test("keeps the table back action in the Fleet Ledger breadcrumb while browsing columns", async () => {
|
||||||
const user = userEvent.setup();
|
const user = userEvent.setup();
|
||||||
server.use(
|
server.use(
|
||||||
http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([patientsTable])),
|
http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([patientsTable])),
|
||||||
@@ -372,9 +407,17 @@ test("keeps only the local back action while browsing Fleet Ledger columns", asy
|
|||||||
name: "View tables for Policlinico San Donato",
|
name: "View tables for Policlinico San Donato",
|
||||||
}));
|
}));
|
||||||
expect(await screen.findByRole("button", { name: "Back to databases" })).toBeVisible();
|
expect(await screen.findByRole("button", { name: "Back to databases" })).toBeVisible();
|
||||||
|
expect(screen.getByRole("combobox", { name: "Table action" })).toBeDisabled();
|
||||||
|
expect(screen.queryByRole("button", { name: "Sync history" })).not.toBeInTheDocument();
|
||||||
|
expect(screen.queryByRole("button", { name: "Description history" })).not.toBeInTheDocument();
|
||||||
|
expect(screen.queryByRole("button", { name: "Sensitive history" })).not.toBeInTheDocument();
|
||||||
|
|
||||||
await user.click(await screen.findByRole("button", { name: "View columns for patients" }));
|
await user.click(await screen.findByRole("button", { name: "View columns for patients" }));
|
||||||
expect(await screen.findByRole("button", { name: "Back to tables" })).toBeVisible();
|
const tableBackAction = await screen.findByRole("button", { name: "Back to tables" });
|
||||||
|
expect(tableBackAction).toBeVisible();
|
||||||
|
expect(screen.getByRole("combobox", { name: "Column action" })).toBeDisabled();
|
||||||
|
expect(screen.getByRole("navigation", { name: "Database hierarchy" }).parentElement)
|
||||||
|
.toContainElement(tableBackAction);
|
||||||
await waitFor(() => {
|
await waitFor(() => {
|
||||||
expect(screen.queryByRole("button", { name: "Back to databases" })).not.toBeInTheDocument();
|
expect(screen.queryByRole("button", { name: "Back to databases" })).not.toBeInTheDocument();
|
||||||
});
|
});
|
||||||
@@ -427,7 +470,7 @@ test("reopens database synchronization history when no run is active", async ()
|
|||||||
renderPage({ rows: [makeDatabase()], presentation: "fleet" });
|
renderPage({ rows: [makeDatabase()], presentation: "fleet" });
|
||||||
|
|
||||||
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
|
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
|
||||||
await user.click(within(databaseRow).getByRole("button", { name: "View synchronization history for Policlinico San Donato" }));
|
await user.click(within(databaseRow).getByText("Policlinico San Donato"));
|
||||||
|
|
||||||
const drawer = await screen.findByRole("dialog", { name: "Schema synchronization" });
|
const drawer = await screen.findByRole("dialog", { name: "Schema synchronization" });
|
||||||
expect(drawer).toBeVisible();
|
expect(drawer).toBeVisible();
|
||||||
@@ -628,6 +671,7 @@ test.each(synchronizationScopes)(
|
|||||||
await user.click(screen.getByRole("menuitem", { name: label }));
|
await user.click(screen.getByRole("menuitem", { name: label }));
|
||||||
|
|
||||||
await waitFor(() => expect(startBody).toEqual({ version: 3, scope, tableIds: [] }));
|
await waitFor(() => expect(startBody).toEqual({ version: 3, scope, tableIds: [] }));
|
||||||
|
expect(screen.getByText("1 selected")).toBeVisible();
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -1001,15 +1045,16 @@ test("selected database Actions confirms and deletes catalog tables for the full
|
|||||||
],
|
],
|
||||||
target: "tables",
|
target: "tables",
|
||||||
}));
|
}));
|
||||||
await waitFor(() => expect(screen.queryByText("2 selected")).not.toBeInTheDocument());
|
expect(screen.getByText("2 selected")).toBeVisible();
|
||||||
await waitFor(() => expect(screen.getByRole("textbox", { name: "Search databases" })).toHaveFocus());
|
await user.click(screen.getByRole("button", { name: "Clear" }));
|
||||||
|
expect(screen.queryByText("2 selected")).not.toBeInTheDocument();
|
||||||
});
|
});
|
||||||
|
|
||||||
test("presents completed synchronization steps as success and skips unneeded confirmation", async () => {
|
test("presents completed database synchronization steps as success and skips unneeded confirmation", async () => {
|
||||||
const user = userEvent.setup();
|
const user = userEvent.setup();
|
||||||
const run = {
|
const run = {
|
||||||
...makeSyncRun("columns"),
|
...makeSyncRun("all"),
|
||||||
counts: { tables: 163, columns: 2_275 },
|
counts: { tables: 163, columns: 2_275, relationships: 18 },
|
||||||
};
|
};
|
||||||
registerCompletedSyncRun(run);
|
registerCompletedSyncRun(run);
|
||||||
server.use(http.post("/api/catalog/databases/:databaseId/sync-runs", () => (
|
server.use(http.post("/api/catalog/databases/:databaseId/sync-runs", () => (
|
||||||
@@ -1022,7 +1067,7 @@ test("presents completed synchronization steps as success and skips unneeded con
|
|||||||
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
|
const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ });
|
||||||
await user.click(within(databaseRow).getByRole("checkbox", { name: /toggle row selection/i }));
|
await user.click(within(databaseRow).getByRole("checkbox", { name: /toggle row selection/i }));
|
||||||
await user.click(screen.getByRole("button", { name: "Actions" }));
|
await user.click(screen.getByRole("button", { name: "Actions" }));
|
||||||
await user.click(await screen.findByRole("menuitem", { name: "Synchronize all columns" }));
|
await user.click(await screen.findByRole("menuitem", { name: "Synchronize all" }));
|
||||||
|
|
||||||
expect(await screen.findByRole("heading", { name: "Succeeded" })).toBeVisible();
|
expect(await screen.findByRole("heading", { name: "Succeeded" })).toBeVisible();
|
||||||
const steps = screen.getByRole("list", { name: "Synchronization steps" });
|
const steps = screen.getByRole("list", { name: "Synchronization steps" });
|
||||||
@@ -1039,7 +1084,7 @@ test("presents completed synchronization steps as success and skips unneeded con
|
|||||||
expect(steps.querySelectorAll(".text-primary")).toHaveLength(0);
|
expect(steps.querySelectorAll(".text-primary")).toHaveLength(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("database Overview exposes every synchronization scope", async () => {
|
test("database Overview exposes the database synchronization scopes", async () => {
|
||||||
const user = userEvent.setup();
|
const user = userEvent.setup();
|
||||||
let startBody: unknown;
|
let startBody: unknown;
|
||||||
const run = makeSyncRun("relationships");
|
const run = makeSyncRun("relationships");
|
||||||
@@ -1520,6 +1565,37 @@ test("starts description generation for multiple selected tables", async () => {
|
|||||||
expect(screen.getByText("2 selected")).toBeVisible();
|
expect(screen.getByText("2 selected")).toBeVisible();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("generates descriptions for every column in selected tables", async () => {
|
||||||
|
const user = userEvent.setup();
|
||||||
|
const queuedRun = makeDescriptionGenerationRun({ scope: "selected_columns", total: 1 });
|
||||||
|
let startBody: unknown;
|
||||||
|
server.use(
|
||||||
|
http.get("/api/catalog/metadata-generation/models", () => HttpResponse.json({ models: [{ id: "local-qwen", label: "Local Qwen" }], default: "local-qwen" })),
|
||||||
|
http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([patientsTable])),
|
||||||
|
http.get(`/api/catalog/databases/:databaseId/tables/${patientsTable.id}/columns`, () => HttpResponse.json([patientIdColumn])),
|
||||||
|
http.post("/api/catalog/databases/:databaseId/description-generation-runs", async ({ request }) => {
|
||||||
|
startBody = await request.json();
|
||||||
|
return HttpResponse.json(queuedRun, { status: 202 });
|
||||||
|
}),
|
||||||
|
http.get("/api/catalog/description-generation-runs/:runId", () => HttpResponse.json(queuedRun)),
|
||||||
|
http.get("/api/catalog/description-generation-runs/:runId/events-list", () => HttpResponse.json([])),
|
||||||
|
);
|
||||||
|
renderPage({ rows: [makeDatabase({ connectionStatus: "reachable", testedVersion: 3 })] });
|
||||||
|
|
||||||
|
await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" }));
|
||||||
|
await user.click(screen.getByRole("tab", { name: "Tables" }));
|
||||||
|
await user.click(within(await screen.findByRole("row", { name: /patients/ })).getByRole("checkbox"));
|
||||||
|
await user.click(screen.getByRole("button", { name: "Actions" }));
|
||||||
|
await user.click(await screen.findByRole("menuitem", { name: "Generate descriptions for all columns" }));
|
||||||
|
|
||||||
|
await waitFor(() => expect(startBody).toEqual({
|
||||||
|
modelId: "local-qwen",
|
||||||
|
scope: "selected_columns",
|
||||||
|
targetIds: [patientIdColumn.id],
|
||||||
|
}));
|
||||||
|
expect(await screen.findByText("Column description generation started for 1 column")).toBeVisible();
|
||||||
|
});
|
||||||
|
|
||||||
test("observes an active run from another browser and reopens a terminal run from history", async () => {
|
test("observes an active run from another browser and reopens a terminal run from history", async () => {
|
||||||
const user = userEvent.setup();
|
const user = userEvent.setup();
|
||||||
const activeRun = makeDescriptionGenerationRun({
|
const activeRun = makeDescriptionGenerationRun({
|
||||||
@@ -1913,6 +1989,12 @@ test("starts one selected column with the configured default model", async () =>
|
|||||||
await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" }));
|
await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" }));
|
||||||
await user.click(screen.getByRole("tab", { name: "Tables" }));
|
await user.click(screen.getByRole("tab", { name: "Tables" }));
|
||||||
await user.click(await screen.findByRole("button", { name: "View columns for patients" }));
|
await user.click(await screen.findByRole("button", { name: "View columns for patients" }));
|
||||||
|
expect(screen.queryByRole("button", { name: "Sync columns" })).not.toBeInTheDocument();
|
||||||
|
expect(screen.getByRole("columnheader", { name: /Select/i })).toBeVisible();
|
||||||
|
expect(await screen.findByRole("checkbox", { name: "Sensitive data for id" })).toHaveAttribute(
|
||||||
|
"title",
|
||||||
|
"Mark this column as sensitive; this does not select it for actions.",
|
||||||
|
);
|
||||||
const columnRow = (await screen.findAllByRole("row", { name: /Patient identifier/ }))
|
const columnRow = (await screen.findAllByRole("row", { name: /Patient identifier/ }))
|
||||||
.find((row) => within(row).queryByRole("checkbox", { name: /toggle row selection/i }));
|
.find((row) => within(row).queryByRole("checkbox", { name: /toggle row selection/i }));
|
||||||
expect(columnRow).toBeDefined();
|
expect(columnRow).toBeDefined();
|
||||||
|
|||||||
@@ -80,6 +80,20 @@ const SYNC_STARTED_MESSAGES: Record<CatalogSyncScope, string> = {
|
|||||||
relationships: "Relationship synchronization started",
|
relationships: "Relationship synchronization started",
|
||||||
all: "Full schema synchronization started",
|
all: "Full schema synchronization started",
|
||||||
};
|
};
|
||||||
|
const DATABASE_SCREEN_STORAGE_KEY = "thothii:database-management-screen";
|
||||||
|
|
||||||
|
function readPersistedDatabaseScreen(): DatabaseScreen {
|
||||||
|
if (import.meta.env.MODE === "test") return { kind: "list" };
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(window.localStorage.getItem(DATABASE_SCREEN_STORAGE_KEY) ?? "null") as { kind?: unknown; workspaceId?: unknown } | null;
|
||||||
|
if ((parsed?.kind === "tables" || parsed?.kind === "relationships") && typeof parsed.workspaceId === "string") {
|
||||||
|
return { kind: parsed.kind, workspaceId: parsed.workspaceId };
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Ignore malformed persisted navigation and return to the safe list view.
|
||||||
|
}
|
||||||
|
return { kind: "list" };
|
||||||
|
}
|
||||||
|
|
||||||
interface Props {
|
interface Props {
|
||||||
canManage: boolean;
|
canManage: boolean;
|
||||||
@@ -156,12 +170,13 @@ export function DatabaseManagementPage({
|
|||||||
(run) => run.status === "running",
|
(run) => run.status === "running",
|
||||||
);
|
);
|
||||||
|
|
||||||
const [screen, setScreen] = useState<DatabaseScreen>({ kind: "list" });
|
const [screen, setScreen] = useState<DatabaseScreen>(readPersistedDatabaseScreen);
|
||||||
const [selectedMetadataModel, setSelectedMetadataModel] = useState("");
|
const [selectedMetadataModel, setSelectedMetadataModel] = useState("");
|
||||||
const [draft, setDraft] = useState<DatabaseFormDraft | null>(null);
|
const [draft, setDraft] = useState<DatabaseFormDraft | null>(null);
|
||||||
const [baseline, setBaseline] = useState("");
|
const [baseline, setBaseline] = useState("");
|
||||||
const [formSource, setFormSource] = useState<FormSource | null>(null);
|
const [formSource, setFormSource] = useState<FormSource | null>(null);
|
||||||
const [search, setSearch] = useState("");
|
const [search, setSearch] = useState("");
|
||||||
|
const [selectedDatabaseId, setSelectedDatabaseId] = useState<string | null>(null);
|
||||||
const [busyAction, setBusyAction] = useState<DatabaseBusyAction>(null);
|
const [busyAction, setBusyAction] = useState<DatabaseBusyAction>(null);
|
||||||
const [stale, setStale] = useState(false);
|
const [stale, setStale] = useState(false);
|
||||||
const [staleBannerOpen, setStaleBannerOpen] = useState(true);
|
const [staleBannerOpen, setStaleBannerOpen] = useState(true);
|
||||||
@@ -171,6 +186,11 @@ export function DatabaseManagementPage({
|
|||||||
busy: false,
|
busy: false,
|
||||||
});
|
});
|
||||||
const [tableNestedNavigationActive, setTableNestedNavigationActive] = useState(false);
|
const [tableNestedNavigationActive, setTableNestedNavigationActive] = useState(false);
|
||||||
|
const [tableNestedNavigationBack, setTableNestedNavigationBack] = useState<(() => void) | null>(null);
|
||||||
|
const handleTableNestedNavigationChange = useCallback((active: boolean, onBack?: () => void) => {
|
||||||
|
setTableNestedNavigationActive(active);
|
||||||
|
setTableNestedNavigationBack(() => (active ? onBack ?? null : null));
|
||||||
|
}, []);
|
||||||
const [activeSyncRun, setActiveSyncRun] = useState<CatalogSyncRun | null>(null);
|
const [activeSyncRun, setActiveSyncRun] = useState<CatalogSyncRun | null>(null);
|
||||||
const [syncHistoryDatabaseId, setSyncHistoryDatabaseId] = useState<string | null>(null);
|
const [syncHistoryDatabaseId, setSyncHistoryDatabaseId] = useState<string | null>(null);
|
||||||
const [syncDrawerOpen, setSyncDrawerOpen] = useState(false);
|
const [syncDrawerOpen, setSyncDrawerOpen] = useState(false);
|
||||||
@@ -184,6 +204,12 @@ export function DatabaseManagementPage({
|
|||||||
suggestions: SensitiveDataSuggestion[];
|
suggestions: SensitiveDataSuggestion[];
|
||||||
} | null>(null);
|
} | null>(null);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (import.meta.env.MODE === "test") return;
|
||||||
|
const persisted = screen.kind === "tables" || screen.kind === "relationships" ? screen : { kind: "list" };
|
||||||
|
window.localStorage.setItem(DATABASE_SCREEN_STORAGE_KEY, JSON.stringify(persisted));
|
||||||
|
}, [screen]);
|
||||||
|
|
||||||
const originRef = useRef<HTMLElement | null>(null);
|
const originRef = useRef<HTMLElement | null>(null);
|
||||||
const searchInputRef = useRef<HTMLInputElement>(null);
|
const searchInputRef = useRef<HTMLInputElement>(null);
|
||||||
const formHeadingRef = useRef<HTMLHeadingElement>(null);
|
const formHeadingRef = useRef<HTMLHeadingElement>(null);
|
||||||
@@ -284,6 +310,7 @@ export function DatabaseManagementPage({
|
|||||||
setPartialSecretFailure(null);
|
setPartialSecretFailure(null);
|
||||||
setTablesNavigationState({ dirty: false, busy: false });
|
setTablesNavigationState({ dirty: false, busy: false });
|
||||||
setTableNestedNavigationActive(false);
|
setTableNestedNavigationActive(false);
|
||||||
|
setTableNestedNavigationBack(null);
|
||||||
restoreListFocus();
|
restoreListFocus();
|
||||||
}, [restoreListFocus]);
|
}, [restoreListFocus]);
|
||||||
|
|
||||||
@@ -346,6 +373,7 @@ export function DatabaseManagementPage({
|
|||||||
setPartialSecretFailure(null);
|
setPartialSecretFailure(null);
|
||||||
setTablesNavigationState({ dirty: false, busy: false });
|
setTablesNavigationState({ dirty: false, busy: false });
|
||||||
setTableNestedNavigationActive(false);
|
setTableNestedNavigationActive(false);
|
||||||
|
setTableNestedNavigationBack(null);
|
||||||
setScreen({ kind: "tables", workspaceId: row.workspaceId });
|
setScreen({ kind: "tables", workspaceId: row.workspaceId });
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
@@ -358,6 +386,8 @@ export function DatabaseManagementPage({
|
|||||||
setStale(false);
|
setStale(false);
|
||||||
setPartialSecretFailure(null);
|
setPartialSecretFailure(null);
|
||||||
setTablesNavigationState({ dirty: false, busy: false });
|
setTablesNavigationState({ dirty: false, busy: false });
|
||||||
|
setTableNestedNavigationActive(false);
|
||||||
|
setTableNestedNavigationBack(null);
|
||||||
setScreen({ kind: "relationships", workspaceId: row.workspaceId });
|
setScreen({ kind: "relationships", workspaceId: row.workspaceId });
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
@@ -856,6 +886,11 @@ export function DatabaseManagementPage({
|
|||||||
}
|
}
|
||||||
}, [invalidateCatalogMetrics, queryClient]);
|
}, [invalidateCatalogMetrics, queryClient]);
|
||||||
|
|
||||||
|
const clearActiveCatalogTables = useCallback(async () => {
|
||||||
|
if (!activeRow?.id) return;
|
||||||
|
await deleteSelectedMetadata([activeRow], "tables");
|
||||||
|
}, [activeRow, deleteSelectedMetadata]);
|
||||||
|
|
||||||
const syncDatabase = useCallback(async (scope: CatalogSyncScope) => {
|
const syncDatabase = useCallback(async (scope: CatalogSyncScope) => {
|
||||||
if (!activeRow?.id || !activeRow.configured) return;
|
if (!activeRow?.id || !activeRow.configured) return;
|
||||||
try {
|
try {
|
||||||
@@ -1038,13 +1073,12 @@ export function DatabaseManagementPage({
|
|||||||
onOpenOverview={() => openOverview(activeRow)}
|
onOpenOverview={() => openOverview(activeRow)}
|
||||||
onOpenRelationships={() => openRelationships(activeRow)}
|
onOpenRelationships={() => openRelationships(activeRow)}
|
||||||
onNavigationStateChange={setTablesNavigationState}
|
onNavigationStateChange={setTablesNavigationState}
|
||||||
onNestedNavigationChange={setTableNestedNavigationActive}
|
onNestedNavigationChange={handleTableNestedNavigationChange}
|
||||||
onRunStarted={rememberSyncRun}
|
onRunStarted={rememberSyncRun}
|
||||||
onOpenSync={() => openSync(activeRow)}
|
onOpenSync={() => openSync(activeRow)}
|
||||||
onOpenDescriptionHistory={openDescriptionGenerationHistory}
|
|
||||||
onOpenSensitiveHistory={openSensitiveDataSuggestionHistory}
|
|
||||||
onDescriptionGenerationRunStarted={rememberDescriptionGenerationRun}
|
onDescriptionGenerationRunStarted={rememberDescriptionGenerationRun}
|
||||||
onSuggestSensitive={suggestActiveDatabaseSensitiveFields}
|
onSuggestSensitive={suggestActiveDatabaseSensitiveFields}
|
||||||
|
onClearCatalogTables={clearActiveCatalogTables}
|
||||||
onCatalogMetricsChanged={invalidateCatalogMetrics}
|
onCatalogMetricsChanged={invalidateCatalogMetrics}
|
||||||
/>
|
/>
|
||||||
) : screen.kind === "relationships" && relationshipsVisible ? (
|
) : screen.kind === "relationships" && relationshipsVisible ? (
|
||||||
@@ -1088,12 +1122,14 @@ export function DatabaseManagementPage({
|
|||||||
onEdit={editRow}
|
onEdit={editRow}
|
||||||
onDelete={deleteRow}
|
onDelete={deleteRow}
|
||||||
onOpenSync={openSync}
|
onOpenSync={openSync}
|
||||||
|
onSelectionChange={(selected) => setSelectedDatabaseId(selected.length === 1 ? selected[0].id ?? null : null)}
|
||||||
onTestSelected={testSelected}
|
onTestSelected={testSelected}
|
||||||
onSyncSelected={syncSelected}
|
onSyncSelected={syncSelected}
|
||||||
selectedMetadataModel={selectedMetadataModelAvailable ? selectedMetadataModel : null}
|
selectedMetadataModel={selectedMetadataModelAvailable ? selectedMetadataModel : null}
|
||||||
descriptionGenerationActive={descriptionGenerationActive}
|
descriptionGenerationActive={descriptionGenerationActive}
|
||||||
onGenerateDescriptions={generateDatabaseDescriptions}
|
onGenerateDescriptions={generateDatabaseDescriptions}
|
||||||
onSuggestSensitive={suggestDatabaseSensitiveFields}
|
onSuggestSensitive={suggestDatabaseSensitiveFields}
|
||||||
|
sensitiveDataSuggestionRuns={sensitiveDataSuggestionRuns}
|
||||||
onDeleteMetadataSelected={deleteSelectedMetadata}
|
onDeleteMetadataSelected={deleteSelectedMetadata}
|
||||||
onRefresh={refreshList}
|
onRefresh={refreshList}
|
||||||
/>
|
/>
|
||||||
@@ -1125,6 +1161,45 @@ export function DatabaseManagementPage({
|
|||||||
actions={(
|
actions={(
|
||||||
<>
|
<>
|
||||||
{fleetMetadataModelControl}
|
{fleetMetadataModelControl}
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
variant="outline"
|
||||||
|
size="sm"
|
||||||
|
className="whitespace-nowrap"
|
||||||
|
aria-label="View schema synchronization logs"
|
||||||
|
title="View schema synchronization progress and history"
|
||||||
|
disabled={!canManage || (!selectedDatabaseId && !activeSyncRun?.databaseId)}
|
||||||
|
onClick={() => openSync(selectedDatabaseId ? rows.find((row) => row.id === selectedDatabaseId) : undefined)}
|
||||||
|
>
|
||||||
|
<RefreshCw aria-hidden="true" />
|
||||||
|
{currentActiveRun ? "Schema sync active" : "Schema sync logs"}
|
||||||
|
</Button>
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
variant="outline"
|
||||||
|
size="sm"
|
||||||
|
className="whitespace-nowrap"
|
||||||
|
aria-label="View AI description generation logs"
|
||||||
|
title="View AI description generation progress and history"
|
||||||
|
disabled={!canManage}
|
||||||
|
onClick={openDescriptionGenerationHistory}
|
||||||
|
>
|
||||||
|
<History aria-hidden="true" />
|
||||||
|
{descriptionGenerationActive ? "AI descriptions active" : "AI description logs"}
|
||||||
|
</Button>
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
variant="outline"
|
||||||
|
size="sm"
|
||||||
|
className="whitespace-nowrap"
|
||||||
|
aria-label="View sensitive analysis progress"
|
||||||
|
title="View sensitive analysis progress and history"
|
||||||
|
disabled={!canManage}
|
||||||
|
onClick={openSensitiveDataSuggestionHistory}
|
||||||
|
>
|
||||||
|
<History aria-hidden="true" />
|
||||||
|
{observedActiveSensitiveDataSuggestionRun ? "Sensitive analysis active" : "Sensitive analysis history"}
|
||||||
|
</Button>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
/>
|
/>
|
||||||
@@ -1144,9 +1219,17 @@ export function DatabaseManagementPage({
|
|||||||
navigation={(
|
navigation={(
|
||||||
<FleetLedgerBreadcrumb
|
<FleetLedgerBreadcrumb
|
||||||
items={fleetBreadcrumb}
|
items={fleetBreadcrumb}
|
||||||
back={screen.kind === "relationships" || (screen.kind === "tables" && !tableNestedNavigationActive)
|
back={screen.kind === "relationships"
|
||||||
? { label: "Back to databases", onBack: showList, disabled: navigationBusy }
|
? { label: "Back to databases", onBack: showList, disabled: navigationBusy }
|
||||||
: undefined}
|
: screen.kind === "tables" && tableNestedNavigationActive
|
||||||
|
? {
|
||||||
|
label: "Back to tables",
|
||||||
|
onBack: () => tableNestedNavigationBack?.(),
|
||||||
|
disabled: navigationBusy || !tableNestedNavigationBack,
|
||||||
|
}
|
||||||
|
: screen.kind === "tables"
|
||||||
|
? { label: "Back to databases", onBack: showList, disabled: navigationBusy }
|
||||||
|
: undefined}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
children={<FleetLedgerContent busy={isLoading || isFetching}>{fleetContent}</FleetLedgerContent>}
|
children={<FleetLedgerContent busy={isLoading || isFetching}>{fleetContent}</FleetLedgerContent>}
|
||||||
@@ -1246,10 +1329,12 @@ export function DatabaseManagementPage({
|
|||||||
onEdit={editRow}
|
onEdit={editRow}
|
||||||
onDelete={deleteRow}
|
onDelete={deleteRow}
|
||||||
onOpenSync={openSync}
|
onOpenSync={openSync}
|
||||||
|
onSelectionChange={(selected) => setSelectedDatabaseId(selected.length === 1 ? selected[0].id ?? null : null)}
|
||||||
onTestSelected={testSelected}
|
onTestSelected={testSelected}
|
||||||
onSyncSelected={syncSelected}
|
onSyncSelected={syncSelected}
|
||||||
selectedMetadataModel={selectedMetadataModelAvailable ? selectedMetadataModel : null}
|
selectedMetadataModel={selectedMetadataModelAvailable ? selectedMetadataModel : null}
|
||||||
descriptionGenerationActive={descriptionGenerationActive}
|
descriptionGenerationActive={descriptionGenerationActive}
|
||||||
|
sensitiveDataSuggestionRuns={sensitiveDataSuggestionRuns}
|
||||||
onGenerateDescriptions={generateDatabaseDescriptions}
|
onGenerateDescriptions={generateDatabaseDescriptions}
|
||||||
onSuggestSensitive={suggestDatabaseSensitiveFields}
|
onSuggestSensitive={suggestDatabaseSensitiveFields}
|
||||||
onDeleteMetadataSelected={deleteSelectedMetadata}
|
onDeleteMetadataSelected={deleteSelectedMetadata}
|
||||||
@@ -1304,10 +1389,9 @@ export function DatabaseManagementPage({
|
|||||||
onNavigationStateChange={setTablesNavigationState}
|
onNavigationStateChange={setTablesNavigationState}
|
||||||
onRunStarted={rememberSyncRun}
|
onRunStarted={rememberSyncRun}
|
||||||
onOpenSync={() => openSync(activeRow)}
|
onOpenSync={() => openSync(activeRow)}
|
||||||
onOpenDescriptionHistory={openDescriptionGenerationHistory}
|
|
||||||
onOpenSensitiveHistory={openSensitiveDataSuggestionHistory}
|
|
||||||
onDescriptionGenerationRunStarted={rememberDescriptionGenerationRun}
|
onDescriptionGenerationRunStarted={rememberDescriptionGenerationRun}
|
||||||
onSuggestSensitive={suggestActiveDatabaseSensitiveFields}
|
onSuggestSensitive={suggestActiveDatabaseSensitiveFields}
|
||||||
|
onClearCatalogTables={clearActiveCatalogTables}
|
||||||
onCatalogMetricsChanged={invalidateCatalogMetrics}
|
onCatalogMetricsChanged={invalidateCatalogMetrics}
|
||||||
/>
|
/>
|
||||||
) : null}
|
) : null}
|
||||||
|
|||||||
@@ -285,7 +285,12 @@ export function CatalogSyncDrawer({
|
|||||||
open
|
open
|
||||||
ariaLabel="Schema synchronization"
|
ariaLabel="Schema synchronization"
|
||||||
eyebrow="Schema synchronization"
|
eyebrow="Schema synchronization"
|
||||||
title={run ? `${stateLabel(run)[0].toUpperCase()}${stateLabel(run).slice(1)}` : "Synchronization history"}
|
title={run ? (
|
||||||
|
<span className="inline-flex items-center gap-2">
|
||||||
|
{!terminal(run) ? <LoaderCircle className="size-5 animate-spin" aria-hidden="true" /> : null}
|
||||||
|
{stateLabel(run)[0].toUpperCase()}{stateLabel(run).slice(1)}
|
||||||
|
</span>
|
||||||
|
) : "Synchronization history"}
|
||||||
description={run ? `${run.scope} · ${elapsed(run, now)}` : undefined}
|
description={run ? `${run.scope} · ${elapsed(run, now)}` : undefined}
|
||||||
onClose={onClose}
|
onClose={onClose}
|
||||||
closeLabel="Close synchronization drawer"
|
closeLabel="Close synchronization drawer"
|
||||||
|
|||||||
@@ -30,9 +30,7 @@ interface Props {
|
|||||||
descriptionGenerationActive: boolean;
|
descriptionGenerationActive: boolean;
|
||||||
onDescriptionGenerationRunStarted: (run: DescriptionGenerationRun) => void;
|
onDescriptionGenerationRunStarted: (run: DescriptionGenerationRun) => void;
|
||||||
onNavigationStateChange: (state: DatabaseNavigationState) => void;
|
onNavigationStateChange: (state: DatabaseNavigationState) => void;
|
||||||
onSync: () => void;
|
|
||||||
onSuggestSensitive: (selection: SensitiveDataSuggestionRequest, scopeLabel: string) => Promise<void>;
|
onSuggestSensitive: (selection: SensitiveDataSuggestionRequest, scopeLabel: string) => Promise<void>;
|
||||||
bindingReady?: boolean;
|
|
||||||
catalogOperationActive?: boolean;
|
catalogOperationActive?: boolean;
|
||||||
onCatalogMetricsChanged?: () => void | Promise<void>;
|
onCatalogMetricsChanged?: () => void | Promise<void>;
|
||||||
presentation?: "legacy" | "fleet";
|
presentation?: "legacy" | "fleet";
|
||||||
@@ -79,6 +77,7 @@ function SensitiveCell({ data, context }: ICellRendererParams<CatalogColumn, unk
|
|||||||
checked={data.sensitive}
|
checked={data.sensitive}
|
||||||
disabled={!context.canManage || context.busy}
|
disabled={!context.canManage || context.busy}
|
||||||
aria-label={`Sensitive data for ${data.name}`}
|
aria-label={`Sensitive data for ${data.name}`}
|
||||||
|
title="Mark this column as sensitive; this does not select it for actions."
|
||||||
onChange={(event) => context.onSensitiveChange(data, event.target.checked)}
|
onChange={(event) => context.onSensitiveChange(data, event.target.checked)}
|
||||||
onClick={(event) => event.stopPropagation()}
|
onClick={(event) => event.stopPropagation()}
|
||||||
/>
|
/>
|
||||||
@@ -94,9 +93,7 @@ export function DatabaseColumns({
|
|||||||
descriptionGenerationActive,
|
descriptionGenerationActive,
|
||||||
onDescriptionGenerationRunStarted,
|
onDescriptionGenerationRunStarted,
|
||||||
onNavigationStateChange,
|
onNavigationStateChange,
|
||||||
onSync,
|
|
||||||
onSuggestSensitive,
|
onSuggestSensitive,
|
||||||
bindingReady = true,
|
|
||||||
catalogOperationActive = false,
|
catalogOperationActive = false,
|
||||||
onCatalogMetricsChanged,
|
onCatalogMetricsChanged,
|
||||||
presentation = "legacy",
|
presentation = "legacy",
|
||||||
@@ -291,7 +288,6 @@ export function DatabaseColumns({
|
|||||||
| "generate-descriptions"
|
| "generate-descriptions"
|
||||||
| "consolidate-descriptions"
|
| "consolidate-descriptions"
|
||||||
| "suggest-sensitive"
|
| "suggest-sensitive"
|
||||||
| "sync-columns"
|
|
||||||
| "save-sensitive";
|
| "save-sensitive";
|
||||||
|
|
||||||
const fleetActions: readonly FleetActionOption<FleetColumnAction>[] = [
|
const fleetActions: readonly FleetActionOption<FleetColumnAction>[] = [
|
||||||
@@ -347,23 +343,6 @@ export function DatabaseColumns({
|
|||||||
? "Another action is running."
|
? "Another action is running."
|
||||||
: undefined,
|
: undefined,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
id: "sync-columns",
|
|
||||||
label: "Synchronize columns",
|
|
||||||
group: "Synchronization",
|
|
||||||
scopeLabel: "All columns in this table",
|
|
||||||
runLabel: "Synchronize",
|
|
||||||
disabled: !canManage || !bindingReady || catalogOperationActive || busy,
|
|
||||||
disabledReason: !canManage
|
|
||||||
? "You do not have permission to synchronize the catalog."
|
|
||||||
: !bindingReady
|
|
||||||
? "Test the current database binding first."
|
|
||||||
: catalogOperationActive
|
|
||||||
? "Wait for the active catalog operation to finish."
|
|
||||||
: busy
|
|
||||||
? "Another action is running."
|
|
||||||
: undefined,
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
id: "save-sensitive",
|
id: "save-sensitive",
|
||||||
label: "Save sensitive-field changes",
|
label: "Save sensitive-field changes",
|
||||||
@@ -386,7 +365,6 @@ export function DatabaseColumns({
|
|||||||
if (action === "generate-descriptions") await generateDescriptions();
|
if (action === "generate-descriptions") await generateDescriptions();
|
||||||
else if (action === "consolidate-descriptions") await consolidateDescriptions();
|
else if (action === "consolidate-descriptions") await consolidateDescriptions();
|
||||||
else if (action === "suggest-sensitive") await suggestSensitive();
|
else if (action === "suggest-sensitive") await suggestSensitive();
|
||||||
else if (action === "sync-columns") onSync();
|
|
||||||
else await saveSensitive();
|
else await saveSensitive();
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -397,7 +375,7 @@ export function DatabaseColumns({
|
|||||||
|
|
||||||
const columns = useMemo<ColDef<CatalogColumn>[]>(() => [
|
const columns = useMemo<ColDef<CatalogColumn>[]>(() => [
|
||||||
{ field: "ordinalPosition", headerName: "#", width: 64, maxWidth: 64, filter: "agNumberColumnFilter" },
|
{ field: "ordinalPosition", headerName: "#", width: 64, maxWidth: 64, filter: "agNumberColumnFilter" },
|
||||||
{ field: "sensitive", headerName: "Sensitive", minWidth: 110, width: 110, sortable: false, filter: false, resizable: false, cellRenderer: SensitiveCell },
|
{ field: "sensitive", headerName: "Sensitive", headerTooltip: "Mark this column as sensitive; this does not select it for actions.", minWidth: 110, width: 110, sortable: false, filter: false, resizable: false, cellRenderer: SensitiveCell },
|
||||||
{ field: "name", headerName: "Name", minWidth: 190, flex: 1, cellClass: "font-mono text-xs" },
|
{ field: "name", headerName: "Name", minWidth: 190, flex: 1, cellClass: "font-mono text-xs" },
|
||||||
{ field: "dataType", headerName: "Type", minWidth: 150, flex: 0.8, cellClass: "font-mono text-xs" },
|
{ field: "dataType", headerName: "Type", minWidth: 150, flex: 0.8, cellClass: "font-mono text-xs" },
|
||||||
{ headerName: "Keys", minWidth: 125, width: 125, sortable: false, filter: false, cellRenderer: KeyCell },
|
{ headerName: "Keys", minWidth: 125, width: 125, sortable: false, filter: false, cellRenderer: KeyCell },
|
||||||
@@ -512,7 +490,6 @@ export function DatabaseColumns({
|
|||||||
<span className="text-xs tabular-nums text-muted-foreground">{data.length}</span>
|
<span className="text-xs tabular-nums text-muted-foreground">{data.length}</span>
|
||||||
{changedSensitiveColumns.length > 0 ? <Button type="button" disabled={!canManage || busy} onClick={() => void saveSensitive()}><Save />{sensitiveAction === "save" ? "Saving…" : "Save sensitive fields"}</Button> : null}
|
{changedSensitiveColumns.length > 0 ? <Button type="button" disabled={!canManage || busy} onClick={() => void saveSensitive()}><Save />{sensitiveAction === "save" ? "Saving…" : "Save sensitive fields"}</Button> : null}
|
||||||
<Button type="button" variant="outline" disabled={isFetching || busy} onClick={() => void refetch()}><RefreshCw className={isFetching ? "animate-spin" : ""} />Refresh</Button>
|
<Button type="button" variant="outline" disabled={isFetching || busy} onClick={() => void refetch()}><RefreshCw className={isFetching ? "animate-spin" : ""} />Refresh</Button>
|
||||||
<Button type="button" disabled={!canManage || busy} onClick={onSync}><RefreshCw />Sync columns</Button>
|
|
||||||
</>
|
</>
|
||||||
)
|
)
|
||||||
)}
|
)}
|
||||||
@@ -529,6 +506,17 @@ export function DatabaseColumns({
|
|||||||
defaultColDef={{ sortable: true, filter: true, resizable: true }}
|
defaultColDef={{ sortable: true, filter: true, resizable: true }}
|
||||||
getRowId={({ data: row }) => row.id}
|
getRowId={({ data: row }) => row.id}
|
||||||
rowSelection={{ mode: "multiRow", selectAll: "filtered", enableClickSelection: false }}
|
rowSelection={{ mode: "multiRow", selectAll: "filtered", enableClickSelection: false }}
|
||||||
|
selectionColumnDef={{
|
||||||
|
headerName: "Select",
|
||||||
|
headerTooltip: "Select columns for actions",
|
||||||
|
width: 72,
|
||||||
|
minWidth: 72,
|
||||||
|
maxWidth: 72,
|
||||||
|
pinned: "left",
|
||||||
|
sortable: false,
|
||||||
|
resizable: false,
|
||||||
|
suppressMovable: true,
|
||||||
|
}}
|
||||||
onSelectionChanged={({ api }) => setSelectedIds(api.getSelectedRows().map((column) => column.id))}
|
onSelectionChanged={({ api }) => setSelectedIds(api.getSelectedRows().map((column) => column.id))}
|
||||||
rowHeight={44}
|
rowHeight={44}
|
||||||
headerHeight={38}
|
headerHeight={38}
|
||||||
|
|||||||
@@ -115,7 +115,6 @@ test("Fleet columns distinguish a failed query from an empty catalog and retry i
|
|||||||
descriptionGenerationActive={false}
|
descriptionGenerationActive={false}
|
||||||
onDescriptionGenerationRunStarted={noop}
|
onDescriptionGenerationRunStarted={noop}
|
||||||
onNavigationStateChange={noop}
|
onNavigationStateChange={noop}
|
||||||
onSync={noop}
|
|
||||||
onSuggestSensitive={noopAsync}
|
onSuggestSensitive={noopAsync}
|
||||||
/>,
|
/>,
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -12,7 +12,6 @@ import { Button } from "../../components/ui/button";
|
|||||||
import type {
|
import type {
|
||||||
CatalogDatabase,
|
CatalogDatabase,
|
||||||
CatalogSecretName,
|
CatalogSecretName,
|
||||||
CatalogSyncScope,
|
|
||||||
CatalogSyncRun,
|
CatalogSyncRun,
|
||||||
DatabaseBinding,
|
DatabaseBinding,
|
||||||
DatabaseTransport,
|
DatabaseTransport,
|
||||||
@@ -23,7 +22,7 @@ import type {
|
|||||||
DatabaseFormMode,
|
DatabaseFormMode,
|
||||||
} from "./model";
|
} from "./model";
|
||||||
import { statusLabel } from "./model";
|
import { statusLabel } from "./model";
|
||||||
import { DatabaseSyncMenu } from "./DatabaseSyncMenu";
|
import { DatabaseSyncMenu, type DatabaseSyncScope } from "./DatabaseSyncMenu";
|
||||||
|
|
||||||
const inputClass = "h-9 w-full rounded-md border border-input bg-card px-3 text-sm outline-none transition focus:border-primary/60 focus:ring-3 focus:ring-ring/15 read-only:bg-muted/40 read-only:text-muted-foreground disabled:bg-muted disabled:text-muted-foreground";
|
const inputClass = "h-9 w-full rounded-md border border-input bg-card px-3 text-sm outline-none transition focus:border-primary/60 focus:ring-3 focus:ring-ring/15 read-only:bg-muted/40 read-only:text-muted-foreground disabled:bg-muted disabled:text-muted-foreground";
|
||||||
const labelClass = "grid min-w-0 gap-1.5 text-xs font-medium text-foreground";
|
const labelClass = "grid min-w-0 gap-1.5 text-xs font-medium text-foreground";
|
||||||
@@ -135,7 +134,7 @@ interface DatabaseFormProps {
|
|||||||
onRetrySecrets: () => void;
|
onRetrySecrets: () => void;
|
||||||
onOpenTables: (origin: HTMLButtonElement) => void;
|
onOpenTables: (origin: HTMLButtonElement) => void;
|
||||||
onOpenRelationships: () => void;
|
onOpenRelationships: () => void;
|
||||||
onSync: (scope: CatalogSyncScope) => void;
|
onSync: (scope: DatabaseSyncScope) => void;
|
||||||
onOpenSync: () => void;
|
onOpenSync: () => void;
|
||||||
onOpenDescriptionHistory?: () => void;
|
onOpenDescriptionHistory?: () => void;
|
||||||
onOpenSensitiveHistory?: () => void;
|
onOpenSensitiveHistory?: () => void;
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { useEffect, useMemo, useRef, useState, type RefObject } from "react";
|
import { useEffect, useMemo, useRef, useState, type RefObject } from "react";
|
||||||
|
import { useQueries } from "@tanstack/react-query";
|
||||||
import { Menu } from "@base-ui/react/menu";
|
import { Menu } from "@base-ui/react/menu";
|
||||||
import { AgGridReact } from "ag-grid-react";
|
import { AgGridReact } from "ag-grid-react";
|
||||||
import {
|
import {
|
||||||
@@ -11,13 +12,15 @@ import "ag-grid-community/styles/ag-grid.css";
|
|||||||
import "ag-grid-community/styles/ag-theme-alpine.css";
|
import "ag-grid-community/styles/ag-theme-alpine.css";
|
||||||
import { ChevronDown, Eye, History, Info, Link2, Pencil, RefreshCw, Rows3, Sparkles, Trash2, X } from "lucide-react";
|
import { ChevronDown, Eye, History, Info, Link2, Pencil, RefreshCw, Rows3, Sparkles, Trash2, X } from "lucide-react";
|
||||||
import { Button } from "../../components/ui/button";
|
import { Button } from "../../components/ui/button";
|
||||||
|
import { getCatalogMetrics } from "../../api/catalog-databases";
|
||||||
import type {
|
import type {
|
||||||
CatalogDatabase,
|
CatalogDatabase,
|
||||||
|
CatalogMetrics,
|
||||||
|
SensitiveDataSuggestionRun,
|
||||||
CatalogDatabaseMetadataDeleteTarget,
|
CatalogDatabaseMetadataDeleteTarget,
|
||||||
CatalogSyncScope,
|
|
||||||
DescriptionGenerationScope,
|
DescriptionGenerationScope,
|
||||||
} from "../../api/catalog-databases";
|
} from "../../api/catalog-databases";
|
||||||
import { databaseSyncItemClass, databaseSyncScopes } from "./DatabaseSyncMenu";
|
import { databaseSyncItemClass, databaseSyncScopes, type DatabaseSyncScope } from "./DatabaseSyncMenu";
|
||||||
import { FleetActionSelector, type FleetActionOption } from "./FleetActionSelector";
|
import { FleetActionSelector, type FleetActionOption } from "./FleetActionSelector";
|
||||||
import { NO_METADATA_GENERATION_LLM_MODEL_MESSAGE } from "./MetadataGenerationModelSelector";
|
import { NO_METADATA_GENERATION_LLM_MODEL_MESSAGE } from "./MetadataGenerationModelSelector";
|
||||||
|
|
||||||
@@ -37,10 +40,12 @@ interface DatabaseGridProps {
|
|||||||
onEdit: (row: CatalogDatabase, origin: HTMLButtonElement) => void;
|
onEdit: (row: CatalogDatabase, origin: HTMLButtonElement) => void;
|
||||||
onDelete: (row: CatalogDatabase, origin: HTMLButtonElement) => void;
|
onDelete: (row: CatalogDatabase, origin: HTMLButtonElement) => void;
|
||||||
onOpenSync: (row: CatalogDatabase) => void;
|
onOpenSync: (row: CatalogDatabase) => void;
|
||||||
|
onSelectionChange?: (rows: CatalogDatabase[]) => void;
|
||||||
onTestSelected: (rows: CatalogDatabase[]) => Promise<void>;
|
onTestSelected: (rows: CatalogDatabase[]) => Promise<void>;
|
||||||
onSyncSelected: (rows: CatalogDatabase[], scope: CatalogSyncScope) => Promise<void>;
|
onSyncSelected: (rows: CatalogDatabase[], scope: DatabaseSyncScope) => Promise<void>;
|
||||||
selectedMetadataModel: string | null;
|
selectedMetadataModel: string | null;
|
||||||
descriptionGenerationActive: boolean;
|
descriptionGenerationActive: boolean;
|
||||||
|
sensitiveDataSuggestionRuns?: SensitiveDataSuggestionRun[];
|
||||||
onGenerateDescriptions: (
|
onGenerateDescriptions: (
|
||||||
rows: CatalogDatabase[],
|
rows: CatalogDatabase[],
|
||||||
scope: Extract<DescriptionGenerationScope, "all" | "missing">,
|
scope: Extract<DescriptionGenerationScope, "all" | "missing">,
|
||||||
@@ -68,7 +73,6 @@ interface DatabaseGridContext {
|
|||||||
type DatabaseFleetAction =
|
type DatabaseFleetAction =
|
||||||
| "test"
|
| "test"
|
||||||
| "sync-tables"
|
| "sync-tables"
|
||||||
| "sync-columns"
|
|
||||||
| "sync-relationships"
|
| "sync-relationships"
|
||||||
| "sync-all"
|
| "sync-all"
|
||||||
| "generate-all"
|
| "generate-all"
|
||||||
@@ -124,87 +128,63 @@ function StateCell({
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function evidenceStatus(row: CatalogDatabase): { label: string; tone: StateTone } {
|
function endpoint(host: string | undefined, port: number | undefined): string {
|
||||||
if (!row.workspaceEvidence) {
|
if (!host) return "Endpoint not configured";
|
||||||
return { label: "Evidence state unavailable", tone: "warning" };
|
return port ? `${host}:${port}` : host;
|
||||||
}
|
|
||||||
switch (row.workspaceEvidence.state) {
|
|
||||||
case "materialized_current_revision":
|
|
||||||
return { label: "Evidence materialized", tone: "success" };
|
|
||||||
case "configuration_required":
|
|
||||||
return { label: "Evidence credentials required", tone: "warning" };
|
|
||||||
case "configured_unverified":
|
|
||||||
return { label: "Evidence configured, unverified", tone: "info" };
|
|
||||||
case "workspace_unavailable":
|
|
||||||
return { label: "Evidence unavailable", tone: "danger" };
|
|
||||||
default:
|
|
||||||
return { label: "No Evidence declared", tone: "neutral" };
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function RevisionEvidenceCell({ row }: { row: CatalogDatabase }) {
|
function accessSummary(row: CatalogDatabase): { label: string; detail: string } {
|
||||||
if (!row.workspaceRevision) {
|
const transport = row.configured
|
||||||
return <StateCell label="Workspace missing" detail="Revision and Evidence unavailable" tone="danger" />;
|
? row.binding.transport
|
||||||
|
: row.runtimeBinding?.transport ?? row.binding.transport;
|
||||||
|
if (transport === "rest_api") {
|
||||||
|
if (!row.binding.baseUrl) return { label: "REST", detail: "Endpoint not configured" };
|
||||||
|
try {
|
||||||
|
return { label: "REST", detail: new URL(row.binding.baseUrl).host };
|
||||||
|
} catch {
|
||||||
|
return { label: "REST", detail: row.binding.baseUrl };
|
||||||
|
}
|
||||||
}
|
}
|
||||||
const evidence = evidenceStatus(row);
|
if (transport === "ssh_tunnel") {
|
||||||
const source = row.workspaceEvidence?.sourceType ? ` · ${row.workspaceEvidence.sourceType}` : "";
|
return { label: "SSH", detail: endpoint(row.binding.sshHost, row.binding.sshPort) };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
label: "Direct PostgreSQL",
|
||||||
|
detail: endpoint(row.binding.host, row.binding.port),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function coverageStatus(total: number, complete: number): { label: string; detail: string; tone: StateTone } {
|
||||||
|
if (total === 0) return { label: "Not started", detail: "No catalog items", tone: "danger" };
|
||||||
|
if (complete >= total) return { label: "Complete", detail: `${complete}/${total}`, tone: "success" };
|
||||||
|
if (complete > 0) return { label: "Partial", detail: `${complete}/${total}`, tone: "warning" };
|
||||||
|
return { label: "Not started", detail: `0/${total}`, tone: "danger" };
|
||||||
|
}
|
||||||
|
|
||||||
|
function CatalogStatusCells({ row, metrics, sensitiveRun }: { row: CatalogDatabase; metrics?: CatalogMetrics; sensitiveRun?: SensitiveDataSuggestionRun }) {
|
||||||
|
const access = accessSummary(row);
|
||||||
|
const synchronized = row.schemaSyncedVersion === row.version;
|
||||||
|
const syncStatus = !row.schemaSyncedVersion
|
||||||
|
? { label: "Not synchronized", detail: "Schema not synchronized", tone: "danger" as const }
|
||||||
|
: synchronized
|
||||||
|
? { label: "Synchronized", detail: "Current schema", tone: "success" as const }
|
||||||
|
: { label: "Partial", detail: "Schema revision changed", tone: "warning" as const };
|
||||||
|
const descriptions = coverageStatus(metrics?.descriptionTargets ?? 0, metrics?.describedTargets ?? 0);
|
||||||
|
const sensitiveProcessed = sensitiveRun ? sensitiveRun.suggestedSensitive + sensitiveRun.suggestedNonSensitive : 0;
|
||||||
|
const sensitive = !sensitiveRun
|
||||||
|
? { label: "Not started", detail: "No analysis run", tone: "danger" as const }
|
||||||
|
: sensitiveRun.status === "completed" && sensitiveProcessed >= sensitiveRun.total
|
||||||
|
? { label: "Complete", detail: `${sensitiveProcessed}/${sensitiveRun.total}`, tone: "success" as const }
|
||||||
|
: ["failed", "interrupted"].includes(sensitiveRun.status)
|
||||||
|
? { label: "Incomplete", detail: `${sensitiveProcessed}/${sensitiveRun.total}`, tone: "danger" as const }
|
||||||
|
: { label: "Partial", detail: `${sensitiveProcessed}/${sensitiveRun.total}`, tone: "warning" as const };
|
||||||
return (
|
return (
|
||||||
<StateCell
|
<div className="grid h-full min-w-[540px] grid-cols-4 gap-3 py-1">
|
||||||
label={`Active revision ${row.workspaceRevision.commit.slice(0, 7)}`}
|
<StateCell label={access.label} detail={access.detail} tone="info" />
|
||||||
detail={`${evidence.label}${source}`}
|
<StateCell label={syncStatus.label} detail={syncStatus.detail} tone={syncStatus.tone} />
|
||||||
tone={evidence.tone}
|
<StateCell label={descriptions.label} detail={`Descriptions · ${descriptions.detail}`} tone={descriptions.tone} />
|
||||||
/>
|
<StateCell label={sensitive.label} detail={`Sensitive · ${sensitive.detail}`} tone={sensitive.tone} />
|
||||||
);
|
</div>
|
||||||
}
|
|
||||||
|
|
||||||
function transportLabel(row: CatalogDatabase): string {
|
|
||||||
const transport = row.runtimeBinding?.transport;
|
|
||||||
if (transport === "postgres_direct") return "Direct PostgreSQL";
|
|
||||||
if (transport === "rest_api") return "REST API";
|
|
||||||
if (transport === "ssh_tunnel") return "SSH tunnel";
|
|
||||||
return "No runtime binding";
|
|
||||||
}
|
|
||||||
|
|
||||||
function RuntimeBindingCell({ row }: { row: CatalogDatabase }) {
|
|
||||||
if (!row.runtimeBinding) {
|
|
||||||
return <StateCell label="Unavailable" detail="Workspace is not in the active revision" tone="danger" />;
|
|
||||||
}
|
|
||||||
if (!row.runtimeBinding.sessionTransportSupported) {
|
|
||||||
return <StateCell label="Unsupported for NL→SQL" detail={transportLabel(row)} tone="danger" />;
|
|
||||||
}
|
|
||||||
if (row.runtimeBinding.configurationState === "configuration_required") {
|
|
||||||
return <StateCell label="Configuration required" detail={transportLabel(row)} tone="warning" />;
|
|
||||||
}
|
|
||||||
return <StateCell label="Ready" detail={transportLabel(row)} tone="success" />;
|
|
||||||
}
|
|
||||||
|
|
||||||
function catalogConnectionDetail(row: CatalogDatabase): { label: string; className?: string } {
|
|
||||||
if (row.activeSyncRun) return { label: "Metadata sync in progress", className: "text-primary" };
|
|
||||||
if (row.testedVersion !== undefined && row.testedVersion !== row.version) {
|
|
||||||
return { label: "Connection retest required", className: "text-amber-700 dark:text-amber-300" };
|
|
||||||
}
|
|
||||||
if (row.connectionStatus === "reachable") return { label: "Connection reachable" };
|
|
||||||
if (row.connectionStatus === "failed") {
|
|
||||||
return { label: "Connection test failed", className: "text-destructive" };
|
|
||||||
}
|
|
||||||
return { label: "Connection not tested" };
|
|
||||||
}
|
|
||||||
|
|
||||||
function MetadataCatalogCell({ row }: { row: CatalogDatabase }) {
|
|
||||||
if (!row.workspaceAvailable) {
|
|
||||||
return <StateCell label="Orphaned configuration" detail="Workspace missing from repository" tone="danger" />;
|
|
||||||
}
|
|
||||||
if (!row.configured) {
|
|
||||||
return <StateCell label="Not configured" detail="Configure this workspace" tone="neutral" />;
|
|
||||||
}
|
|
||||||
const connection = catalogConnectionDetail(row);
|
|
||||||
return (
|
|
||||||
<StateCell
|
|
||||||
label="Configured"
|
|
||||||
detail={connection.label}
|
|
||||||
detailClassName={connection.className}
|
|
||||||
tone="success"
|
|
||||||
/>
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -259,11 +239,6 @@ function DatabaseActionsCell({
|
|||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="flex h-full items-center justify-end gap-0.5" onClick={(event) => event.stopPropagation()}>
|
<div className="flex h-full items-center justify-end gap-0.5" onClick={(event) => event.stopPropagation()}>
|
||||||
{data.activeSyncRun || (context.presentation === "fleet" && data.configured && data.id) ? (
|
|
||||||
<Button type="button" variant="ghost" size="icon-lg" className={context.presentation === "fleet" ? "thot-fleet-icon-action" : undefined} data-tooltip={context.presentation === "fleet" ? "Synchronization history" : undefined} title={context.presentation === "legacy" ? `View synchronization for ${data.workspaceName}` : undefined} aria-label={`View synchronization history for ${data.workspaceName}`} onClick={() => context.onOpenSync(data)}>
|
|
||||||
{data.activeSyncRun ? <RefreshCw className="animate-spin" aria-hidden="true" /> : <History aria-hidden="true" />}
|
|
||||||
</Button>
|
|
||||||
) : null}
|
|
||||||
{context.presentation === "fleet" && context.onOpenTables ? (
|
{context.presentation === "fleet" && context.onOpenTables ? (
|
||||||
<span className="thot-fleet-icon-action inline-flex" data-tooltip="Tables">
|
<span className="thot-fleet-icon-action inline-flex" data-tooltip="Tables">
|
||||||
<Button
|
<Button
|
||||||
@@ -357,16 +332,37 @@ export function DatabaseGrid({
|
|||||||
onEdit,
|
onEdit,
|
||||||
onDelete,
|
onDelete,
|
||||||
onOpenSync,
|
onOpenSync,
|
||||||
|
onSelectionChange,
|
||||||
onTestSelected,
|
onTestSelected,
|
||||||
onSyncSelected,
|
onSyncSelected,
|
||||||
selectedMetadataModel,
|
selectedMetadataModel,
|
||||||
descriptionGenerationActive,
|
descriptionGenerationActive,
|
||||||
|
sensitiveDataSuggestionRuns = [],
|
||||||
onGenerateDescriptions,
|
onGenerateDescriptions,
|
||||||
onSuggestSensitive,
|
onSuggestSensitive,
|
||||||
onDeleteMetadataSelected,
|
onDeleteMetadataSelected,
|
||||||
onRefresh,
|
onRefresh,
|
||||||
presentation = "legacy",
|
presentation = "legacy",
|
||||||
}: DatabaseGridProps) {
|
}: DatabaseGridProps) {
|
||||||
|
const metricQueries = useQueries({
|
||||||
|
queries: rows.filter((row) => row.id).map((row) => ({
|
||||||
|
queryKey: ["catalog-metrics", row.id],
|
||||||
|
queryFn: () => getCatalogMetrics(row.id!),
|
||||||
|
enabled: canManage,
|
||||||
|
staleTime: 10_000,
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
const metricsByDatabase = useMemo(() => {
|
||||||
|
const result: Record<string, CatalogMetrics> = {};
|
||||||
|
rows.filter((row) => row.id).forEach((row, index) => {
|
||||||
|
const metrics = metricQueries[index]?.data;
|
||||||
|
if (metrics && row.id) result[row.id] = metrics;
|
||||||
|
});
|
||||||
|
return result;
|
||||||
|
}, [metricQueries, rows]);
|
||||||
|
const sensitiveRunsByDatabase = useMemo(() => new Map(
|
||||||
|
rows.filter((row) => row.id).map((row) => [row.id!, sensitiveDataSuggestionRuns.find((run) => run.databaseId === row.id)]),
|
||||||
|
), [rows, sensitiveDataSuggestionRuns]);
|
||||||
const compact = useCompactViewport();
|
const compact = useCompactViewport();
|
||||||
const gridRef = useRef<AgGridReact<CatalogDatabase>>(null);
|
const gridRef = useRef<AgGridReact<CatalogDatabase>>(null);
|
||||||
const actionsTriggerRef = useRef<HTMLButtonElement>(null);
|
const actionsTriggerRef = useRef<HTMLButtonElement>(null);
|
||||||
@@ -390,68 +386,16 @@ export function DatabaseGrid({
|
|||||||
flex: compact ? undefined : 1.35,
|
flex: compact ? undefined : 1.35,
|
||||||
width: compact ? 112 : undefined,
|
width: compact ? 112 : undefined,
|
||||||
};
|
};
|
||||||
const revisionEvidence: ColDef<CatalogDatabase> = {
|
const status: ColDef<CatalogDatabase> = {
|
||||||
headerName: "Revision / Evidence",
|
headerName: "Catalog status",
|
||||||
minWidth: 215,
|
minWidth: 580,
|
||||||
flex: 1.2,
|
flex: 2.5,
|
||||||
valueGetter: ({ data }) => {
|
cellRenderer: ({ data }: ICellRendererParams<CatalogDatabase>) => {
|
||||||
if (!data) return "";
|
const databaseId = data?.id;
|
||||||
const revision = data.workspaceRevision?.commit.slice(0, 7) ?? "workspace missing";
|
return data
|
||||||
return `${revision} ${evidenceStatus(data).label}`;
|
? <CatalogStatusCells row={data} metrics={databaseId ? metricsByDatabase[databaseId] : undefined} sensitiveRun={databaseId ? sensitiveRunsByDatabase.get(databaseId) : undefined} />
|
||||||
|
: null;
|
||||||
},
|
},
|
||||||
cellRenderer: ({ data }: ICellRendererParams<CatalogDatabase>) => (
|
|
||||||
data ? <RevisionEvidenceCell row={data} /> : null
|
|
||||||
),
|
|
||||||
};
|
|
||||||
const runtimeBinding: ColDef<CatalogDatabase> = {
|
|
||||||
headerName: "NL→SQL runtime",
|
|
||||||
minWidth: 175,
|
|
||||||
flex: 1,
|
|
||||||
valueGetter: ({ data }) => {
|
|
||||||
if (!data?.runtimeBinding) return "Unavailable";
|
|
||||||
if (!data.runtimeBinding.sessionTransportSupported) return "Unsupported for NL→SQL";
|
|
||||||
return data.runtimeBinding.configurationState === "ready" ? "Ready" : "Configuration required";
|
|
||||||
},
|
|
||||||
cellRenderer: ({ data }: ICellRendererParams<CatalogDatabase>) => (
|
|
||||||
data ? <RuntimeBindingCell row={data} /> : null
|
|
||||||
),
|
|
||||||
};
|
|
||||||
const database: ColDef<CatalogDatabase> = {
|
|
||||||
field: "databaseName", headerName: "Database", minWidth: 145, flex: 1,
|
|
||||||
};
|
|
||||||
const schema: ColDef<CatalogDatabase> = {
|
|
||||||
field: "schema", headerName: "Schema", minWidth: 140, flex: 0.9,
|
|
||||||
};
|
|
||||||
const endpoint: ColDef<CatalogDatabase> = {
|
|
||||||
headerName: "Endpoint",
|
|
||||||
minWidth: 190,
|
|
||||||
flex: 1.2,
|
|
||||||
valueGetter: ({ data }) => {
|
|
||||||
if (!data) return "";
|
|
||||||
if (data.binding.transport === "rest_api") return data.binding.baseUrl ?? "";
|
|
||||||
if (data.binding.transport === "ssh_tunnel") return data.binding.sshHost ?? "";
|
|
||||||
return data.binding.host ? `${data.binding.host}:${data.binding.port ?? 5432}` : "";
|
|
||||||
},
|
|
||||||
};
|
|
||||||
const metadataCatalog: ColDef<CatalogDatabase> = {
|
|
||||||
headerName: "Metadata Catalog",
|
|
||||||
minWidth: 185,
|
|
||||||
flex: 1.05,
|
|
||||||
valueGetter: ({ data }) => {
|
|
||||||
if (!data) return "";
|
|
||||||
if (!data.workspaceAvailable) return "Orphaned configuration";
|
|
||||||
return data.configured ? `Configured ${catalogConnectionDetail(data).label}` : "Not configured";
|
|
||||||
},
|
|
||||||
cellRenderer: ({ data }: ICellRendererParams<CatalogDatabase>) => (
|
|
||||||
data ? <MetadataCatalogCell row={data} /> : null
|
|
||||||
),
|
|
||||||
};
|
|
||||||
const updated: ColDef<CatalogDatabase> = {
|
|
||||||
field: "updatedAt",
|
|
||||||
headerName: "Updated",
|
|
||||||
minWidth: 170,
|
|
||||||
flex: 0.9,
|
|
||||||
valueFormatter: ({ value }) => value ? new Date(String(value)).toLocaleString() : "",
|
|
||||||
};
|
};
|
||||||
const actionsWidth = presentation === "fleet" ? 232 : 210;
|
const actionsWidth = presentation === "fleet" ? 232 : 210;
|
||||||
const actions: ColDef<CatalogDatabase> = {
|
const actions: ColDef<CatalogDatabase> = {
|
||||||
@@ -471,10 +415,8 @@ export function DatabaseGrid({
|
|||||||
cellRenderer: DatabaseActionsCell,
|
cellRenderer: DatabaseActionsCell,
|
||||||
};
|
};
|
||||||
|
|
||||||
return compact
|
return [workspace, status, actions];
|
||||||
? [workspace, revisionEvidence, runtimeBinding, metadataCatalog, database, schema, endpoint, updated, actions]
|
}, [compact, metricsByDatabase, presentation, sensitiveRunsByDatabase]);
|
||||||
: [workspace, revisionEvidence, runtimeBinding, database, schema, endpoint, metadataCatalog, updated, actions];
|
|
||||||
}, [compact, presentation]);
|
|
||||||
|
|
||||||
const configuredCount = rows.filter((row) => row.configured).length;
|
const configuredCount = rows.filter((row) => row.configured).length;
|
||||||
const hiddenSelected = selectedRows.filter((row) => {
|
const hiddenSelected = selectedRows.filter((row) => {
|
||||||
@@ -533,7 +475,6 @@ export function DatabaseGrid({
|
|||||||
return [
|
return [
|
||||||
{ id: "test", label: "Test connections", group: "Connection", disabled: !canTestSelection, disabledReason: testReason },
|
{ id: "test", label: "Test connections", group: "Connection", disabled: !canTestSelection, disabledReason: testReason },
|
||||||
{ id: "sync-tables", label: "Synchronize tables", group: "Synchronization", disabled: !canSyncSelection, disabledReason: syncReason },
|
{ id: "sync-tables", label: "Synchronize tables", group: "Synchronization", disabled: !canSyncSelection, disabledReason: syncReason },
|
||||||
{ id: "sync-columns", label: "Synchronize all columns", group: "Synchronization", disabled: !canSyncSelection, disabledReason: syncReason },
|
|
||||||
{ id: "sync-relationships", label: "Synchronize relationships", group: "Synchronization", disabled: !canSyncSelection, disabledReason: syncReason },
|
{ id: "sync-relationships", label: "Synchronize relationships", group: "Synchronization", disabled: !canSyncSelection, disabledReason: syncReason },
|
||||||
{ id: "sync-all", label: "Synchronize all", group: "Synchronization", disabled: !canSyncSelection, disabledReason: syncReason },
|
{ id: "sync-all", label: "Synchronize all", group: "Synchronization", disabled: !canSyncSelection, disabledReason: syncReason },
|
||||||
{ id: "generate-missing", label: "Generate missing descriptions", group: "Descriptions", disabled: !canGenerateDescriptions, disabledReason: generationReason },
|
{ id: "generate-missing", label: "Generate missing descriptions", group: "Descriptions", disabled: !canGenerateDescriptions, disabledReason: generationReason },
|
||||||
@@ -580,15 +521,10 @@ export function DatabaseGrid({
|
|||||||
const perform = async (
|
const perform = async (
|
||||||
kind: "test" | "sync" | "generate" | "suggest" | "delete",
|
kind: "test" | "sync" | "generate" | "suggest" | "delete",
|
||||||
operation: () => Promise<void>,
|
operation: () => Promise<void>,
|
||||||
clearSelection = true,
|
|
||||||
) => {
|
) => {
|
||||||
setAction(kind);
|
setAction(kind);
|
||||||
try {
|
try {
|
||||||
await operation();
|
await operation();
|
||||||
if (clearSelection) {
|
|
||||||
gridRef.current?.api.deselectAll();
|
|
||||||
setSelectedRows([]);
|
|
||||||
}
|
|
||||||
setPendingDelete(null);
|
setPendingDelete(null);
|
||||||
setPendingGenerationScope(null);
|
setPendingGenerationScope(null);
|
||||||
if (kind === "delete") window.setTimeout(() => searchInputRef.current?.focus(), 0);
|
if (kind === "delete") window.setTimeout(() => searchInputRef.current?.focus(), 0);
|
||||||
@@ -612,11 +548,11 @@ export function DatabaseGrid({
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (selectedAction.startsWith("sync-")) {
|
if (selectedAction.startsWith("sync-")) {
|
||||||
const scope = selectedAction.slice("sync-".length) as CatalogSyncScope;
|
const scope = selectedAction.slice("sync-".length) as DatabaseSyncScope;
|
||||||
await perform("sync", () => onSyncSelected(selectedRows, scope));
|
await perform("sync", () => onSyncSelected(selectedRows, scope));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
await perform("suggest", () => onSuggestSensitive(selectedRows), false);
|
await perform("suggest", () => onSuggestSensitive(selectedRows));
|
||||||
};
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
@@ -785,7 +721,7 @@ export function DatabaseGrid({
|
|||||||
variant="outline"
|
variant="outline"
|
||||||
disabled={!canManage || action !== null || descriptionGenerationActive}
|
disabled={!canManage || action !== null || descriptionGenerationActive}
|
||||||
title={descriptionGenerationActive ? "Wait for the active description generation to finish" : undefined}
|
title={descriptionGenerationActive ? "Wait for the active description generation to finish" : undefined}
|
||||||
onClick={() => void perform("suggest", () => onSuggestSensitive(selectedRows), false)}
|
onClick={() => void perform("suggest", () => onSuggestSensitive(selectedRows))}
|
||||||
>
|
>
|
||||||
<Sparkles />{action === "suggest" ? "Suggesting…" : "Suggest sensitive fields"}
|
<Sparkles />{action === "suggest" ? "Suggesting…" : "Suggest sensitive fields"}
|
||||||
</Button>
|
</Button>
|
||||||
@@ -833,7 +769,9 @@ export function DatabaseGrid({
|
|||||||
onSelectionChanged={({ api }) => {
|
onSelectionChanged={({ api }) => {
|
||||||
if (pendingDelete && action === null) setPendingDelete(null);
|
if (pendingDelete && action === null) setPendingDelete(null);
|
||||||
if (pendingGenerationScope && action === null) setPendingGenerationScope(null);
|
if (pendingGenerationScope && action === null) setPendingGenerationScope(null);
|
||||||
setSelectedRows(api.getSelectedRows());
|
const rows = api.getSelectedRows();
|
||||||
|
setSelectedRows(rows);
|
||||||
|
onSelectionChange?.(rows);
|
||||||
}}
|
}}
|
||||||
rowHeight={52}
|
rowHeight={52}
|
||||||
headerHeight={38}
|
headerHeight={38}
|
||||||
|
|||||||
@@ -3,14 +3,15 @@ import { ChevronDown, RefreshCw } from "lucide-react";
|
|||||||
import { buttonVariants } from "../../components/ui/button";
|
import { buttonVariants } from "../../components/ui/button";
|
||||||
import type { CatalogSyncScope } from "../../api/catalog-databases";
|
import type { CatalogSyncScope } from "../../api/catalog-databases";
|
||||||
|
|
||||||
|
export type DatabaseSyncScope = Exclude<CatalogSyncScope, "columns">;
|
||||||
|
|
||||||
export const databaseSyncItemClass = [
|
export const databaseSyncItemClass = [
|
||||||
"flex cursor-default select-none items-center rounded-md px-3 py-2 text-sm outline-none",
|
"flex cursor-default select-none items-center rounded-md px-3 py-2 text-sm outline-none",
|
||||||
"data-[highlighted]:bg-muted data-[disabled]:opacity-45",
|
"data-[highlighted]:bg-muted data-[disabled]:opacity-45",
|
||||||
].join(" ");
|
].join(" ");
|
||||||
|
|
||||||
export const databaseSyncScopes: ReadonlyArray<{ scope: CatalogSyncScope; label: string }> = [
|
export const databaseSyncScopes: ReadonlyArray<{ scope: DatabaseSyncScope; label: string }> = [
|
||||||
{ scope: "tables", label: "Synchronize tables" },
|
{ scope: "tables", label: "Synchronize tables" },
|
||||||
{ scope: "columns", label: "Synchronize all columns" },
|
|
||||||
{ scope: "relationships", label: "Synchronize relationships" },
|
{ scope: "relationships", label: "Synchronize relationships" },
|
||||||
{ scope: "all", label: "Synchronize all" },
|
{ scope: "all", label: "Synchronize all" },
|
||||||
];
|
];
|
||||||
@@ -18,7 +19,7 @@ export const databaseSyncScopes: ReadonlyArray<{ scope: CatalogSyncScope; label:
|
|||||||
interface DatabaseSyncMenuProps {
|
interface DatabaseSyncMenuProps {
|
||||||
disabled: boolean;
|
disabled: boolean;
|
||||||
disabledReason?: string;
|
disabledReason?: string;
|
||||||
onSelect: (scope: CatalogSyncScope) => void;
|
onSelect: (scope: DatabaseSyncScope) => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function DatabaseSyncMenu({ disabled, disabledReason, onSelect }: DatabaseSyncMenuProps) {
|
export function DatabaseSyncMenu({ disabled, disabledReason, onSelect }: DatabaseSyncMenuProps) {
|
||||||
|
|||||||
@@ -1,15 +1,16 @@
|
|||||||
import { useEffect, useMemo, useRef, useState } from "react";
|
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||||
import { Menu } from "@base-ui/react/menu";
|
import { Menu } from "@base-ui/react/menu";
|
||||||
import { useQuery, useQueryClient } from "@tanstack/react-query";
|
import { useQuery, useQueryClient } from "@tanstack/react-query";
|
||||||
import { AgGridReact } from "ag-grid-react";
|
import { AgGridReact } from "ag-grid-react";
|
||||||
import type { ColDef, ICellRendererParams } from "ag-grid-community";
|
import type { ColDef, ICellRendererParams } from "ag-grid-community";
|
||||||
import { ArrowLeft, ChevronDown, Columns3, History, Pencil, RefreshCw, Save, Sparkles, Trash2, X } from "lucide-react";
|
import { ArrowLeft, ChevronDown, Columns3, Pencil, RefreshCw, Save, Sparkles, Trash2, X } from "lucide-react";
|
||||||
import { toast } from "sonner";
|
import { toast } from "sonner";
|
||||||
import { Button } from "../../components/ui/button";
|
import { Button } from "../../components/ui/button";
|
||||||
import { ApiError, apiErrorMessage } from "../../api/client";
|
import { ApiError, apiErrorMessage } from "../../api/client";
|
||||||
import {
|
import {
|
||||||
consolidateCatalogDescriptions,
|
consolidateCatalogDescriptions,
|
||||||
deleteCatalogTableMetadata,
|
deleteCatalogTableMetadata,
|
||||||
|
listCatalogColumns,
|
||||||
listCatalogTables,
|
listCatalogTables,
|
||||||
startCatalogSync,
|
startCatalogSync,
|
||||||
startDescriptionGenerationRun,
|
startDescriptionGenerationRun,
|
||||||
@@ -37,11 +38,10 @@ interface Props {
|
|||||||
onOpenOverview: () => void;
|
onOpenOverview: () => void;
|
||||||
onOpenRelationships: () => void;
|
onOpenRelationships: () => void;
|
||||||
onNavigationStateChange: (state: DatabaseNavigationState) => void;
|
onNavigationStateChange: (state: DatabaseNavigationState) => void;
|
||||||
onNestedNavigationChange?: (active: boolean) => void;
|
onNestedNavigationChange?: (active: boolean, onBack?: () => void) => void;
|
||||||
onRunStarted: (run: CatalogSyncRun) => void;
|
onRunStarted: (run: CatalogSyncRun) => void;
|
||||||
onOpenSync: () => void;
|
onOpenSync: () => void;
|
||||||
onOpenDescriptionHistory?: () => void;
|
onClearCatalogTables?: () => Promise<void>;
|
||||||
onOpenSensitiveHistory?: () => void;
|
|
||||||
onDescriptionGenerationRunStarted: (run: DescriptionGenerationRun) => void;
|
onDescriptionGenerationRunStarted: (run: DescriptionGenerationRun) => void;
|
||||||
onSuggestSensitive: (selection: SensitiveDataSuggestionRequest, scopeLabel: string) => Promise<void>;
|
onSuggestSensitive: (selection: SensitiveDataSuggestionRequest, scopeLabel: string) => Promise<void>;
|
||||||
onCatalogMetricsChanged?: () => void | Promise<void>;
|
onCatalogMetricsChanged?: () => void | Promise<void>;
|
||||||
@@ -95,8 +95,7 @@ export function DatabaseTables({
|
|||||||
onNestedNavigationChange,
|
onNestedNavigationChange,
|
||||||
onRunStarted,
|
onRunStarted,
|
||||||
onOpenSync,
|
onOpenSync,
|
||||||
onOpenDescriptionHistory,
|
onClearCatalogTables,
|
||||||
onOpenSensitiveHistory,
|
|
||||||
onDescriptionGenerationRunStarted,
|
onDescriptionGenerationRunStarted,
|
||||||
onSuggestSensitive,
|
onSuggestSensitive,
|
||||||
onCatalogMetricsChanged,
|
onCatalogMetricsChanged,
|
||||||
@@ -134,9 +133,22 @@ export function DatabaseTables({
|
|||||||
const nestedNavigationActive = presentation === "fleet" && Boolean(activeTable) && tableSection === "columns";
|
const nestedNavigationActive = presentation === "fleet" && Boolean(activeTable) && tableSection === "columns";
|
||||||
const bindingReady = database.connectionStatus === "reachable" && database.testedVersion === database.version;
|
const bindingReady = database.connectionStatus === "reachable" && database.testedVersion === database.version;
|
||||||
const currentRun = activeRun ?? database.activeSyncRun;
|
const currentRun = activeRun ?? database.activeSyncRun;
|
||||||
|
const clearAllCatalogTables = async () => {
|
||||||
|
if (!onClearCatalogTables || !window.confirm("Clear all catalog tables, columns, and relationships for this database? The source database will not be changed.")) return;
|
||||||
|
await onClearCatalogTables();
|
||||||
|
};
|
||||||
|
const leaveTable = useCallback(() => {
|
||||||
|
if (navigationBusy) return;
|
||||||
|
if (dirty && !window.confirm("Discard unsaved metadata?")) return;
|
||||||
|
setActiveTableId(null);
|
||||||
|
setTableSection("overview");
|
||||||
|
window.setTimeout(() => originRef.current?.focus(), 0);
|
||||||
|
}, [dirty, navigationBusy]);
|
||||||
|
|
||||||
useEffect(() => { onNavigationStateChange({ dirty, busy: navigationBusy }); }, [dirty, navigationBusy, onNavigationStateChange]);
|
useEffect(() => { onNavigationStateChange({ dirty, busy: navigationBusy }); }, [dirty, navigationBusy, onNavigationStateChange]);
|
||||||
useEffect(() => { onNestedNavigationChange?.(nestedNavigationActive); }, [nestedNavigationActive, onNestedNavigationChange]);
|
useEffect(() => {
|
||||||
|
onNestedNavigationChange?.(nestedNavigationActive, nestedNavigationActive ? leaveTable : undefined);
|
||||||
|
}, [leaveTable, nestedNavigationActive, onNestedNavigationChange]);
|
||||||
useEffect(() => () => { onNestedNavigationChange?.(false); }, [onNestedNavigationChange]);
|
useEffect(() => () => { onNestedNavigationChange?.(false); }, [onNestedNavigationChange]);
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!activeTableId || !activeTable || editorVersion === activeTable.version || busy) return;
|
if (!activeTableId || !activeTable || editorVersion === activeTable.version || busy) return;
|
||||||
@@ -168,13 +180,6 @@ export function DatabaseTables({
|
|||||||
setStaleBannerOpen(true);
|
setStaleBannerOpen(true);
|
||||||
setColumnNavigation({ dirty: false, busy: false });
|
setColumnNavigation({ dirty: false, busy: false });
|
||||||
};
|
};
|
||||||
const leaveTable = () => {
|
|
||||||
if (navigationBusy) return;
|
|
||||||
if (dirty && !window.confirm("Discard unsaved metadata?")) return;
|
|
||||||
setActiveTableId(null);
|
|
||||||
setTableSection("overview");
|
|
||||||
window.setTimeout(() => originRef.current?.focus(), 0);
|
|
||||||
};
|
|
||||||
const navigateDatabase = (target: "databases" | "overview" | "relationships") => {
|
const navigateDatabase = (target: "databases" | "overview" | "relationships") => {
|
||||||
if (navigationBusy) return;
|
if (navigationBusy) return;
|
||||||
if (dirty && !window.confirm("Discard unsaved metadata?")) return;
|
if (dirty && !window.confirm("Discard unsaved metadata?")) return;
|
||||||
@@ -281,6 +286,28 @@ export function DatabaseTables({
|
|||||||
toast.error(apiErrorMessage(error));
|
toast.error(apiErrorMessage(error));
|
||||||
} finally { setBusy(null); }
|
} finally { setBusy(null); }
|
||||||
};
|
};
|
||||||
|
const generateColumnDescriptions = async () => {
|
||||||
|
if (selectedIds.length === 0 || !selectedMetadataModel) return;
|
||||||
|
setBusy("generate");
|
||||||
|
try {
|
||||||
|
const columns = (await Promise.all(selectedIds.map((tableId) => listCatalogColumns(databaseId, tableId)))).flat();
|
||||||
|
const columnIds = columns.map((column) => column.id);
|
||||||
|
if (columnIds.length === 0) {
|
||||||
|
toast.error("The selected tables have no synchronized catalog columns.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const run = await startDescriptionGenerationRun(
|
||||||
|
databaseId,
|
||||||
|
selectedMetadataModel,
|
||||||
|
"selected_columns",
|
||||||
|
columnIds,
|
||||||
|
);
|
||||||
|
onDescriptionGenerationRunStarted(run);
|
||||||
|
toast.success(`Column description generation started for ${columnIds.length} column${columnIds.length === 1 ? "" : "s"}`);
|
||||||
|
} catch (error) {
|
||||||
|
toast.error(apiErrorMessage(error));
|
||||||
|
} finally { setBusy(null); }
|
||||||
|
};
|
||||||
const suggestSensitive = async () => {
|
const suggestSensitive = async () => {
|
||||||
if (selectedIds.length === 0) return;
|
if (selectedIds.length === 0) return;
|
||||||
setBusy("suggest");
|
setBusy("suggest");
|
||||||
@@ -298,6 +325,7 @@ export function DatabaseTables({
|
|||||||
|
|
||||||
type FleetTableAction =
|
type FleetTableAction =
|
||||||
| "generate-descriptions"
|
| "generate-descriptions"
|
||||||
|
| "generate-column-descriptions"
|
||||||
| "sync-tables"
|
| "sync-tables"
|
||||||
| "sync-columns"
|
| "sync-columns"
|
||||||
| "consolidate-descriptions"
|
| "consolidate-descriptions"
|
||||||
@@ -324,6 +352,24 @@ export function DatabaseTables({
|
|||||||
? "Another action is running."
|
? "Another action is running."
|
||||||
: undefined,
|
: undefined,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
id: "generate-column-descriptions",
|
||||||
|
label: "Generate column descriptions",
|
||||||
|
group: "Descriptions",
|
||||||
|
runLabel: "Generate",
|
||||||
|
disabled: !canManage || selectedIds.length === 0 || !selectedMetadataModel || descriptionGenerationActive || busy !== null || Boolean(currentRun),
|
||||||
|
disabledReason: !canManage
|
||||||
|
? "You do not have permission to generate descriptions."
|
||||||
|
: selectedIds.length === 0
|
||||||
|
? "Select at least one table."
|
||||||
|
: !selectedMetadataModel
|
||||||
|
? NO_METADATA_GENERATION_LLM_MODEL_MESSAGE
|
||||||
|
: descriptionGenerationActive || Boolean(currentRun)
|
||||||
|
? "Wait for the active catalog operation to finish."
|
||||||
|
: busy !== null
|
||||||
|
? "Another action is running."
|
||||||
|
: undefined,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
id: "consolidate-descriptions",
|
id: "consolidate-descriptions",
|
||||||
label: "Move generated to Description",
|
label: "Move generated to Description",
|
||||||
@@ -421,6 +467,7 @@ export function DatabaseTables({
|
|||||||
|
|
||||||
const runFleetAction = async (action: FleetTableAction) => {
|
const runFleetAction = async (action: FleetTableAction) => {
|
||||||
if (action === "generate-descriptions") await generateDescriptions();
|
if (action === "generate-descriptions") await generateDescriptions();
|
||||||
|
else if (action === "generate-column-descriptions") await generateColumnDescriptions();
|
||||||
else if (action === "sync-tables") await synchronize("tables");
|
else if (action === "sync-tables") await synchronize("tables");
|
||||||
else if (action === "consolidate-descriptions") await consolidateDescriptions();
|
else if (action === "consolidate-descriptions") await consolidateDescriptions();
|
||||||
else if (action === "sync-columns") await synchronize("columns", selectedIds);
|
else if (action === "sync-columns") await synchronize("columns", selectedIds);
|
||||||
@@ -464,7 +511,7 @@ export function DatabaseTables({
|
|||||||
const databaseHeader = (
|
const databaseHeader = (
|
||||||
<div className="px-4 pt-5 sm:px-5">
|
<div className="px-4 pt-5 sm:px-5">
|
||||||
<Button type="button" variant="ghost" className="-ml-2 mb-4" disabled={navigationBusy} onClick={() => navigateDatabase("databases")}><ArrowLeft />Back to databases</Button>
|
<Button type="button" variant="ghost" className="-ml-2 mb-4" disabled={navigationBusy} onClick={() => navigateDatabase("databases")}><ArrowLeft />Back to databases</Button>
|
||||||
<div className="flex flex-wrap items-start justify-between gap-4 border-b border-border pb-5">
|
<div className="flex flex-wrap items-start justify-between gap-4 border-b border-border pb-5">
|
||||||
<div>
|
<div>
|
||||||
<p className="thot-label mb-1">Database management</p>
|
<p className="thot-label mb-1">Database management</p>
|
||||||
<h2 ref={headingRef} tabIndex={-1} className="font-heading text-2xl font-semibold tracking-tight outline-none">{database.workspaceName}</h2>
|
<h2 ref={headingRef} tabIndex={-1} className="font-heading text-2xl font-semibold tracking-tight outline-none">{database.workspaceName}</h2>
|
||||||
@@ -472,6 +519,7 @@ export function DatabaseTables({
|
|||||||
</div>
|
</div>
|
||||||
<div className="flex gap-2">
|
<div className="flex gap-2">
|
||||||
{currentRun ? <Button type="button" variant="outline" onClick={onOpenSync}><RefreshCw className="animate-spin" />View sync</Button> : null}
|
{currentRun ? <Button type="button" variant="outline" onClick={onOpenSync}><RefreshCw className="animate-spin" />View sync</Button> : null}
|
||||||
|
{onClearCatalogTables ? <Button type="button" variant="outline" className="text-destructive" disabled={!canManage || busy !== null || Boolean(currentRun)} onClick={() => void clearAllCatalogTables()}>Clear catalog tables</Button> : null}
|
||||||
<Button type="button" disabled={!canManage || !bindingReady || busy !== null || Boolean(currentRun)} onClick={() => void synchronize("all")}><RefreshCw />Sync all</Button>
|
<Button type="button" disabled={!canManage || !bindingReady || busy !== null || Boolean(currentRun)} onClick={() => void synchronize("all")}><RefreshCw />Sync all</Button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -488,18 +536,6 @@ export function DatabaseTables({
|
|||||||
return (
|
return (
|
||||||
<section aria-label={`Columns for ${activeTable.name}`} className="flex min-h-0 flex-1 flex-col overflow-hidden bg-background">
|
<section aria-label={`Columns for ${activeTable.name}`} className="flex min-h-0 flex-1 flex-col overflow-hidden bg-background">
|
||||||
<div className="flex min-h-12 flex-wrap items-center gap-3 border-b border-border bg-muted/25 px-3 py-2">
|
<div className="flex min-h-12 flex-wrap items-center gap-3 border-b border-border bg-muted/25 px-3 py-2">
|
||||||
<Button
|
|
||||||
type="button"
|
|
||||||
variant="ghost"
|
|
||||||
size="sm"
|
|
||||||
className="thot-fleet-icon-action thot-fleet-back-action"
|
|
||||||
disabled={navigationBusy}
|
|
||||||
data-tooltip="Tables"
|
|
||||||
aria-label="Back to tables"
|
|
||||||
onClick={leaveTable}
|
|
||||||
>
|
|
||||||
<ArrowLeft aria-hidden="true" /> Back to tables
|
|
||||||
</Button>
|
|
||||||
<div className="min-w-0">
|
<div className="min-w-0">
|
||||||
<p className="thot-label">Physical table</p>
|
<p className="thot-label">Physical table</p>
|
||||||
<h3 className="truncate font-heading text-lg font-semibold">{activeTable.name}</h3>
|
<h3 className="truncate font-heading text-lg font-semibold">{activeTable.name}</h3>
|
||||||
@@ -513,9 +549,7 @@ export function DatabaseTables({
|
|||||||
descriptionGenerationActive={descriptionGenerationActive}
|
descriptionGenerationActive={descriptionGenerationActive}
|
||||||
onDescriptionGenerationRunStarted={onDescriptionGenerationRunStarted}
|
onDescriptionGenerationRunStarted={onDescriptionGenerationRunStarted}
|
||||||
onNavigationStateChange={setColumnNavigation}
|
onNavigationStateChange={setColumnNavigation}
|
||||||
onSync={() => void synchronize("columns", [activeTable.id])}
|
|
||||||
onSuggestSensitive={onSuggestSensitive}
|
onSuggestSensitive={onSuggestSensitive}
|
||||||
bindingReady={bindingReady}
|
|
||||||
catalogOperationActive={Boolean(currentRun)}
|
catalogOperationActive={Boolean(currentRun)}
|
||||||
onCatalogMetricsChanged={onCatalogMetricsChanged}
|
onCatalogMetricsChanged={onCatalogMetricsChanged}
|
||||||
presentation="fleet"
|
presentation="fleet"
|
||||||
@@ -626,7 +660,6 @@ export function DatabaseTables({
|
|||||||
descriptionGenerationActive={descriptionGenerationActive}
|
descriptionGenerationActive={descriptionGenerationActive}
|
||||||
onDescriptionGenerationRunStarted={onDescriptionGenerationRunStarted}
|
onDescriptionGenerationRunStarted={onDescriptionGenerationRunStarted}
|
||||||
onNavigationStateChange={setColumnNavigation}
|
onNavigationStateChange={setColumnNavigation}
|
||||||
onSync={() => void synchronize("columns", [activeTable.id])}
|
|
||||||
onSuggestSensitive={onSuggestSensitive}
|
onSuggestSensitive={onSuggestSensitive}
|
||||||
onCatalogMetricsChanged={onCatalogMetricsChanged}
|
onCatalogMetricsChanged={onCatalogMetricsChanged}
|
||||||
/>
|
/>
|
||||||
@@ -728,6 +761,7 @@ export function DatabaseTables({
|
|||||||
<Menu.Positioner side="bottom" align="start" sideOffset={4}>
|
<Menu.Positioner side="bottom" align="start" sideOffset={4}>
|
||||||
<Menu.Popup className="z-50 min-w-64 rounded-lg bg-popover p-1 text-popover-foreground shadow-md ring-1 ring-foreground/10 outline-none">
|
<Menu.Popup className="z-50 min-w-64 rounded-lg bg-popover p-1 text-popover-foreground shadow-md ring-1 ring-foreground/10 outline-none">
|
||||||
<Menu.Item className="rounded-md px-3 py-2 text-sm outline-none data-[highlighted]:bg-muted data-[disabled]:opacity-45" disabled={!canManage || !selectedMetadataModel || descriptionGenerationActive || busy !== null || Boolean(currentRun)} onClick={() => void generateDescriptions()}>Generate {selectedIds.length === 1 ? "description" : "descriptions"}</Menu.Item>
|
<Menu.Item className="rounded-md px-3 py-2 text-sm outline-none data-[highlighted]:bg-muted data-[disabled]:opacity-45" disabled={!canManage || !selectedMetadataModel || descriptionGenerationActive || busy !== null || Boolean(currentRun)} onClick={() => void generateDescriptions()}>Generate {selectedIds.length === 1 ? "description" : "descriptions"}</Menu.Item>
|
||||||
|
<Menu.Item className="rounded-md px-3 py-2 text-sm outline-none data-[highlighted]:bg-muted data-[disabled]:opacity-45" disabled={!canManage || !selectedMetadataModel || descriptionGenerationActive || busy !== null || Boolean(currentRun)} onClick={() => void generateColumnDescriptions()}>Generate descriptions for all columns</Menu.Item>
|
||||||
<Menu.Item className="rounded-md px-3 py-2 text-sm outline-none data-[highlighted]:bg-muted data-[disabled]:opacity-45" disabled={!canManage || !bindingReady || busy !== null || Boolean(currentRun)} onClick={() => void synchronize("columns", selectedIds)}>Synchronize columns</Menu.Item>
|
<Menu.Item className="rounded-md px-3 py-2 text-sm outline-none data-[highlighted]:bg-muted data-[disabled]:opacity-45" disabled={!canManage || !bindingReady || busy !== null || Boolean(currentRun)} onClick={() => void synchronize("columns", selectedIds)}>Synchronize columns</Menu.Item>
|
||||||
<Menu.Item className="rounded-md px-3 py-2 text-sm outline-none data-[highlighted]:bg-muted data-[disabled]:opacity-45" disabled={!canManage || busy !== null || Boolean(currentRun)} onClick={() => void consolidateDescriptions()}>Move generated description to Description</Menu.Item>
|
<Menu.Item className="rounded-md px-3 py-2 text-sm outline-none data-[highlighted]:bg-muted data-[disabled]:opacity-45" disabled={!canManage || busy !== null || Boolean(currentRun)} onClick={() => void consolidateDescriptions()}>Move generated description to Description</Menu.Item>
|
||||||
<Menu.Separator className="my-1 h-px bg-border" />
|
<Menu.Separator className="my-1 h-px bg-border" />
|
||||||
@@ -769,13 +803,6 @@ export function DatabaseTables({
|
|||||||
</>
|
</>
|
||||||
)
|
)
|
||||||
)}
|
)}
|
||||||
{presentation === "fleet" ? (
|
|
||||||
<Button type="button" size="sm" variant="outline" className="thot-fleet-nav-action thot-fleet-nav-action--below" data-tooltip="Synchronization history" onClick={onOpenSync}>
|
|
||||||
{currentRun ? <RefreshCw className="animate-spin" aria-hidden="true" /> : <History aria-hidden="true" />} Sync history
|
|
||||||
</Button>
|
|
||||||
) : null}
|
|
||||||
{onOpenDescriptionHistory ? <Button type="button" size="sm" variant="outline" onClick={onOpenDescriptionHistory}><History aria-hidden="true" /> Description history</Button> : null}
|
|
||||||
{onOpenSensitiveHistory ? <Button type="button" size="sm" variant="outline" onClick={onOpenSensitiveHistory}><History aria-hidden="true" /> Sensitive history</Button> : null}
|
|
||||||
</div>
|
</div>
|
||||||
{!bindingReady ? <div className="border-b border-border bg-muted/35 px-4 py-3 text-sm text-muted-foreground">Test the current database binding from Overview before synchronizing tables.</div> : null}
|
{!bindingReady ? <div className="border-b border-border bg-muted/35 px-4 py-3 text-sm text-muted-foreground">Test the current database binding from Overview before synchronizing tables.</div> : null}
|
||||||
{fleetQueryError ?? <div className="relative min-h-[280px] flex-1">
|
{fleetQueryError ?? <div className="relative min-h-[280px] flex-1">
|
||||||
|
|||||||
@@ -109,6 +109,9 @@ test("replays ordered SSE events, reconnects from the latest sequence, and keeps
|
|||||||
renderDrawer();
|
renderDrawer();
|
||||||
|
|
||||||
const log = await screen.findByRole("log", { name: "Description generation events" });
|
const log = await screen.findByRole("log", { name: "Description generation events" });
|
||||||
|
const progress = screen.getByRole("table", { name: "Description generation progress" });
|
||||||
|
expect(within(progress).getAllByRole("columnheader")).toHaveLength(5);
|
||||||
|
expect(log).toHaveClass("thot-catalog-drawer__event-log");
|
||||||
expect(await within(log).findByText("Run queued")).toBeVisible();
|
expect(await within(log).findByText("Run queued")).toBeVisible();
|
||||||
await waitFor(() => expect(FakeEventSource.instances).toHaveLength(1));
|
await waitFor(() => expect(FakeEventSource.instances).toHaveLength(1));
|
||||||
expect(FakeEventSource.instances[0].url).toBe(
|
expect(FakeEventSource.instances[0].url).toBe(
|
||||||
|
|||||||
@@ -300,6 +300,16 @@ export function DescriptionGenerationDrawer({
|
|||||||
}
|
}
|
||||||
const apparentlyStale = isApparentlyStale(run);
|
const apparentlyStale = isApparentlyStale(run);
|
||||||
const finalError = safeFinalError(run.errorSummary);
|
const finalError = safeFinalError(run.errorSummary);
|
||||||
|
const modelDisplay = modelLabel && modelLabel !== run.modelId
|
||||||
|
? `${modelLabel} (${run.modelId})`
|
||||||
|
: run.modelId;
|
||||||
|
const progressCounters = [
|
||||||
|
["total", "Total"],
|
||||||
|
["processed", "Processed"],
|
||||||
|
["generated", "Generated"],
|
||||||
|
["nonGeneratable", "Not generated"],
|
||||||
|
["failed", "Failed"],
|
||||||
|
] as const;
|
||||||
const footer = ["queued", "running"].includes(run.status) ? (
|
const footer = ["queued", "running"].includes(run.status) ? (
|
||||||
<div className="w-full">
|
<div className="w-full">
|
||||||
{apparentlyStale ? (
|
{apparentlyStale ? (
|
||||||
@@ -341,55 +351,80 @@ export function DescriptionGenerationDrawer({
|
|||||||
open
|
open
|
||||||
ariaLabel="Description generation"
|
ariaLabel="Description generation"
|
||||||
eyebrow="Description generation"
|
eyebrow="Description generation"
|
||||||
title={statusLabel(run)}
|
title={(
|
||||||
description={`${modelLabel}${modelLabel !== run.modelId ? ` (${run.modelId})` : ""} · ${run.scope.replaceAll("_", " ")}`}
|
<span className="inline-flex items-center gap-2">
|
||||||
|
{!isTerminal(run) ? <LoaderCircle className="size-5 animate-spin" aria-hidden="true" /> : null}
|
||||||
|
{statusLabel(run)}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
description={run.scope.replaceAll("_", " ")}
|
||||||
onClose={onClose}
|
onClose={onClose}
|
||||||
closeLabel="Close description generation"
|
closeLabel="Close description generation"
|
||||||
busy={action !== null}
|
busy={action !== null}
|
||||||
bodyClassName="thot-catalog-drawer__history-layout"
|
bodyClassName="thot-catalog-drawer__history-layout thot-catalog-drawer__run-layout"
|
||||||
footer={footer}
|
footer={footer}
|
||||||
>
|
>
|
||||||
{runQuery.isError ? (
|
<div className="thot-catalog-drawer__run-main">
|
||||||
|
{runQuery.isError ? (
|
||||||
<div role="alert" className="rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive">
|
<div role="alert" className="rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive">
|
||||||
{apiErrorMessage(runQuery.error)}
|
{apiErrorMessage(runQuery.error)}
|
||||||
</div>
|
</div>
|
||||||
) : null}
|
) : null}
|
||||||
|
|
||||||
<section aria-label="Description generation counters">
|
<div className={`grid gap-4 sm:grid-cols-2${runQuery.isError ? " mt-4" : ""}`}>
|
||||||
|
<section aria-label="AI usage">
|
||||||
|
<h3 className="thot-label mb-2">AI usage</h3>
|
||||||
|
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1 text-xs">
|
||||||
|
<dt>Model</dt><dd className="truncate text-right" title={modelDisplay}>{modelDisplay}</dd>
|
||||||
|
<dt>Input tokens</dt><dd className="text-right tabular-nums">{(run.inputTokens ?? 0).toLocaleString("en-US")}</dd>
|
||||||
|
<dt>Cache tokens</dt><dd className="text-right tabular-nums">{(run.cacheReadTokens ?? 0).toLocaleString("en-US")}</dd>
|
||||||
|
<dt>Output tokens</dt><dd className="text-right tabular-nums">{(run.outputTokens ?? 0).toLocaleString("en-US")}</dd>
|
||||||
|
</dl>
|
||||||
|
</section>
|
||||||
|
<section aria-label="Description generation timestamps">
|
||||||
|
<h3 className="thot-label mb-2">Timestamps</h3>
|
||||||
|
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1 text-xs">
|
||||||
|
<dt className="text-muted-foreground">Created</dt><dd className="text-right tabular-nums">{timestamp(run.createdAt)}</dd>
|
||||||
|
<dt className="text-muted-foreground">Started</dt><dd className="text-right tabular-nums">{timestamp(run.startedAt)}</dd>
|
||||||
|
<dt className="text-muted-foreground">Updated</dt><dd className="text-right tabular-nums">{timestamp(run.updatedAt)}</dd>
|
||||||
|
<dt className="text-muted-foreground">Finished</dt><dd className="text-right tabular-nums">{timestamp(run.finishedAt)}</dd>
|
||||||
|
</dl>
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<section className="mt-4" aria-label="Description generation counters">
|
||||||
<h3 className="thot-label mb-2">Progress</h3>
|
<h3 className="thot-label mb-2">Progress</h3>
|
||||||
<div className="grid grid-cols-2 gap-2">
|
<div className="overflow-x-auto rounded-md border border-border bg-muted/20">
|
||||||
{([
|
<table className="w-full min-w-[28rem] table-fixed" aria-label="Description generation progress">
|
||||||
["total", "Total"],
|
<thead>
|
||||||
["processed", "Processed"],
|
<tr className="border-b border-border bg-muted/45">
|
||||||
["generated", "Generated"],
|
{progressCounters.map(([name, label]) => (
|
||||||
["nonGeneratable", "Not generated"],
|
<th key={name} scope="col" className="border-l border-border px-2 py-1.5 text-left text-[10px] font-semibold uppercase tracking-wide text-muted-foreground first:border-l-0">
|
||||||
["failed", "Failed"],
|
{label}
|
||||||
] as const).map(([name, label]) => (
|
</th>
|
||||||
<div key={name} className="rounded-md border border-border bg-muted/25 px-3 py-2">
|
))}
|
||||||
<p className="text-[11px] font-semibold uppercase tracking-wide text-muted-foreground">{label}</p>
|
</tr>
|
||||||
<p data-count={name} className="mt-1 text-lg font-semibold tabular-nums">{run[name]}</p>
|
</thead>
|
||||||
</div>
|
<tbody>
|
||||||
))}
|
<tr>
|
||||||
|
{progressCounters.map(([name]) => (
|
||||||
|
<td key={name} data-count={name} className="border-l border-border px-2 py-1.5 text-base font-semibold tabular-nums first:border-l-0">
|
||||||
|
{run[name]}
|
||||||
|
</td>
|
||||||
|
))}
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
<section className="mt-5" aria-label="Description generation timestamps">
|
|
||||||
<h3 className="thot-label mb-2">Timestamps</h3>
|
|
||||||
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1.5 text-sm">
|
|
||||||
<dt className="text-muted-foreground">Created</dt><dd className="text-right">{timestamp(run.createdAt)}</dd>
|
|
||||||
<dt className="text-muted-foreground">Started</dt><dd className="text-right">{timestamp(run.startedAt)}</dd>
|
|
||||||
<dt className="text-muted-foreground">Updated</dt><dd className="text-right">{timestamp(run.updatedAt)}</dd>
|
|
||||||
<dt className="text-muted-foreground">Finished</dt><dd className="text-right">{timestamp(run.finishedAt)}</dd>
|
|
||||||
</dl>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
{finalError ? (
|
{finalError ? (
|
||||||
<div className="mt-5 rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive">
|
<div className="mt-3 rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive">
|
||||||
{finalError}
|
{finalError}
|
||||||
</div>
|
</div>
|
||||||
) : null}
|
) : null}
|
||||||
|
|
||||||
<section className="mt-5" aria-label="Description generation log">
|
<section className="thot-catalog-drawer__events mt-3" aria-label="Description generation log">
|
||||||
<div className="mb-2 flex items-center justify-between">
|
<div className="mb-2 flex items-center justify-between">
|
||||||
<h3 className="thot-label">Events</h3>
|
<h3 className="thot-label">Events</h3>
|
||||||
<span className="text-xs tabular-nums text-muted-foreground">{events.length}</span>
|
<span className="text-xs tabular-nums text-muted-foreground">{events.length}</span>
|
||||||
@@ -397,7 +432,7 @@ export function DescriptionGenerationDrawer({
|
|||||||
<div
|
<div
|
||||||
role="log"
|
role="log"
|
||||||
aria-label="Description generation events"
|
aria-label="Description generation events"
|
||||||
className="max-h-72 overflow-y-auto rounded-md bg-zinc-950 p-3 font-mono text-xs leading-5 text-zinc-200"
|
className="thot-catalog-drawer__event-log overflow-y-auto rounded-md bg-zinc-950 p-3 font-mono text-xs leading-5 text-zinc-200"
|
||||||
>
|
>
|
||||||
{events.length === 0 ? <p className="text-zinc-500">Waiting for events…</p> : events.map((event) => (
|
{events.length === 0 ? <p className="text-zinc-500">Waiting for events…</p> : events.map((event) => (
|
||||||
<p key={event.sequence} className={event.level === "error" ? "text-red-300" : event.level === "warning" ? "text-amber-300" : undefined}>
|
<p key={event.sequence} className={event.level === "error" ? "text-red-300" : event.level === "warning" ? "text-amber-300" : undefined}>
|
||||||
@@ -407,8 +442,9 @@ export function DescriptionGenerationDrawer({
|
|||||||
</div>
|
</div>
|
||||||
{eventQuery.isError ? <p role="alert" className="mt-2 text-sm text-destructive">{apiErrorMessage(eventQuery.error)}</p> : null}
|
{eventQuery.isError ? <p role="alert" className="mt-2 text-sm text-destructive">{apiErrorMessage(eventQuery.error)}</p> : null}
|
||||||
</section>
|
</section>
|
||||||
|
</div>
|
||||||
|
|
||||||
<section className="thot-catalog-drawer__history mt-5" aria-label="Description generation history">
|
<section className="thot-catalog-drawer__history" aria-label="Description generation history">
|
||||||
<div className="mb-2 flex items-center justify-between gap-3">
|
<div className="mb-2 flex items-center justify-between gap-3">
|
||||||
<h3 className="thot-label">Recent runs</h3>
|
<h3 className="thot-label">Recent runs</h3>
|
||||||
<span className="text-xs tabular-nums text-muted-foreground">
|
<span className="text-xs tabular-nums text-muted-foreground">
|
||||||
@@ -424,7 +460,7 @@ export function DescriptionGenerationDrawer({
|
|||||||
) : visibleHistory.length === 0 ? (
|
) : visibleHistory.length === 0 ? (
|
||||||
<p className="text-sm text-muted-foreground">No description generation runs yet.</p>
|
<p className="text-sm text-muted-foreground">No description generation runs yet.</p>
|
||||||
) : (
|
) : (
|
||||||
<div className="max-h-56 divide-y divide-border overflow-y-auto rounded-md border border-border">
|
<div className="thot-catalog-drawer__history-list divide-y divide-border overflow-y-auto rounded-md border border-border">
|
||||||
{visibleHistory.map((item) => (
|
{visibleHistory.map((item) => (
|
||||||
<button
|
<button
|
||||||
key={item.id}
|
key={item.id}
|
||||||
|
|||||||
@@ -15,6 +15,19 @@ const actions = [
|
|||||||
},
|
},
|
||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
|
test("disables the picker when no rows are selected", () => {
|
||||||
|
render(
|
||||||
|
<FleetActionSelector
|
||||||
|
actions={actions}
|
||||||
|
selectedCount={0}
|
||||||
|
onRun={vi.fn()}
|
||||||
|
onClear={vi.fn()}
|
||||||
|
/>,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(screen.getByRole("combobox", { name: "Batch action" })).toBeDisabled();
|
||||||
|
});
|
||||||
|
|
||||||
test("keeps unavailable commands discoverable and explains why Run is disabled", async () => {
|
test("keeps unavailable commands discoverable and explains why Run is disabled", async () => {
|
||||||
const user = userEvent.setup();
|
const user = userEvent.setup();
|
||||||
render(
|
render(
|
||||||
@@ -50,6 +63,7 @@ test("runs an enabled command explicitly and resets the picker", async () => {
|
|||||||
await user.selectOptions(picker, "sync");
|
await user.selectOptions(picker, "sync");
|
||||||
expect(screen.getByText("All tables in this database")).toBeVisible();
|
expect(screen.getByText("All tables in this database")).toBeVisible();
|
||||||
expect(screen.getByText("Action scope")).toBeVisible();
|
expect(screen.getByText("Action scope")).toBeVisible();
|
||||||
|
expect(screen.getByRole("button", { name: "Run action" })).toHaveClass("bg-destructive");
|
||||||
await user.click(screen.getByRole("button", { name: "Run action" }));
|
await user.click(screen.getByRole("button", { name: "Run action" }));
|
||||||
|
|
||||||
expect(onRun).toHaveBeenCalledWith("sync");
|
expect(onRun).toHaveBeenCalledWith("sync");
|
||||||
|
|||||||
@@ -44,7 +44,6 @@ export function FleetActionSelector<ActionId extends string>({
|
|||||||
selectedCount,
|
selectedCount,
|
||||||
hiddenSelectedCount = 0,
|
hiddenSelectedCount = 0,
|
||||||
busy = false,
|
busy = false,
|
||||||
busyLabel = "Running…",
|
|
||||||
selectRef,
|
selectRef,
|
||||||
onRun,
|
onRun,
|
||||||
onClear,
|
onClear,
|
||||||
@@ -102,7 +101,7 @@ export function FleetActionSelector<ActionId extends string>({
|
|||||||
className="thot-fleet-action-selector__select"
|
className="thot-fleet-action-selector__select"
|
||||||
aria-label={label}
|
aria-label={label}
|
||||||
value={selectedActionId}
|
value={selectedActionId}
|
||||||
disabled={busy}
|
disabled={busy || selectedCount === 0}
|
||||||
onChange={(event) => setSelectedActionId(event.target.value as ActionId | "")}
|
onChange={(event) => setSelectedActionId(event.target.value as ActionId | "")}
|
||||||
>
|
>
|
||||||
<option value="">Choose an action…</option>
|
<option value="">Choose an action…</option>
|
||||||
@@ -123,13 +122,14 @@ export function FleetActionSelector<ActionId extends string>({
|
|||||||
<Button
|
<Button
|
||||||
type="button"
|
type="button"
|
||||||
size="sm"
|
size="sm"
|
||||||
variant={selectedAction?.tone === "destructive" ? "destructive" : "default"}
|
variant="destructive"
|
||||||
|
className="bg-destructive text-destructive-foreground hover:bg-destructive/90"
|
||||||
disabled={!selectedAction || selectedAction.disabled || busy}
|
disabled={!selectedAction || selectedAction.disabled || busy}
|
||||||
aria-describedby={unavailableReason ? `${hintId}-reason` : undefined}
|
aria-describedby={unavailableReason ? `${hintId}-reason` : undefined}
|
||||||
onClick={() => void run()}
|
onClick={() => void run()}
|
||||||
>
|
>
|
||||||
<Play aria-hidden="true" />
|
<Play aria-hidden="true" />
|
||||||
{busy ? busyLabel : selectedAction?.runLabel ?? "Run action"}
|
Run action
|
||||||
</Button>
|
</Button>
|
||||||
<Button type="button" size="sm" variant="ghost" disabled={busy} onClick={onClear}>
|
<Button type="button" size="sm" variant="ghost" disabled={busy} onClick={onClear}>
|
||||||
<X aria-hidden="true" /> Clear
|
<X aria-hidden="true" /> Clear
|
||||||
|
|||||||
@@ -402,6 +402,7 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
.thot-fleet-action-selector__select {
|
.thot-fleet-action-selector__select {
|
||||||
|
flex: 1 1 13rem;
|
||||||
width: min(100%, 22rem);
|
width: min(100%, 22rem);
|
||||||
min-width: 13rem;
|
min-width: 13rem;
|
||||||
height: 2rem;
|
height: 2rem;
|
||||||
@@ -459,11 +460,11 @@
|
|||||||
.thot-fleet-ledger__drawer {
|
.thot-fleet-ledger__drawer {
|
||||||
position: absolute;
|
position: absolute;
|
||||||
z-index: 50;
|
z-index: 50;
|
||||||
inset-block: 0.75rem;
|
inset-block: clamp(0.625rem, 2vh, 1rem);
|
||||||
inset-inline-start: 50%;
|
inset-inline-start: 50%;
|
||||||
inset-inline-end: auto;
|
inset-inline-end: auto;
|
||||||
display: flex;
|
display: flex;
|
||||||
width: 60%;
|
width: min(75rem, calc(100% - 2rem));
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
@@ -479,10 +480,10 @@
|
|||||||
|
|
||||||
.thot-fleet-ledger .thot-fleet-ledger__context-panel {
|
.thot-fleet-ledger .thot-fleet-ledger__context-panel {
|
||||||
position: absolute;
|
position: absolute;
|
||||||
inset-block: 0.75rem;
|
inset-block: clamp(0.625rem, 2vh, 1rem);
|
||||||
inset-inline-start: 50%;
|
inset-inline-start: 50%;
|
||||||
inset-inline-end: auto;
|
inset-inline-end: auto;
|
||||||
width: 60%;
|
width: min(75rem, calc(100% - 2rem));
|
||||||
max-width: none;
|
max-width: none;
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
transform: translateX(-50%);
|
transform: translateX(-50%);
|
||||||
@@ -537,6 +538,20 @@
|
|||||||
box-shadow: inset 0 -1px 0 oklch(var(--primary-foreground) / 0.18);
|
box-shadow: inset 0 -1px 0 oklch(var(--primary-foreground) / 0.18);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.thot-sensitive-run--success .thot-fleet-ledger__drawer-header,
|
||||||
|
.thot-sensitive-run--success .thot-catalog-drawer__header {
|
||||||
|
color: oklch(var(--primary-foreground));
|
||||||
|
background-color: oklch(var(--success));
|
||||||
|
border-color: oklch(var(--primary-foreground) / 0.18);
|
||||||
|
}
|
||||||
|
|
||||||
|
.thot-sensitive-run--failure .thot-fleet-ledger__drawer-header,
|
||||||
|
.thot-sensitive-run--failure .thot-catalog-drawer__header {
|
||||||
|
color: oklch(var(--destructive-foreground));
|
||||||
|
background-color: oklch(var(--destructive));
|
||||||
|
border-color: oklch(var(--destructive-foreground) / 0.18);
|
||||||
|
}
|
||||||
|
|
||||||
.thot-fleet-ledger__drawer-header .thot-fleet-ledger__drawer-title,
|
.thot-fleet-ledger__drawer-header .thot-fleet-ledger__drawer-title,
|
||||||
.thot-fleet-ledger__drawer-header .thot-fleet-ledger__drawer-description,
|
.thot-fleet-ledger__drawer-header .thot-fleet-ledger__drawer-description,
|
||||||
.thot-fleet-ledger__drawer-header .thot-fleet-ledger__eyebrow,
|
.thot-fleet-ledger__drawer-header .thot-fleet-ledger__eyebrow,
|
||||||
@@ -575,6 +590,19 @@
|
|||||||
box-shadow: inset 0 0 0 1px oklch(var(--success) / 0.5);
|
box-shadow: inset 0 0 0 1px oklch(var(--success) / 0.5);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.thot-catalog-drawer__event-log {
|
||||||
|
min-height: 8rem;
|
||||||
|
max-height: 18rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.thot-catalog-drawer__history-list {
|
||||||
|
max-height: 14rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.thot-catalog-drawer__run-layout > .thot-catalog-drawer__history {
|
||||||
|
margin-top: 1.25rem;
|
||||||
|
}
|
||||||
|
|
||||||
.thot-fleet-grid-surface {
|
.thot-fleet-grid-surface {
|
||||||
position: relative;
|
position: relative;
|
||||||
}
|
}
|
||||||
@@ -583,6 +611,13 @@
|
|||||||
background: oklch(var(--muted) / 0.32);
|
background: oklch(var(--muted) / 0.32);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@media (min-width: 48rem) {
|
||||||
|
.thot-fleet-grid-toolbar > .thot-fleet-action-selector {
|
||||||
|
flex: 1 1 33rem;
|
||||||
|
min-width: 33rem;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
.thot-fleet-ledger-grid {
|
.thot-fleet-ledger-grid {
|
||||||
--ag-header-background-color: oklch(var(--muted) / 0.84);
|
--ag-header-background-color: oklch(var(--muted) / 0.84);
|
||||||
--ag-odd-row-background-color: oklch(var(--card));
|
--ag-odd-row-background-color: oklch(var(--card));
|
||||||
@@ -605,6 +640,14 @@
|
|||||||
background: oklch(var(--muted) / 0.12);
|
background: oklch(var(--muted) / 0.12);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Keep selected rows readable without overpowering the status indicators. */
|
||||||
|
.thot-fleet-ledger-grid .ag-row.ag-row-selected,
|
||||||
|
.thot-fleet-ledger-grid .ag-row.ag-row-selected .ag-cell,
|
||||||
|
.thot-fleet-ledger-grid .ag-row.ag-row-selected .ag-grid-pinned-left-cells,
|
||||||
|
.thot-fleet-ledger-grid .ag-row.ag-row-selected .ag-grid-pinned-right-cells {
|
||||||
|
background-color: oklch(93% 0.002 17.2) !important;
|
||||||
|
}
|
||||||
|
|
||||||
.thot-fleet-icon-action {
|
.thot-fleet-icon-action {
|
||||||
position: relative;
|
position: relative;
|
||||||
}
|
}
|
||||||
@@ -744,6 +787,10 @@
|
|||||||
padding: 0;
|
padding: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.thot-fleet-ledger__drawer-body.thot-catalog-drawer__run-layout {
|
||||||
|
padding: clamp(0.75rem, 1.5cqi, 1.25rem);
|
||||||
|
}
|
||||||
|
|
||||||
.thot-fleet-ledger__drawer-footer {
|
.thot-fleet-ledger__drawer-footer {
|
||||||
display: flex;
|
display: flex;
|
||||||
flex-wrap: wrap;
|
flex-wrap: wrap;
|
||||||
@@ -784,8 +831,8 @@
|
|||||||
@container fleet-surface (max-width: 47.999rem) {
|
@container fleet-surface (max-width: 47.999rem) {
|
||||||
.thot-fleet-ledger__drawer,
|
.thot-fleet-ledger__drawer,
|
||||||
.thot-fleet-ledger .thot-fleet-ledger__context-panel {
|
.thot-fleet-ledger .thot-fleet-ledger__context-panel {
|
||||||
inset-block: 0.5rem;
|
inset-block: 0.625rem;
|
||||||
width: calc(100% - 1rem);
|
width: calc(100% - 1.5rem);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -816,11 +863,11 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@media (min-width: 40rem) {
|
@container work-area-panel (min-width: 48rem) {
|
||||||
.thot-catalog-drawer__history-layout {
|
.thot-catalog-drawer__history-layout {
|
||||||
display: grid;
|
display: grid;
|
||||||
grid-template-columns: minmax(0, 1fr) 13.5rem;
|
grid-template-columns: minmax(0, 1fr) clamp(12.5rem, 25cqi, 15rem);
|
||||||
gap: 0 1rem;
|
gap: 0 clamp(0.75rem, 2cqi, 1.25rem);
|
||||||
align-content: start;
|
align-content: start;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -836,6 +883,45 @@
|
|||||||
align-self: start;
|
align-self: start;
|
||||||
margin-top: 0;
|
margin-top: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.thot-catalog-drawer__run-layout {
|
||||||
|
height: 100%;
|
||||||
|
min-height: 0;
|
||||||
|
grid-template-rows: minmax(0, 1fr);
|
||||||
|
align-content: stretch;
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
|
||||||
|
.thot-catalog-drawer__run-layout > .thot-catalog-drawer__run-main {
|
||||||
|
display: flex;
|
||||||
|
min-height: 0;
|
||||||
|
flex-direction: column;
|
||||||
|
grid-column: 1;
|
||||||
|
grid-row: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
.thot-catalog-drawer__run-layout > .thot-catalog-drawer__history {
|
||||||
|
position: static;
|
||||||
|
display: flex;
|
||||||
|
min-height: 0;
|
||||||
|
flex-direction: column;
|
||||||
|
align-self: stretch;
|
||||||
|
grid-row: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
.thot-catalog-drawer__run-layout .thot-catalog-drawer__events {
|
||||||
|
display: flex;
|
||||||
|
min-height: 10rem;
|
||||||
|
flex: 1;
|
||||||
|
flex-direction: column;
|
||||||
|
}
|
||||||
|
|
||||||
|
.thot-catalog-drawer__run-layout .thot-catalog-drawer__event-log,
|
||||||
|
.thot-catalog-drawer__run-layout .thot-catalog-drawer__history-list {
|
||||||
|
min-height: 0;
|
||||||
|
max-height: none;
|
||||||
|
flex: 1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@media (max-width: 47.999rem) {
|
@media (max-width: 47.999rem) {
|
||||||
|
|||||||
@@ -24,4 +24,17 @@ describe("Recent runs success styling contract", () => {
|
|||||||
expect(styles).toContain('button[data-status="succeeded"]');
|
expect(styles).toContain('button[data-status="succeeded"]');
|
||||||
expect(styles).toMatch(/oklch\(var\(--success\)/);
|
expect(styles).toMatch(/oklch\(var\(--success\)/);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("run drawers keep responsive gutters and switch columns from their own width", () => {
|
||||||
|
const styles = readFileSync(resolve(sourceRoot, "FleetLedgerShell.css"), "utf8");
|
||||||
|
|
||||||
|
expect(styles).toContain("width: min(75rem, calc(100% - 2rem));");
|
||||||
|
expect(styles).toContain(
|
||||||
|
".thot-fleet-ledger__drawer-body.thot-catalog-drawer__run-layout",
|
||||||
|
);
|
||||||
|
expect(styles).toContain("padding: clamp(0.75rem, 1.5cqi, 1.25rem);");
|
||||||
|
expect(styles).toContain("@container work-area-panel (min-width: 48rem)");
|
||||||
|
expect(styles).toContain(".thot-catalog-drawer__history-list");
|
||||||
|
expect(styles).toContain(".thot-catalog-drawer__event-log");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -97,6 +97,11 @@ test("shows sensitive suggestion results, safe events, and lets operators inspec
|
|||||||
|
|
||||||
const drawer = await screen.findByRole("dialog", { name: "Sensitive suggestion history" });
|
const drawer = await screen.findByRole("dialog", { name: "Sensitive suggestion history" });
|
||||||
expect(within(drawer).getByRole("heading", { name: "Running" })).toBeVisible();
|
expect(within(drawer).getByRole("heading", { name: "Running" })).toBeVisible();
|
||||||
|
const progress = within(drawer).getByRole("table", { name: "Sensitive suggestion progress" });
|
||||||
|
expect(within(progress).getAllByRole("columnheader")).toHaveLength(3);
|
||||||
|
expect(within(drawer).getByRole("log", { name: "Sensitive suggestion events" })).toHaveClass(
|
||||||
|
"thot-catalog-drawer__event-log",
|
||||||
|
);
|
||||||
expect(await within(drawer).findByText("Classifying columns")).toBeVisible();
|
expect(await within(drawer).findByText("Classifying columns")).toBeVisible();
|
||||||
expect(within(drawer).queryByRole("button", { name: "Stop" })).not.toBeInTheDocument();
|
expect(within(drawer).queryByRole("button", { name: "Stop" })).not.toBeInTheDocument();
|
||||||
expect(within(drawer).queryByRole("button", { name: "Unlock stale run" })).not.toBeInTheDocument();
|
expect(within(drawer).queryByRole("button", { name: "Unlock stale run" })).not.toBeInTheDocument();
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { useEffect, useMemo } from "react";
|
import { useEffect, useMemo } from "react";
|
||||||
|
import { LoaderCircle } from "lucide-react";
|
||||||
import { useQuery, useQueryClient } from "@tanstack/react-query";
|
import { useQuery, useQueryClient } from "@tanstack/react-query";
|
||||||
import { apiErrorMessage } from "../../api/client";
|
import { apiErrorMessage } from "../../api/client";
|
||||||
import {
|
import {
|
||||||
@@ -30,6 +31,12 @@ function isTerminal(run?: SensitiveDataSuggestionRun): boolean {
|
|||||||
return Boolean(run && ["completed", "failed", "interrupted"].includes(run.status));
|
return Boolean(run && ["completed", "failed", "interrupted"].includes(run.status));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function outcomeClass(run: SensitiveDataSuggestionRun): string {
|
||||||
|
if (run.status === "completed") return "thot-sensitive-run--success";
|
||||||
|
if (["failed", "interrupted"].includes(run.status)) return "thot-sensitive-run--failure";
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
function safeFinalError(summary: string | null): string | null {
|
function safeFinalError(summary: string | null): string | null {
|
||||||
const value = summary?.trim();
|
const value = summary?.trim();
|
||||||
if (!value) return null;
|
if (!value) return null;
|
||||||
@@ -133,56 +140,92 @@ export function SensitiveDataSuggestionHistoryDrawer({
|
|||||||
|
|
||||||
const events = eventQuery.data ?? [];
|
const events = eventQuery.data ?? [];
|
||||||
const finalError = safeFinalError(run.errorSummary);
|
const finalError = safeFinalError(run.errorSummary);
|
||||||
|
const modelDisplay = modelLabel && modelLabel !== run.modelId
|
||||||
|
? `${modelLabel} (${run.modelId})`
|
||||||
|
: run.modelId;
|
||||||
|
const progressCounters = [
|
||||||
|
["total", "Total columns"],
|
||||||
|
["suggestedSensitive", "Sensitive"],
|
||||||
|
["suggestedNonSensitive", "Not sensitive"],
|
||||||
|
] as const;
|
||||||
return (
|
return (
|
||||||
<FleetLedgerDrawer
|
<FleetLedgerDrawer
|
||||||
open
|
open
|
||||||
|
className={outcomeClass(run)}
|
||||||
ariaLabel="Sensitive suggestion history"
|
ariaLabel="Sensitive suggestion history"
|
||||||
eyebrow="Sensitive data suggestions"
|
eyebrow="Sensitive data suggestions"
|
||||||
title={statusLabel(run)}
|
title={(
|
||||||
description={`${modelLabel}${modelLabel !== run.modelId ? ` (${run.modelId})` : ""} · ${run.scope.replaceAll("_", " ")}`}
|
<span className="inline-flex items-center gap-2">
|
||||||
|
{!isTerminal(run) ? <LoaderCircle className="size-5 animate-spin" aria-hidden="true" /> : null}
|
||||||
|
{statusLabel(run)}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
description={run.scope.replaceAll("_", " ")}
|
||||||
onClose={onClose}
|
onClose={onClose}
|
||||||
closeLabel="Close sensitive suggestion history"
|
closeLabel="Close sensitive suggestion history"
|
||||||
bodyClassName="thot-catalog-drawer__history-layout"
|
bodyClassName="thot-catalog-drawer__history-layout thot-catalog-drawer__run-layout"
|
||||||
>
|
>
|
||||||
{runQuery.isError ? (
|
<div className="thot-catalog-drawer__run-main">
|
||||||
|
{runQuery.isError ? (
|
||||||
<div role="alert" className="rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive">
|
<div role="alert" className="rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive">
|
||||||
{apiErrorMessage(runQuery.error)}
|
{apiErrorMessage(runQuery.error)}
|
||||||
</div>
|
</div>
|
||||||
) : null}
|
) : null}
|
||||||
|
|
||||||
<section aria-label="Sensitive suggestion counters">
|
<div className={`grid gap-4 sm:grid-cols-2${runQuery.isError ? " mt-4" : ""}`}>
|
||||||
|
<section aria-label="AI usage">
|
||||||
|
<h3 className="thot-label mb-2">AI usage</h3>
|
||||||
|
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1 text-xs">
|
||||||
|
<dt>Model</dt><dd className="truncate text-right" title={modelDisplay}>{modelDisplay}</dd>
|
||||||
|
<dt>Input tokens</dt><dd className="text-right tabular-nums">{(run.inputTokens ?? 0).toLocaleString("en-US")}</dd>
|
||||||
|
<dt>Cache tokens</dt><dd className="text-right tabular-nums">{(run.cacheReadTokens ?? 0).toLocaleString("en-US")}</dd>
|
||||||
|
<dt>Output tokens</dt><dd className="text-right tabular-nums">{(run.outputTokens ?? 0).toLocaleString("en-US")}</dd>
|
||||||
|
</dl>
|
||||||
|
</section>
|
||||||
|
<section aria-label="Sensitive suggestion timestamps">
|
||||||
|
<h3 className="thot-label mb-2">Timestamps</h3>
|
||||||
|
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1 text-xs">
|
||||||
|
<dt className="text-muted-foreground">Created</dt><dd className="text-right tabular-nums">{timestamp(run.createdAt)}</dd>
|
||||||
|
<dt className="text-muted-foreground">Started</dt><dd className="text-right tabular-nums">{timestamp(run.startedAt)}</dd>
|
||||||
|
<dt className="text-muted-foreground">Updated</dt><dd className="text-right tabular-nums">{timestamp(run.updatedAt)}</dd>
|
||||||
|
<dt className="text-muted-foreground">Finished</dt><dd className="text-right tabular-nums">{timestamp(run.finishedAt)}</dd>
|
||||||
|
</dl>
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<section className="mt-4" aria-label="Sensitive suggestion counters">
|
||||||
<h3 className="thot-label mb-2">Results</h3>
|
<h3 className="thot-label mb-2">Results</h3>
|
||||||
<div className="grid grid-cols-2 gap-2">
|
<div className="overflow-x-auto rounded-md border border-border bg-muted/20">
|
||||||
{([
|
<table className="w-full min-w-[20rem] table-fixed" aria-label="Sensitive suggestion progress">
|
||||||
["total", "Total columns"],
|
<thead>
|
||||||
["suggestedSensitive", "Sensitive"],
|
<tr className="border-b border-border bg-muted/45">
|
||||||
["suggestedNonSensitive", "Not sensitive"],
|
{progressCounters.map(([name, label]) => (
|
||||||
] as const).map(([name, label]) => (
|
<th key={name} scope="col" className="border-l border-border px-2 py-1.5 text-left text-[10px] font-semibold uppercase tracking-wide text-muted-foreground first:border-l-0">
|
||||||
<div key={name} className="rounded-md border border-border bg-muted/25 px-3 py-2">
|
{label}
|
||||||
<p className="text-[11px] font-semibold uppercase tracking-wide text-muted-foreground">{label}</p>
|
</th>
|
||||||
<p data-count={name} className="mt-1 text-lg font-semibold tabular-nums">{run[name]}</p>
|
))}
|
||||||
</div>
|
</tr>
|
||||||
))}
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
{progressCounters.map(([name]) => (
|
||||||
|
<td key={name} data-count={name} className="border-l border-border px-2 py-1.5 text-base font-semibold tabular-nums first:border-l-0">
|
||||||
|
{run[name]}
|
||||||
|
</td>
|
||||||
|
))}
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
<section className="mt-5" aria-label="Sensitive suggestion timestamps">
|
|
||||||
<h3 className="thot-label mb-2">Timestamps</h3>
|
|
||||||
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1.5 text-sm">
|
|
||||||
<dt className="text-muted-foreground">Created</dt><dd className="text-right">{timestamp(run.createdAt)}</dd>
|
|
||||||
<dt className="text-muted-foreground">Started</dt><dd className="text-right">{timestamp(run.startedAt)}</dd>
|
|
||||||
<dt className="text-muted-foreground">Updated</dt><dd className="text-right">{timestamp(run.updatedAt)}</dd>
|
|
||||||
<dt className="text-muted-foreground">Finished</dt><dd className="text-right">{timestamp(run.finishedAt)}</dd>
|
|
||||||
</dl>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
{finalError ? (
|
{finalError ? (
|
||||||
<div className="mt-5 rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive">
|
<div className="mt-3 rounded-md border border-destructive/35 bg-destructive/5 px-4 py-3 text-sm text-destructive">
|
||||||
{finalError}
|
{finalError}
|
||||||
</div>
|
</div>
|
||||||
) : null}
|
) : null}
|
||||||
|
|
||||||
<section className="mt-5" aria-label="Sensitive suggestion log">
|
<section className="thot-catalog-drawer__events mt-3" aria-label="Sensitive suggestion log">
|
||||||
<div className="mb-2 flex items-center justify-between">
|
<div className="mb-2 flex items-center justify-between">
|
||||||
<h3 className="thot-label">Events</h3>
|
<h3 className="thot-label">Events</h3>
|
||||||
<span className="text-xs tabular-nums text-muted-foreground">{events.length}</span>
|
<span className="text-xs tabular-nums text-muted-foreground">{events.length}</span>
|
||||||
@@ -190,7 +233,7 @@ export function SensitiveDataSuggestionHistoryDrawer({
|
|||||||
<div
|
<div
|
||||||
role="log"
|
role="log"
|
||||||
aria-label="Sensitive suggestion events"
|
aria-label="Sensitive suggestion events"
|
||||||
className="max-h-72 overflow-y-auto rounded-md bg-zinc-950 p-3 font-mono text-xs leading-5 text-zinc-200"
|
className="thot-catalog-drawer__event-log overflow-y-auto rounded-md bg-zinc-950 p-3 font-mono text-xs leading-5 text-zinc-200"
|
||||||
>
|
>
|
||||||
{events.length === 0 ? <p className="text-zinc-500">Waiting for events…</p> : events.map((event) => (
|
{events.length === 0 ? <p className="text-zinc-500">Waiting for events…</p> : events.map((event) => (
|
||||||
<p key={event.sequence} className={event.level === "error" ? "text-red-300" : event.level === "warning" ? "text-amber-300" : undefined}>
|
<p key={event.sequence} className={event.level === "error" ? "text-red-300" : event.level === "warning" ? "text-amber-300" : undefined}>
|
||||||
@@ -200,8 +243,9 @@ export function SensitiveDataSuggestionHistoryDrawer({
|
|||||||
</div>
|
</div>
|
||||||
{eventQuery.isError ? <p role="alert" className="mt-2 text-sm text-destructive">{apiErrorMessage(eventQuery.error)}</p> : null}
|
{eventQuery.isError ? <p role="alert" className="mt-2 text-sm text-destructive">{apiErrorMessage(eventQuery.error)}</p> : null}
|
||||||
</section>
|
</section>
|
||||||
|
</div>
|
||||||
|
|
||||||
<section className="thot-catalog-drawer__history mt-5" aria-label="Sensitive suggestion run history">
|
<section className="thot-catalog-drawer__history" aria-label="Sensitive suggestion run history">
|
||||||
<div className="mb-2 flex items-center justify-between gap-3">
|
<div className="mb-2 flex items-center justify-between gap-3">
|
||||||
<h3 className="thot-label">Recent runs</h3>
|
<h3 className="thot-label">Recent runs</h3>
|
||||||
<span className="text-xs tabular-nums text-muted-foreground">
|
<span className="text-xs tabular-nums text-muted-foreground">
|
||||||
@@ -215,7 +259,7 @@ export function SensitiveDataSuggestionHistoryDrawer({
|
|||||||
) : visibleHistory.length === 0 ? (
|
) : visibleHistory.length === 0 ? (
|
||||||
<p className="text-sm text-muted-foreground">No sensitive suggestion runs yet.</p>
|
<p className="text-sm text-muted-foreground">No sensitive suggestion runs yet.</p>
|
||||||
) : (
|
) : (
|
||||||
<div className="max-h-56 divide-y divide-border overflow-y-auto rounded-md border border-border">
|
<div className="thot-catalog-drawer__history-list divide-y divide-border overflow-y-auto rounded-md border border-border">
|
||||||
{visibleHistory.map((item) => (
|
{visibleHistory.map((item) => (
|
||||||
<button
|
<button
|
||||||
key={item.id}
|
key={item.id}
|
||||||
|
|||||||
@@ -265,6 +265,36 @@ def _response_content(response: object) -> str:
|
|||||||
return content
|
return content
|
||||||
|
|
||||||
|
|
||||||
|
def _response_usage(response: object) -> dict[str, int] | None:
|
||||||
|
"""Normalize LiteLLM usage, tolerating providers that omit it."""
|
||||||
|
try:
|
||||||
|
usage = _field(response, "usage")
|
||||||
|
prompt = _field(usage, "prompt_tokens")
|
||||||
|
output = _field(usage, "completion_tokens")
|
||||||
|
if type(prompt) is not int or prompt < 0 or type(output) is not int or output < 0:
|
||||||
|
return None
|
||||||
|
cached = 0
|
||||||
|
for source, name in ((usage, "cache_read_input_tokens"), (usage, "cached_tokens")):
|
||||||
|
try:
|
||||||
|
value = _field(source, name)
|
||||||
|
except _InvalidResponse:
|
||||||
|
continue
|
||||||
|
if type(value) is int and value >= 0:
|
||||||
|
cached = value
|
||||||
|
break
|
||||||
|
try:
|
||||||
|
details = _field(usage, "prompt_tokens_details")
|
||||||
|
value = _field(details, "cached_tokens")
|
||||||
|
if type(value) is int and value >= 0:
|
||||||
|
cached = value
|
||||||
|
except _InvalidResponse:
|
||||||
|
pass
|
||||||
|
cached = min(cached, prompt)
|
||||||
|
return {"input": prompt - cached, "cacheRead": cached, "output": output}
|
||||||
|
except _InvalidResponse:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def handle_request(request: object, *, completion: Completion | None = None) -> Result:
|
def handle_request(request: object, *, completion: Completion | None = None) -> Result:
|
||||||
"""Validate one request and normalize one injected or LiteLLM completion."""
|
"""Validate one request and normalize one injected or LiteLLM completion."""
|
||||||
try:
|
try:
|
||||||
@@ -284,7 +314,11 @@ def handle_request(request: object, *, completion: Completion | None = None) ->
|
|||||||
content = _response_content(response)
|
content = _response_content(response)
|
||||||
except Exception: # noqa: BLE001 - arbitrary provider objects are untrusted response data
|
except Exception: # noqa: BLE001 - arbitrary provider objects are untrusted response data
|
||||||
return {"ok": False, "error": "invalid_response"}
|
return {"ok": False, "error": "invalid_response"}
|
||||||
return {"ok": True, "content": content}
|
result: dict[str, object] = {"ok": True, "content": content}
|
||||||
|
usage = _response_usage(response)
|
||||||
|
if usage is not None:
|
||||||
|
result["usage"] = usage
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
def _write_json(payload: Mapping[str, object]) -> None:
|
def _write_json(payload: Mapping[str, object]) -> None:
|
||||||
|
|||||||
@@ -75,6 +75,10 @@ func TestLoadAcceptsMixedKeyedAndKeylessMetadataGenerationModels(t *testing.T) {
|
|||||||
label: DeepSeek V4 Pro
|
label: DeepSeek V4 Pro
|
||||||
litellm: {provider: deepseek, model: deepseek-v4-pro}
|
litellm: {provider: deepseek, model: deepseek-v4-pro}
|
||||||
apiKeyEnv: DEEPSEEK_API_KEY
|
apiKeyEnv: DEEPSEEK_API_KEY
|
||||||
|
- id: deepseek-v4-flash
|
||||||
|
label: DeepSeek V4 Flash
|
||||||
|
litellm: {provider: deepseek, model: deepseek-v4-flash}
|
||||||
|
apiKeyEnv: DEEPSEEK_API_KEY
|
||||||
- id: glm-53
|
- id: glm-53
|
||||||
label: GLM 5.3
|
label: GLM 5.3
|
||||||
litellm:
|
litellm:
|
||||||
@@ -95,10 +99,10 @@ func TestLoadAcceptsMixedKeyedAndKeylessMetadataGenerationModels(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Load() error = %v", err)
|
t.Fatalf("Load() error = %v", err)
|
||||||
}
|
}
|
||||||
if len(installation.MetadataGeneration.Models) != 3 {
|
if len(installation.MetadataGeneration.Models) != 4 {
|
||||||
t.Fatalf("metadata models = %#v", installation.MetadataGeneration.Models)
|
t.Fatalf("metadata models = %#v", installation.MetadataGeneration.Models)
|
||||||
}
|
}
|
||||||
qwen := installation.MetadataGeneration.Models[2]
|
qwen := installation.MetadataGeneration.Models[3]
|
||||||
if qwen.APIKeyEnv != "" || !qwen.LiteLLM.DisableThinking || qwen.LiteLLM.Endpoint == nil {
|
if qwen.APIKeyEnv != "" || !qwen.LiteLLM.DisableThinking || qwen.LiteLLM.Endpoint == nil {
|
||||||
t.Fatalf("keyless qwen model = %#v", qwen)
|
t.Fatalf("keyless qwen model = %#v", qwen)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user