feat: add AI catalog description generation
This commit is contained in:
Codex
+51
-2
@@ -2,7 +2,7 @@ import Fastify, { type FastifyInstance, type FastifyRequest } from "fastify";
|
||||
import cors from "@fastify/cors";
|
||||
import cookie from "@fastify/cookie";
|
||||
import rateLimit from "@fastify/rate-limit";
|
||||
import { join } from "node:path";
|
||||
import { dirname, isAbsolute, join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import type { AppConfig } from "./config.js";
|
||||
import { ThtRunner } from "./tht/tht-runner.js";
|
||||
@@ -48,6 +48,19 @@ import { catalogTableRoutes } from "./routes/catalog-tables.js";
|
||||
import { ConcreteCatalogSchemaIntrospector, type CatalogSchemaIntrospector } from "./catalog/schema-introspector.js";
|
||||
import { CatalogSyncWorker } from "./catalog/sync-worker.js";
|
||||
import { catalogSchemaRoutes } from "./routes/catalog-schema.js";
|
||||
import {
|
||||
loadMetadataGenerationModels,
|
||||
type MetadataGenerationModels,
|
||||
} from "./catalog/metadata-generation-models.js";
|
||||
import { metadataGenerationModelRoutes } from "./routes/metadata-generation-models.js";
|
||||
import { catalogDescriptionConsolidationRoutes } from "./routes/catalog-description-consolidation.js";
|
||||
import { PythonModelCompleter, type ModelCompleter } from "./catalog/model-completer.js";
|
||||
import { DescriptionGenerationWorker } from "./catalog/description-generation-worker.js";
|
||||
import {
|
||||
PostgresDescriptionSourceSampler,
|
||||
type DescriptionSourceSampler,
|
||||
} from "./catalog/description-source-sampler.js";
|
||||
import { catalogDescriptionGenerationRoutes } from "./routes/catalog-description-generation.js";
|
||||
|
||||
export interface BuildAppDeps {
|
||||
thtRunner?: ThtRunner;
|
||||
@@ -67,6 +80,9 @@ export interface BuildAppDeps {
|
||||
catalogSchemaIntrospector?: CatalogSchemaIntrospector;
|
||||
catalogSyncWorker?: CatalogSyncWorker;
|
||||
catalogOperationCoordinator?: CatalogOperationCoordinator;
|
||||
metadataGenerationModels?: MetadataGenerationModels;
|
||||
modelCompleter?: ModelCompleter;
|
||||
descriptionSourceSampler?: DescriptionSourceSampler;
|
||||
workspaceRuntimeSupport?: (workspace: WorkspaceDescriptor) => boolean;
|
||||
maintenanceBarrier?: MaintenanceBarrier;
|
||||
piManagement?: PiManagementService;
|
||||
@@ -141,10 +157,28 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
const workspaceRegistry = deps?.workspaceRegistry ?? new WorkspaceRegistry(config.workspaceRegistry);
|
||||
const catalogRepository = deps?.catalogRepository ?? createCatalogRepository(config.catalogDatabase);
|
||||
const catalogOperationCoordinator = deps?.catalogOperationCoordinator ?? new CatalogOperationCoordinator();
|
||||
const metadataGenerationModels = deps?.metadataGenerationModels ?? loadMetadataGenerationModels({
|
||||
installationFile: config.installationConfigFile,
|
||||
secretsFile: config.secretsFile,
|
||||
});
|
||||
const modelCompleter = deps?.modelCompleter ?? new PythonModelCompleter({
|
||||
pythonExecutable: isAbsolute(config.thtBin) ? join(dirname(config.thtBin), "python") : "python3",
|
||||
cwd: config.harnessDir,
|
||||
});
|
||||
const catalogPostgresAccess = deps?.catalogPostgresAccess ?? new ConcreteCatalogPostgresAccess(
|
||||
workspaceSecretStore,
|
||||
{ connectTimeoutMs: config.workspaceDiagnosticTimeoutMs },
|
||||
);
|
||||
const descriptionSourceSampler = deps?.descriptionSourceSampler
|
||||
?? new PostgresDescriptionSourceSampler(catalogPostgresAccess);
|
||||
const descriptionGenerationWorker = new DescriptionGenerationWorker(
|
||||
catalogRepository,
|
||||
workspaceRegistry,
|
||||
metadataGenerationModels,
|
||||
modelCompleter,
|
||||
catalogOperationCoordinator,
|
||||
descriptionSourceSampler,
|
||||
);
|
||||
const catalogService = deps?.catalogService ?? new CatalogService(
|
||||
catalogRepository,
|
||||
workspaceRegistry,
|
||||
@@ -166,10 +200,12 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
config.catalogSyncTimeoutMs,
|
||||
);
|
||||
app.addHook("onReady", async () => { await catalogSyncWorker.initialize(); });
|
||||
app.addHook("onReady", async () => { await descriptionGenerationWorker.initialize(); });
|
||||
if (!deps?.catalogRepository && catalogRepository.close) {
|
||||
app.addHook("onClose", async () => { await catalogRepository.close?.(); });
|
||||
}
|
||||
app.addHook("onClose", async () => { await catalogSyncWorker.stop(); });
|
||||
app.addHook("onClose", async () => { await descriptionGenerationWorker.stop(); });
|
||||
const workspaceDiagnoser = deps?.workspaceDiagnoser
|
||||
?? createProductionWorkspaceDiagnoser(config.workspaceDiagnosticTimeoutMs, undefined, {
|
||||
internalQdrantUrl: config.internalQdrantUrl,
|
||||
@@ -384,12 +420,25 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
secretStore: workspaceSecretStore,
|
||||
});
|
||||
catalogDatabaseRoutes(app, { repository: catalogRepository, service: catalogService, operations: catalogOperationCoordinator });
|
||||
catalogTableRoutes(app, { repository: catalogRepository, service: catalogTableService });
|
||||
catalogTableRoutes(app, {
|
||||
repository: catalogRepository,
|
||||
service: catalogTableService,
|
||||
operations: catalogOperationCoordinator,
|
||||
});
|
||||
catalogSchemaRoutes(app, {
|
||||
repository: catalogRepository,
|
||||
worker: catalogSyncWorker,
|
||||
operations: catalogOperationCoordinator,
|
||||
});
|
||||
catalogDescriptionConsolidationRoutes(app, {
|
||||
repository: catalogRepository,
|
||||
operations: catalogOperationCoordinator,
|
||||
});
|
||||
metadataGenerationModelRoutes(app, metadataGenerationModels);
|
||||
catalogDescriptionGenerationRoutes(app, {
|
||||
repository: catalogRepository,
|
||||
worker: descriptionGenerationWorker,
|
||||
});
|
||||
settingsRoutes(app, { cfg: config, listModels, getSettings });
|
||||
piManagementRoutes(app, { service: piManagement });
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,163 @@
|
||||
import type { CatalogPostgresAccess } from "./postgres-access.js";
|
||||
import type { WorkspaceDatabase } from "./types.js";
|
||||
|
||||
const MAX_SOURCE_ROWS = 5;
|
||||
const MAX_REPRESENTATIVE_VALUES = 5;
|
||||
const MAX_SOURCE_COLUMNS_PER_TARGET = 8;
|
||||
const MAX_SOURCE_VALUE_BYTES = 256;
|
||||
|
||||
export type DescriptionSourceSampleValue = string | number | boolean | null;
|
||||
|
||||
export interface DescriptionSourceSampleField {
|
||||
name: string;
|
||||
value: DescriptionSourceSampleValue;
|
||||
}
|
||||
|
||||
export interface DescriptionSourceSampleRow {
|
||||
fields: readonly DescriptionSourceSampleField[];
|
||||
}
|
||||
|
||||
export interface DescriptionSourceRepresentativeValues {
|
||||
column: string;
|
||||
values: readonly Exclude<DescriptionSourceSampleValue, null>[];
|
||||
}
|
||||
|
||||
export interface DescriptionTargetSourceSample {
|
||||
targetId: string;
|
||||
tableName: string;
|
||||
rows: readonly DescriptionSourceSampleRow[];
|
||||
representativeValues: readonly DescriptionSourceRepresentativeValues[];
|
||||
}
|
||||
|
||||
export interface DescriptionSourceSamplingTarget {
|
||||
targetId: string;
|
||||
tableName: string;
|
||||
columnNames: readonly string[];
|
||||
}
|
||||
|
||||
/** Optional, transient source context for one model-completion batch. */
|
||||
export interface DescriptionSourceSampler {
|
||||
sample(
|
||||
database: WorkspaceDatabase,
|
||||
targets: readonly DescriptionSourceSamplingTarget[],
|
||||
signal: AbortSignal,
|
||||
): Promise<readonly DescriptionTargetSourceSample[]>;
|
||||
}
|
||||
|
||||
function quoteIdentifier(identifier: string): string {
|
||||
return `"${identifier.replaceAll('"', '""')}"`;
|
||||
}
|
||||
|
||||
function boundedUtf8(value: string, maxBytes: number): string {
|
||||
const normalized = value
|
||||
.normalize("NFC")
|
||||
.replace(/\r\n?/g, "\n")
|
||||
.replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, " ");
|
||||
if (Buffer.byteLength(normalized, "utf8") <= maxBytes) return normalized;
|
||||
let result = "";
|
||||
let bytes = 0;
|
||||
for (const character of normalized) {
|
||||
const characterBytes = Buffer.byteLength(character, "utf8");
|
||||
if (bytes + characterBytes > maxBytes) break;
|
||||
result += character;
|
||||
bytes += characterBytes;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function normalizeValue(value: unknown): DescriptionSourceSampleValue | undefined {
|
||||
if (value === null) return null;
|
||||
if (typeof value === "string") return boundedUtf8(value, MAX_SOURCE_VALUE_BYTES);
|
||||
if (typeof value === "boolean") return value;
|
||||
if (typeof value === "number") return Number.isFinite(value) ? value : undefined;
|
||||
if (typeof value === "bigint") return boundedUtf8(String(value), MAX_SOURCE_VALUE_BYTES);
|
||||
if (value instanceof Date && !Number.isNaN(value.valueOf())) return value.toISOString();
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function distinctKey(value: Exclude<DescriptionSourceSampleValue, null>): string {
|
||||
return `${typeof value}:${String(value)}`;
|
||||
}
|
||||
|
||||
/** PostgreSQL-wire sampler. REST bindings remain unsupported by CatalogPostgresAccess. */
|
||||
export class PostgresDescriptionSourceSampler implements DescriptionSourceSampler {
|
||||
constructor(private readonly access: CatalogPostgresAccess) {}
|
||||
|
||||
async sample(
|
||||
database: WorkspaceDatabase,
|
||||
targets: readonly DescriptionSourceSamplingTarget[],
|
||||
signal: AbortSignal,
|
||||
): Promise<readonly DescriptionTargetSourceSample[]> {
|
||||
const client = await this.access.connect(database, signal);
|
||||
let transactionOpen = false;
|
||||
try {
|
||||
await client.query("BEGIN TRANSACTION READ ONLY", []);
|
||||
transactionOpen = true;
|
||||
const samples: DescriptionTargetSourceSample[] = [];
|
||||
for (const target of targets) {
|
||||
const columnNames = [...new Set(target.columnNames)].slice(0, MAX_SOURCE_COLUMNS_PER_TARGET);
|
||||
if (columnNames.length === 0) {
|
||||
samples.push({
|
||||
targetId: target.targetId,
|
||||
tableName: target.tableName,
|
||||
rows: [],
|
||||
representativeValues: [],
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const projections = columnNames.map((columnName) => {
|
||||
const identifier = quoteIdentifier(columnName);
|
||||
return `LEFT((${identifier})::text, $1) AS ${identifier}`;
|
||||
});
|
||||
const sql = [
|
||||
`SELECT ${projections.join(", ")}`,
|
||||
`FROM ${quoteIdentifier(database.schema)}.${quoteIdentifier(target.tableName)}`,
|
||||
"LIMIT $2",
|
||||
].join(" ");
|
||||
const result = await client.query(sql, [MAX_SOURCE_VALUE_BYTES, MAX_SOURCE_ROWS]);
|
||||
const rows = result.rows.slice(0, MAX_SOURCE_ROWS).map((row) => ({
|
||||
fields: columnNames.flatMap((name) => {
|
||||
const value = normalizeValue(row[name]);
|
||||
return value === undefined ? [] : [{ name, value }];
|
||||
}),
|
||||
}));
|
||||
const valuesByColumn = new Map<
|
||||
string,
|
||||
Exclude<DescriptionSourceSampleValue, null>[]
|
||||
>();
|
||||
const seenByColumn = new Map<string, Set<string>>();
|
||||
let representativeValueCount = 0;
|
||||
for (const row of rows) {
|
||||
for (const field of row.fields) {
|
||||
if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break;
|
||||
if (field.value === null) continue;
|
||||
const seen = seenByColumn.get(field.name) ?? new Set<string>();
|
||||
const key = distinctKey(field.value);
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
seenByColumn.set(field.name, seen);
|
||||
const values = valuesByColumn.get(field.name) ?? [];
|
||||
values.push(field.value);
|
||||
valuesByColumn.set(field.name, values);
|
||||
representativeValueCount += 1;
|
||||
}
|
||||
if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break;
|
||||
}
|
||||
const representativeValues = columnNames.flatMap((column) => {
|
||||
const values = valuesByColumn.get(column);
|
||||
return values && values.length > 0 ? [{ column, values }] : [];
|
||||
});
|
||||
samples.push({
|
||||
targetId: target.targetId,
|
||||
tableName: target.tableName,
|
||||
rows,
|
||||
representativeValues,
|
||||
});
|
||||
}
|
||||
return samples;
|
||||
} finally {
|
||||
if (transactionOpen) await client.query("ROLLBACK", []).catch(() => undefined);
|
||||
await client.end().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,10 @@ import { randomUUID } from "node:crypto";
|
||||
import {
|
||||
CatalogConflictError,
|
||||
CatalogConnectorError,
|
||||
DescriptionGenerationRunActiveError,
|
||||
type CatalogColumn,
|
||||
type CatalogDescriptionConsolidationCounts,
|
||||
type CatalogDescriptionTarget,
|
||||
type CatalogDatabaseMetadataDeleteTarget,
|
||||
type CatalogMetadataDeleteCounts,
|
||||
type CatalogRelationship,
|
||||
@@ -19,6 +22,10 @@ import {
|
||||
type DatabaseTestResult,
|
||||
type ObservedCatalogTable,
|
||||
type ObservedSchemaSnapshot,
|
||||
type DescriptionGenerationEvent,
|
||||
type DescriptionGenerationRun,
|
||||
type DescriptionGenerationRunUpdate,
|
||||
type DescriptionGenerationScope,
|
||||
type TableSyncRepositoryResult,
|
||||
type WorkspaceDatabase,
|
||||
} from "./types.js";
|
||||
@@ -33,6 +40,8 @@ export class MemoryCatalogRepository implements CatalogRepository {
|
||||
private readonly tables = new Map<string, CatalogTable>();
|
||||
private readonly columns = new Map<string, CatalogColumn>();
|
||||
private readonly relationships = new Map<string, CatalogRelationship>();
|
||||
private readonly descriptionGenerationRuns = new Map<string, DescriptionGenerationRun>();
|
||||
private readonly descriptionGenerationEvents = new Map<string, DescriptionGenerationEvent[]>();
|
||||
private readonly syncRuns = new Map<string, CatalogSyncRun>();
|
||||
private readonly syncEvents = new Map<string, CatalogSyncEvent[]>();
|
||||
|
||||
@@ -122,6 +131,11 @@ export class MemoryCatalogRepository implements CatalogRepository {
|
||||
for (const [relationshipId, relationship] of this.relationships) {
|
||||
if (relationship.databaseId === id) this.relationships.delete(relationshipId);
|
||||
}
|
||||
for (const [runId, run] of this.descriptionGenerationRuns) {
|
||||
if (run.databaseId !== id) continue;
|
||||
this.descriptionGenerationRuns.delete(runId);
|
||||
this.descriptionGenerationEvents.delete(runId);
|
||||
}
|
||||
return this.records.delete(id);
|
||||
}
|
||||
async listTables(databaseId: string): Promise<CatalogTable[]> {
|
||||
@@ -196,6 +210,166 @@ export class MemoryCatalogRepository implements CatalogRepository {
|
||||
return structuredClone(updated);
|
||||
}
|
||||
|
||||
async consolidateGeneratedDescriptions(
|
||||
databaseId: string,
|
||||
target: CatalogDescriptionTarget,
|
||||
targetIds: readonly string[],
|
||||
): Promise<CatalogDescriptionConsolidationCounts | undefined> {
|
||||
const selectedTargetIds = [...new Set(targetIds)];
|
||||
if (!this.records.has(databaseId) || selectedTargetIds.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
const now = new Date().toISOString();
|
||||
if (target === "tables") {
|
||||
const targets = selectedTargetIds.map((id) => this.tables.get(id));
|
||||
if (targets.some((table) => !table || table.databaseId !== databaseId)) return undefined;
|
||||
const copied = targets.filter((table) => Boolean(table!.generatedDescription?.trim()));
|
||||
for (const table of copied) {
|
||||
this.tables.set(table!.id, {
|
||||
...table!,
|
||||
description: table!.generatedDescription,
|
||||
version: table!.version + 1,
|
||||
updatedAt: now,
|
||||
});
|
||||
}
|
||||
return { copied: copied.length, skipped: targets.length - copied.length };
|
||||
}
|
||||
|
||||
const targets = selectedTargetIds.map((id) => this.columns.get(id));
|
||||
if (targets.some((column) => (
|
||||
!column || this.tables.get(column.tableId)?.databaseId !== databaseId
|
||||
))) return undefined;
|
||||
const copied = targets.filter((column) => Boolean(column!.generatedDescription?.trim()));
|
||||
for (const column of copied) {
|
||||
this.columns.set(column!.id, {
|
||||
...column!,
|
||||
description: column!.generatedDescription,
|
||||
version: column!.version + 1,
|
||||
updatedAt: now,
|
||||
});
|
||||
}
|
||||
return { copied: copied.length, skipped: targets.length - copied.length };
|
||||
}
|
||||
|
||||
async createDescriptionGenerationRun(
|
||||
databaseId: string,
|
||||
scope: DescriptionGenerationScope,
|
||||
modelId: string,
|
||||
language: DescriptionGenerationRun["language"],
|
||||
total: number,
|
||||
): Promise<DescriptionGenerationRun> {
|
||||
if ([...this.descriptionGenerationRuns.values()].some((run) => (
|
||||
run.status === "queued" || run.status === "running"
|
||||
))) {
|
||||
throw new DescriptionGenerationRunActiveError("A description generation run is already active");
|
||||
}
|
||||
const now = new Date().toISOString();
|
||||
const run: DescriptionGenerationRun = {
|
||||
id: randomUUID(),
|
||||
databaseId,
|
||||
scope,
|
||||
modelId,
|
||||
language,
|
||||
status: "queued",
|
||||
total,
|
||||
processed: 0,
|
||||
generated: 0,
|
||||
nonGeneratable: 0,
|
||||
failed: 0,
|
||||
createdAt: now,
|
||||
startedAt: null,
|
||||
updatedAt: now,
|
||||
finishedAt: null,
|
||||
errorSummary: null,
|
||||
};
|
||||
this.descriptionGenerationRuns.set(run.id, run);
|
||||
return structuredClone(run);
|
||||
}
|
||||
|
||||
async getDescriptionGenerationRun(runId: string): Promise<DescriptionGenerationRun | undefined> {
|
||||
const run = this.descriptionGenerationRuns.get(runId);
|
||||
return run ? structuredClone(run) : undefined;
|
||||
}
|
||||
|
||||
async listDescriptionGenerationRuns(limit = 50): Promise<DescriptionGenerationRun[]> {
|
||||
return [...this.descriptionGenerationRuns.values()]
|
||||
.sort((a, b) => b.createdAt.localeCompare(a.createdAt) || b.id.localeCompare(a.id))
|
||||
.slice(0, limit)
|
||||
.map((run) => structuredClone(run));
|
||||
}
|
||||
|
||||
async getActiveDescriptionGenerationRun(): Promise<DescriptionGenerationRun | undefined> {
|
||||
const run = [...this.descriptionGenerationRuns.values()]
|
||||
.filter((candidate) => candidate.status === "queued" || candidate.status === "running")
|
||||
.sort((a, b) => b.createdAt.localeCompare(a.createdAt))[0];
|
||||
return run ? structuredClone(run) : undefined;
|
||||
}
|
||||
|
||||
async interruptActiveDescriptionGenerationRuns(
|
||||
errorSummary: string,
|
||||
): Promise<DescriptionGenerationRun[]> {
|
||||
const interrupted: DescriptionGenerationRun[] = [];
|
||||
for (const run of this.descriptionGenerationRuns.values()) {
|
||||
if (run.status !== "queued" && run.status !== "running") continue;
|
||||
const now = new Date().toISOString();
|
||||
const updated: DescriptionGenerationRun = {
|
||||
...run,
|
||||
status: "interrupted",
|
||||
updatedAt: now,
|
||||
finishedAt: now,
|
||||
errorSummary,
|
||||
};
|
||||
this.descriptionGenerationRuns.set(run.id, updated);
|
||||
interrupted.push(structuredClone(updated));
|
||||
}
|
||||
return interrupted;
|
||||
}
|
||||
|
||||
async updateDescriptionGenerationRun(
|
||||
runId: string,
|
||||
update: DescriptionGenerationRunUpdate,
|
||||
): Promise<DescriptionGenerationRun | undefined> {
|
||||
const current = this.descriptionGenerationRuns.get(runId);
|
||||
if (!current) return undefined;
|
||||
const updated = {
|
||||
...current,
|
||||
...structuredClone(update),
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
this.descriptionGenerationRuns.set(runId, updated);
|
||||
return structuredClone(updated);
|
||||
}
|
||||
|
||||
async appendDescriptionGenerationEvent(
|
||||
runId: string,
|
||||
level: DescriptionGenerationEvent["level"],
|
||||
message: string,
|
||||
): Promise<DescriptionGenerationEvent> {
|
||||
if (!this.descriptionGenerationRuns.has(runId)) {
|
||||
throw new CatalogConflictError("Description Generation Run does not exist");
|
||||
}
|
||||
const events = this.descriptionGenerationEvents.get(runId) ?? [];
|
||||
const event: DescriptionGenerationEvent = {
|
||||
runId,
|
||||
sequence: events.length + 1,
|
||||
level,
|
||||
message,
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
events.push(event);
|
||||
this.descriptionGenerationEvents.set(runId, events);
|
||||
return structuredClone(event);
|
||||
}
|
||||
|
||||
async listDescriptionGenerationEvents(
|
||||
runId: string,
|
||||
afterSequence = 0,
|
||||
): Promise<DescriptionGenerationEvent[]> {
|
||||
return (this.descriptionGenerationEvents.get(runId) ?? [])
|
||||
.filter((event) => event.sequence > afterSequence)
|
||||
.map((event) => structuredClone(event));
|
||||
}
|
||||
|
||||
async listRelationships(databaseId: string): Promise<CatalogRelationship[]> {
|
||||
return [...this.relationships.values()].filter((relationship) => relationship.databaseId === databaseId)
|
||||
.sort((a, b) => `${a.sourceTableName}.${a.constraintName}`.localeCompare(`${b.sourceTableName}.${b.constraintName}`))
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
import {
|
||||
closeSync, constants, fstatSync, lstatSync, openSync, readFileSync,
|
||||
type Stats,
|
||||
} from "node:fs";
|
||||
import { parseAllDocuments } from "yaml";
|
||||
import { z } from "zod";
|
||||
import {
|
||||
loadSecretBundle,
|
||||
METADATA_GENERATION_SECRET_KEYS,
|
||||
} from "../config/secret-bundle.js";
|
||||
|
||||
const MAX_INSTALLATION_BYTES = 1024 * 1024;
|
||||
const RUNTIME_INSTALLATION_FILE = "/run/thothii-installation/thothii-installation.yaml";
|
||||
const modelId = z.string().regex(/^[a-z][a-z0-9._-]{0,63}$/);
|
||||
const apiKeyEnvironment = z.enum(METADATA_GENERATION_SECRET_KEYS);
|
||||
const endpointSchema = z.object({
|
||||
baseUrl: z.string().min(1).max(2048).refine((value) => {
|
||||
try {
|
||||
const url = new URL(value);
|
||||
return (url.protocol === "http:" || url.protocol === "https:")
|
||||
&& url.username === "" && url.password === "" && url.search === "" && url.hash === "";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}),
|
||||
apiVersion: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/).optional(),
|
||||
}).strict();
|
||||
const configuredModelSchema = z.object({
|
||||
id: modelId,
|
||||
label: z.string().min(1).max(128).refine((value) => value.trim() === value && !/\p{Cc}/u.test(value)),
|
||||
litellm: z.object({
|
||||
provider: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/),
|
||||
model: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/),
|
||||
disableThinking: z.literal(true).optional(),
|
||||
endpoint: endpointSchema.optional(),
|
||||
}).strict(),
|
||||
apiKeyEnv: apiKeyEnvironment.optional(),
|
||||
}).strict().superRefine((value, context) => {
|
||||
if (value.apiKeyEnv === undefined && value.litellm.endpoint === undefined) {
|
||||
context.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ["apiKeyEnv"],
|
||||
message: "keyless models require an explicit endpoint",
|
||||
});
|
||||
}
|
||||
if (value.litellm.disableThinking === true && value.litellm.endpoint === undefined) {
|
||||
context.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ["litellm", "disableThinking"],
|
||||
message: "thinking may be disabled only for an explicit endpoint",
|
||||
});
|
||||
}
|
||||
});
|
||||
const metadataGenerationSchema = z.object({
|
||||
default: modelId.optional(),
|
||||
models: z.array(configuredModelSchema).max(64).default([]),
|
||||
}).strict();
|
||||
const installationSchema = z.object({
|
||||
metadataGeneration: metadataGenerationSchema.optional(),
|
||||
}).passthrough();
|
||||
|
||||
export interface MetadataGenerationModelChoice {
|
||||
id: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
export interface MetadataGenerationModelCatalog {
|
||||
models: MetadataGenerationModelChoice[];
|
||||
default: string | null;
|
||||
}
|
||||
|
||||
export interface ResolvedMetadataGenerationModel {
|
||||
readonly id: string;
|
||||
readonly provider: string;
|
||||
readonly model: string;
|
||||
readonly disableThinking?: true;
|
||||
readonly endpoint?: Readonly<{ baseUrl: string; apiVersion?: string }>;
|
||||
readonly apiKeyEnv?: string;
|
||||
readonly apiKey?: string;
|
||||
}
|
||||
|
||||
export class MetadataGenerationModelUnavailableError extends Error {
|
||||
constructor() {
|
||||
super("metadata-generation model is unavailable");
|
||||
this.name = "MetadataGenerationModelUnavailableError";
|
||||
}
|
||||
}
|
||||
|
||||
/** The complete interface callers need: safe discovery plus fail-closed runtime resolution. */
|
||||
export interface MetadataGenerationModels {
|
||||
catalog(): MetadataGenerationModelCatalog;
|
||||
resolve(selection: string): ResolvedMetadataGenerationModel;
|
||||
}
|
||||
|
||||
class RestartLoadedMetadataGenerationModels implements MetadataGenerationModels {
|
||||
readonly #models: ReadonlyMap<string, ResolvedMetadataGenerationModel>;
|
||||
readonly #catalog: MetadataGenerationModelCatalog;
|
||||
|
||||
constructor(
|
||||
models: ReadonlyMap<string, ResolvedMetadataGenerationModel> = new Map(),
|
||||
defaultModel: string | null = null,
|
||||
choices: MetadataGenerationModelChoice[] = [],
|
||||
) {
|
||||
this.#models = models;
|
||||
this.#catalog = {
|
||||
models: choices.map((choice) => ({ ...choice })),
|
||||
default: defaultModel,
|
||||
};
|
||||
}
|
||||
|
||||
catalog(): MetadataGenerationModelCatalog {
|
||||
return {
|
||||
models: this.#catalog.models.map((choice) => ({ ...choice })),
|
||||
default: this.#catalog.default,
|
||||
};
|
||||
}
|
||||
|
||||
resolve(selection: string): ResolvedMetadataGenerationModel {
|
||||
const model = typeof selection === "string" ? this.#models.get(selection) : undefined;
|
||||
if (!model) throw new MetadataGenerationModelUnavailableError();
|
||||
return model;
|
||||
}
|
||||
}
|
||||
|
||||
function invalid(message = "metadata-generation configuration is invalid"): Error {
|
||||
return new Error(message);
|
||||
}
|
||||
|
||||
function protectedInstallationStat(file: string, info: Stats): boolean {
|
||||
const mode = info.mode & 0o777;
|
||||
if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1
|
||||
|| info.size < 1 || info.size > MAX_INSTALLATION_BYTES) return false;
|
||||
if (file === RUNTIME_INSTALLATION_FILE && info.uid === 0 && mode === 0o444) return true;
|
||||
return info.uid === (process.getuid?.() ?? info.uid) && (mode === 0o400 || mode === 0o600);
|
||||
}
|
||||
|
||||
function readProtectedInstallation(file: string): string {
|
||||
let descriptor: number | undefined;
|
||||
try {
|
||||
const before = lstatSync(file);
|
||||
if (!protectedInstallationStat(file, before)) throw new Error("unavailable");
|
||||
descriptor = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW);
|
||||
const opened = fstatSync(descriptor);
|
||||
if (!protectedInstallationStat(file, opened)
|
||||
|| before.dev !== opened.dev || before.ino !== opened.ino) throw new Error("unavailable");
|
||||
const source = readFileSync(descriptor, "utf8");
|
||||
const after = fstatSync(descriptor);
|
||||
const current = lstatSync(file);
|
||||
if (!protectedInstallationStat(file, after) || !protectedInstallationStat(file, current)
|
||||
|| opened.dev !== after.dev || opened.ino !== after.ino
|
||||
|| opened.dev !== current.dev || opened.ino !== current.ino) throw new Error("unavailable");
|
||||
return source;
|
||||
} finally {
|
||||
if (descriptor !== undefined) try { closeSync(descriptor); } catch { /* sanitized below */ }
|
||||
}
|
||||
}
|
||||
|
||||
function readInstallation(file: string): unknown {
|
||||
try {
|
||||
const documents = parseAllDocuments(readProtectedInstallation(file), { uniqueKeys: true });
|
||||
if (documents.length !== 1) throw invalid("metadata-generation installation must contain one YAML document");
|
||||
const document = documents[0];
|
||||
if (document.errors.length > 0 || document.warnings.length > 0) {
|
||||
throw invalid("metadata-generation installation contains invalid YAML");
|
||||
}
|
||||
return document.toJSON();
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message.startsWith("metadata-generation")) throw error;
|
||||
throw invalid("metadata-generation installation is unavailable");
|
||||
}
|
||||
}
|
||||
|
||||
export function loadMetadataGenerationModels(options: {
|
||||
installationFile?: string;
|
||||
secretsFile?: string;
|
||||
}): MetadataGenerationModels {
|
||||
if (!options.installationFile) return new RestartLoadedMetadataGenerationModels();
|
||||
const installation = installationSchema.safeParse(readInstallation(options.installationFile));
|
||||
if (!installation.success) throw invalid();
|
||||
const configured = installation.data.metadataGeneration;
|
||||
if (!configured || configured.models.length === 0) {
|
||||
if (configured?.default !== undefined) throw invalid("metadata-generation default does not identify a configured model");
|
||||
return new RestartLoadedMetadataGenerationModels();
|
||||
}
|
||||
if (!configured.default) throw invalid("metadata-generation default is required when models are configured");
|
||||
|
||||
const seen = new Set<string>();
|
||||
for (const model of configured.models) {
|
||||
if (seen.has(model.id)) throw invalid(`metadata-generation model id "${model.id}" is duplicated`);
|
||||
seen.add(model.id);
|
||||
}
|
||||
if (!seen.has(configured.default)) {
|
||||
throw invalid(`metadata-generation default "${configured.default}" is not configured`);
|
||||
}
|
||||
const requiresSecrets = configured.models.some((model) => model.apiKeyEnv !== undefined);
|
||||
let secrets: ReadonlyMap<string, string> = new Map();
|
||||
if (requiresSecrets) {
|
||||
if (!options.secretsFile) throw invalid("metadata-generation keyed models require THT_SECRETS_FILE");
|
||||
try {
|
||||
secrets = loadSecretBundle(options.secretsFile);
|
||||
} catch {
|
||||
throw invalid("metadata-generation secrets are unavailable");
|
||||
}
|
||||
}
|
||||
const models = new Map<string, ResolvedMetadataGenerationModel>();
|
||||
for (const configuredModel of configured.models) {
|
||||
let apiKey: string | undefined;
|
||||
if (configuredModel.apiKeyEnv !== undefined) {
|
||||
apiKey = secrets.get(configuredModel.apiKeyEnv);
|
||||
if (!apiKey) {
|
||||
throw invalid(`metadata-generation model "${configuredModel.id}" secret "${configuredModel.apiKeyEnv}" is missing`);
|
||||
}
|
||||
if (apiKey.length > 16 * 1024 || /\s/u.test(apiKey)) {
|
||||
throw invalid(`metadata-generation model "${configuredModel.id}" secret "${configuredModel.apiKeyEnv}" is unusable`);
|
||||
}
|
||||
}
|
||||
models.set(configuredModel.id, Object.freeze({
|
||||
id: configuredModel.id,
|
||||
provider: configuredModel.litellm.provider,
|
||||
model: configuredModel.litellm.model,
|
||||
...(configuredModel.litellm.disableThinking === true ? { disableThinking: true as const } : {}),
|
||||
...(configuredModel.litellm.endpoint === undefined
|
||||
? {}
|
||||
: { endpoint: Object.freeze({ ...configuredModel.litellm.endpoint }) }),
|
||||
...(configuredModel.apiKeyEnv === undefined
|
||||
? {}
|
||||
: { apiKeyEnv: configuredModel.apiKeyEnv, apiKey }),
|
||||
}));
|
||||
}
|
||||
return new RestartLoadedMetadataGenerationModels(
|
||||
models,
|
||||
configured.default,
|
||||
configured.models.map(({ id, label }) => ({ id, label })),
|
||||
);
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import * as initialMigration from "./migrations/001_workspace_databases.js";
|
||||
import * as catalogTablesMigration from "./migrations/002_catalog_tables.js";
|
||||
import * as catalogSchemaSyncMigration from "./migrations/003_catalog_schema_sync.js";
|
||||
import * as catalogRuntimeSequencePrivilegesMigration from "./migrations/004_catalog_runtime_sequence_privileges.js";
|
||||
import * as descriptionGenerationRunsMigration from "./migrations/005_description_generation_runs.js";
|
||||
|
||||
const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL;
|
||||
const host = process.env.THT_CATALOG_DB_HOST;
|
||||
@@ -36,6 +37,7 @@ const provider: MigrationProvider = {
|
||||
"002_catalog_tables": catalogTablesMigration,
|
||||
"003_catalog_schema_sync": catalogSchemaSyncMigration,
|
||||
"004_catalog_runtime_sequence_privileges": catalogRuntimeSequencePrivilegesMigration,
|
||||
"005_description_generation_runs": descriptionGenerationRunsMigration,
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
import { sql, type Kysely } from "kysely";
|
||||
import type { CatalogDatabase } from "../repository.js";
|
||||
|
||||
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.createTable("description_generation_runs")
|
||||
.addColumn("id", "uuid", (column) => column.primaryKey())
|
||||
.addColumn("database_id", "uuid", (column) => column.notNull()
|
||||
.references("workspace_databases.id").onDelete("cascade"))
|
||||
.addColumn("scope", "text", (column) => column.notNull())
|
||||
.addColumn("model_id", "text", (column) => column.notNull())
|
||||
.addColumn("language", "text", (column) => column.notNull())
|
||||
.addColumn("status", "text", (column) => column.notNull())
|
||||
.addColumn("total", "integer", (column) => column.notNull())
|
||||
.addColumn("processed", "integer", (column) => column.notNull().defaultTo(0))
|
||||
.addColumn("generated", "integer", (column) => column.notNull().defaultTo(0))
|
||||
.addColumn("non_generatable", "integer", (column) => column.notNull().defaultTo(0))
|
||||
.addColumn("failed", "integer", (column) => column.notNull().defaultTo(0))
|
||||
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("started_at", "timestamptz")
|
||||
.addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("finished_at", "timestamptz")
|
||||
.addColumn("error_summary", "text")
|
||||
.addCheckConstraint(
|
||||
"description_generation_runs_scope_check",
|
||||
sql`scope in ('selected_columns', 'selected_tables', 'all', 'missing')`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"description_generation_runs_model_id_check",
|
||||
sql`model_id ~ '^[a-z][a-z0-9._-]{0,63}$'`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"description_generation_runs_language_check",
|
||||
sql`language in ('en', 'it')`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"description_generation_runs_status_check",
|
||||
sql`status in (
|
||||
'queued', 'running', 'completed', 'completed_with_errors',
|
||||
'cancelled', 'failed', 'interrupted'
|
||||
)`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"description_generation_runs_counters_check",
|
||||
sql`total > 0
|
||||
and processed between 0 and total
|
||||
and generated >= 0
|
||||
and non_generatable >= 0
|
||||
and failed >= 0
|
||||
and generated + non_generatable + failed <= processed`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"description_generation_runs_error_summary_check",
|
||||
sql`error_summary is null or char_length(error_summary) between 1 and 2000`,
|
||||
)
|
||||
.execute();
|
||||
await db.schema.createIndex("description_generation_runs_database_created_idx")
|
||||
.on("description_generation_runs")
|
||||
.columns(["database_id", "created_at"])
|
||||
.execute();
|
||||
await sql`CREATE UNIQUE INDEX description_generation_runs_one_active
|
||||
ON description_generation_runs ((true))
|
||||
WHERE status IN ('queued', 'running')`.execute(db);
|
||||
|
||||
await db.schema.createTable("description_generation_events")
|
||||
.addColumn("run_id", "uuid", (column) => column.notNull()
|
||||
.references("description_generation_runs.id").onDelete("cascade"))
|
||||
.addColumn("sequence", "integer", (column) => column.notNull())
|
||||
.addColumn("level", "text", (column) => column.notNull())
|
||||
.addColumn("message", "text", (column) => column.notNull())
|
||||
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addPrimaryKeyConstraint("description_generation_events_pkey", ["run_id", "sequence"])
|
||||
.addCheckConstraint("description_generation_events_sequence_check", sql`sequence > 0`)
|
||||
.addCheckConstraint(
|
||||
"description_generation_events_level_check",
|
||||
sql`level in ('info', 'warning', 'error')`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"description_generation_events_message_check",
|
||||
sql`char_length(message) between 1 and 2000`,
|
||||
)
|
||||
.execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.dropTable("description_generation_events").execute();
|
||||
await db.schema.dropTable("description_generation_runs").execute();
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { z } from "zod";
|
||||
import type { ResolvedMetadataGenerationModel } from "./metadata-generation-models.js";
|
||||
|
||||
const MAX_HELPER_OUTPUT_BYTES = 64 * 1024;
|
||||
const helperOutputSchema = z.discriminatedUnion("ok", [
|
||||
z.object({ ok: z.literal(true), content: z.string() }).strict(),
|
||||
z.object({ ok: z.literal(false), error: z.literal("provider_failure") }).strict(),
|
||||
]);
|
||||
|
||||
export interface ModelCompletionMessage {
|
||||
role: "system" | "user";
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface ModelCompletionRequest {
|
||||
model: ResolvedMetadataGenerationModel;
|
||||
messages: readonly ModelCompletionMessage[];
|
||||
signal: AbortSignal;
|
||||
}
|
||||
|
||||
/** The provider boundary used by Description Generation. */
|
||||
export interface ModelCompleter {
|
||||
complete(request: ModelCompletionRequest): Promise<string>;
|
||||
}
|
||||
|
||||
export class ModelCompletionProviderError extends Error {
|
||||
constructor() {
|
||||
super("model completion failed");
|
||||
this.name = "ModelCompletionProviderError";
|
||||
}
|
||||
}
|
||||
|
||||
export class ModelCompletionCancelledError extends Error {
|
||||
constructor() {
|
||||
super("model completion cancelled");
|
||||
this.name = "ModelCompletionCancelledError";
|
||||
}
|
||||
}
|
||||
|
||||
export class PythonModelCompleter implements ModelCompleter {
|
||||
constructor(private readonly options: {
|
||||
pythonExecutable: string;
|
||||
cwd: string;
|
||||
helperModule?: string;
|
||||
timeoutMs?: number;
|
||||
terminationGraceMs?: number;
|
||||
}) {}
|
||||
|
||||
async complete(request: ModelCompletionRequest): Promise<string> {
|
||||
if (request.signal.aborted) throw new ModelCompletionCancelledError();
|
||||
const payload = {
|
||||
model: `${request.model.provider}/${request.model.model}`,
|
||||
...(request.model.apiKey === undefined ? {} : { api_key: request.model.apiKey }),
|
||||
messages: request.messages.map((message) => ({ ...message })),
|
||||
...(request.model.endpoint?.baseUrl === undefined
|
||||
? {}
|
||||
: { api_base: request.model.endpoint.baseUrl }),
|
||||
...(request.model.endpoint?.apiVersion === undefined
|
||||
? {}
|
||||
: { api_version: request.model.endpoint.apiVersion }),
|
||||
...(request.model.disableThinking === true ? { disable_thinking: true } : {}),
|
||||
};
|
||||
|
||||
return await new Promise<string>((resolve, reject) => {
|
||||
const child = spawn(
|
||||
this.options.pythonExecutable,
|
||||
["-m", this.options.helperModule ?? "tht.internal.litellm_completion"],
|
||||
{
|
||||
cwd: this.options.cwd,
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
},
|
||||
);
|
||||
let stdout = "";
|
||||
let settled = false;
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
let killFallback: ReturnType<typeof setTimeout> | undefined;
|
||||
let terminatingWith: Error | undefined;
|
||||
const cleanup = () => {
|
||||
if (timeout) clearTimeout(timeout);
|
||||
if (killFallback) clearTimeout(killFallback);
|
||||
request.signal.removeEventListener("abort", cancel);
|
||||
};
|
||||
const fail = (error: Error = new ModelCompletionProviderError()) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(error);
|
||||
};
|
||||
const terminate = (error: Error) => {
|
||||
if (settled || terminatingWith) return;
|
||||
terminatingWith = error;
|
||||
if (timeout) clearTimeout(timeout);
|
||||
try {
|
||||
child.kill("SIGTERM");
|
||||
} catch {
|
||||
fail(error);
|
||||
return;
|
||||
}
|
||||
killFallback = setTimeout(() => {
|
||||
if (settled || child.exitCode !== null || child.signalCode !== null) return;
|
||||
try {
|
||||
child.kill("SIGKILL");
|
||||
} catch {
|
||||
fail(error);
|
||||
}
|
||||
}, this.options.terminationGraceMs ?? 250);
|
||||
};
|
||||
const cancel = () => {
|
||||
terminate(new ModelCompletionCancelledError());
|
||||
};
|
||||
timeout = setTimeout(() => {
|
||||
terminate(new ModelCompletionProviderError());
|
||||
}, this.options.timeoutMs ?? 120_000);
|
||||
request.signal.addEventListener("abort", cancel, { once: true });
|
||||
if (request.signal.aborted) cancel();
|
||||
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk: string) => {
|
||||
if (terminatingWith) return;
|
||||
stdout += chunk;
|
||||
if (Buffer.byteLength(stdout, "utf8") > MAX_HELPER_OUTPUT_BYTES) {
|
||||
terminate(new ModelCompletionProviderError());
|
||||
}
|
||||
});
|
||||
// Helper and provider diagnostics are deliberately not copied into application logs.
|
||||
child.stderr.resume();
|
||||
child.once("error", () => fail(terminatingWith ?? new ModelCompletionProviderError()));
|
||||
child.once("close", (code) => {
|
||||
if (settled) return;
|
||||
if (terminatingWith) return fail(terminatingWith);
|
||||
try {
|
||||
if (code !== 0) return fail();
|
||||
const output = helperOutputSchema.parse(JSON.parse(stdout));
|
||||
if (!output.ok) return fail();
|
||||
settled = true;
|
||||
cleanup();
|
||||
resolve(output.content);
|
||||
} catch {
|
||||
fail();
|
||||
}
|
||||
});
|
||||
child.stdin.once("error", () => {
|
||||
if (terminatingWith) return;
|
||||
terminate(new ModelCompletionProviderError());
|
||||
});
|
||||
child.stdin.end(JSON.stringify(payload));
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,22 +1,34 @@
|
||||
import { CatalogOperationInProgressError } from "./types.js";
|
||||
|
||||
/** Serializes connection tests, synchronization, and metadata cleanup for each catalog database. */
|
||||
/** Serializes mutable catalog operations for each Workspace Database. */
|
||||
export class CatalogOperationCoordinator {
|
||||
private readonly active = new Set<string>();
|
||||
private readonly active = new Map<
|
||||
string,
|
||||
{ token: symbol; owner: "catalog_operation" | "description_generation" }
|
||||
>();
|
||||
|
||||
reserve(databaseId: string): () => void {
|
||||
reserve(
|
||||
databaseId: string,
|
||||
owner: "catalog_operation" | "description_generation" = "catalog_operation",
|
||||
): () => void {
|
||||
if (this.active.has(databaseId)) {
|
||||
throw new CatalogOperationInProgressError("A database operation is already in progress");
|
||||
}
|
||||
this.active.add(databaseId);
|
||||
const token = Symbol(databaseId);
|
||||
this.active.set(databaseId, { token, owner });
|
||||
let released = false;
|
||||
return () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
this.active.delete(databaseId);
|
||||
if (this.active.get(databaseId)?.token === token) this.active.delete(databaseId);
|
||||
};
|
||||
}
|
||||
|
||||
/** Administrative recovery for a reservation whose owning local operation is no longer live. */
|
||||
releaseStale(databaseId: string, owner: "description_generation"): void {
|
||||
if (this.active.get(databaseId)?.owner === owner) this.active.delete(databaseId);
|
||||
}
|
||||
|
||||
async run<T>(databaseId: string, operation: () => Promise<T>): Promise<T> {
|
||||
const release = this.reserve(databaseId);
|
||||
try {
|
||||
|
||||
@@ -180,13 +180,17 @@ export class ConcreteCatalogPostgresAccess implements CatalogPostgresAccess {
|
||||
materialized.release();
|
||||
};
|
||||
const abort = () => { void close(); };
|
||||
signal.addEventListener("abort", abort, { once: true });
|
||||
|
||||
try {
|
||||
if (signal.aborted) throw new CatalogConnectorError("PostgreSQL connector aborted");
|
||||
const passwordFile = materialized.files.get(CATALOG_SECRET_IDS.password);
|
||||
if (!passwordFile) throw new CatalogConnectorError("Database password is not configured");
|
||||
const password = await readFile(passwordFile, "utf8");
|
||||
if (signal.aborted) throw new CatalogConnectorError("PostgreSQL connector aborted");
|
||||
const tlsCaFile = materialized.files.get(CATALOG_SECRET_IDS.tlsCa);
|
||||
const tlsCa = tlsCaFile ? await readFile(tlsCaFile, "utf8") : undefined;
|
||||
if (signal.aborted) throw new CatalogConnectorError("PostgreSQL connector aborted");
|
||||
let host: string;
|
||||
let port: number;
|
||||
|
||||
@@ -243,8 +247,8 @@ export class ConcreteCatalogPostgresAccess implements CatalogPostgresAccess {
|
||||
connectionTimeoutMillis: this.connectTimeoutMs,
|
||||
...(stream ? { stream: () => stream } : {}),
|
||||
});
|
||||
signal.addEventListener("abort", abort, { once: true });
|
||||
await client.connect();
|
||||
if (signal.aborted) throw new CatalogConnectorError("PostgreSQL connector aborted");
|
||||
return {
|
||||
query: async (sql, values) => await client!.query(sql, [...values]),
|
||||
end: close,
|
||||
|
||||
@@ -15,7 +15,10 @@ import {
|
||||
CatalogConflictError,
|
||||
CatalogConnectorError,
|
||||
CatalogUnavailableError,
|
||||
DescriptionGenerationRunActiveError,
|
||||
type CatalogColumn,
|
||||
type CatalogDescriptionConsolidationCounts,
|
||||
type CatalogDescriptionTarget,
|
||||
type CatalogDatabaseMetadataDeleteTarget,
|
||||
type CatalogMetadataDeleteCounts,
|
||||
type CatalogRelationship,
|
||||
@@ -31,6 +34,10 @@ import {
|
||||
type DatabaseBinding,
|
||||
type DatabaseConfigurationInput,
|
||||
type DatabaseTestResult,
|
||||
type DescriptionGenerationEvent,
|
||||
type DescriptionGenerationRun,
|
||||
type DescriptionGenerationRunUpdate,
|
||||
type DescriptionGenerationScope,
|
||||
type ObservedCatalogTable,
|
||||
type ObservedSchemaSnapshot,
|
||||
type TableSyncRepositoryResult,
|
||||
@@ -131,6 +138,33 @@ interface CatalogRelationshipColumnTable {
|
||||
targetColumnId: string;
|
||||
}
|
||||
|
||||
interface DescriptionGenerationRunTable {
|
||||
id: string;
|
||||
databaseId: string;
|
||||
scope: DescriptionGenerationScope;
|
||||
modelId: string;
|
||||
language: DescriptionGenerationRun["language"];
|
||||
status: DescriptionGenerationRun["status"];
|
||||
total: number;
|
||||
processed: number;
|
||||
generated: number;
|
||||
nonGeneratable: number;
|
||||
failed: number;
|
||||
createdAt: Timestamp;
|
||||
startedAt: Timestamp | null;
|
||||
updatedAt: Timestamp;
|
||||
finishedAt: Timestamp | null;
|
||||
errorSummary: string | null;
|
||||
}
|
||||
|
||||
interface DescriptionGenerationEventTable {
|
||||
runId: string;
|
||||
sequence: number;
|
||||
level: DescriptionGenerationEvent["level"];
|
||||
message: string;
|
||||
createdAt: Timestamp;
|
||||
}
|
||||
|
||||
interface CatalogSyncRunTable {
|
||||
id: string;
|
||||
databaseId: string;
|
||||
@@ -176,6 +210,8 @@ export interface CatalogDatabase {
|
||||
catalogColumns: CatalogColumnTable;
|
||||
catalogRelationships: CatalogRelationshipTable;
|
||||
catalogRelationshipColumns: CatalogRelationshipColumnTable;
|
||||
descriptionGenerationRuns: DescriptionGenerationRunTable;
|
||||
descriptionGenerationEvents: DescriptionGenerationEventTable;
|
||||
catalogSyncRuns: CatalogSyncRunTable;
|
||||
catalogSyncEvents: CatalogSyncEventTable;
|
||||
}
|
||||
@@ -281,6 +317,25 @@ function serializeSyncEvent(row: Selectable<CatalogSyncEventTable>): CatalogSync
|
||||
return { ...row, id: Number(row.id), data: row.data ?? {}, createdAt: new Date(row.createdAt).toISOString() };
|
||||
}
|
||||
|
||||
function serializeDescriptionGenerationRun(
|
||||
row: Selectable<DescriptionGenerationRunTable>,
|
||||
): DescriptionGenerationRun {
|
||||
const stamp = (value: Date | string | null) => value === null ? null : new Date(value).toISOString();
|
||||
return {
|
||||
...row,
|
||||
createdAt: new Date(row.createdAt).toISOString(),
|
||||
startedAt: stamp(row.startedAt),
|
||||
updatedAt: new Date(row.updatedAt).toISOString(),
|
||||
finishedAt: stamp(row.finishedAt),
|
||||
};
|
||||
}
|
||||
|
||||
function serializeDescriptionGenerationEvent(
|
||||
row: Selectable<DescriptionGenerationEventTable>,
|
||||
): DescriptionGenerationEvent {
|
||||
return { ...row, createdAt: new Date(row.createdAt).toISOString() };
|
||||
}
|
||||
|
||||
function bindingValues(databaseId: string, binding: DatabaseBinding) {
|
||||
return {
|
||||
databaseId,
|
||||
@@ -520,6 +575,202 @@ export class KyselyCatalogRepository implements CatalogRepository {
|
||||
return row ? await this.getColumn(databaseId, tableId, row.id) : undefined;
|
||||
}
|
||||
|
||||
async consolidateGeneratedDescriptions(
|
||||
databaseId: string,
|
||||
target: CatalogDescriptionTarget,
|
||||
targetIds: readonly string[],
|
||||
): Promise<CatalogDescriptionConsolidationCounts | undefined> {
|
||||
const selectedTargetIds = [...new Set(targetIds)];
|
||||
if (selectedTargetIds.length === 0) return undefined;
|
||||
return await this.db.transaction().execute(async (trx) => {
|
||||
const database = await trx.selectFrom("workspaceDatabases").select("id")
|
||||
.where("id", "=", databaseId).forUpdate().executeTakeFirst();
|
||||
if (!database) return undefined;
|
||||
if (target === "tables") {
|
||||
const rows = await trx.selectFrom("catalogTables")
|
||||
.select(["id", "generatedDescription"])
|
||||
.where("databaseId", "=", databaseId)
|
||||
.where("id", "in", selectedTargetIds)
|
||||
.orderBy("id")
|
||||
.forUpdate()
|
||||
.execute();
|
||||
if (rows.length !== selectedTargetIds.length) return undefined;
|
||||
const copiedIds = rows
|
||||
.filter((row) => Boolean(row.generatedDescription?.trim()))
|
||||
.map((row) => row.id);
|
||||
if (copiedIds.length > 0) {
|
||||
await trx.updateTable("catalogTables").set({
|
||||
description: sql`generated_description`,
|
||||
version: sql`version + 1`,
|
||||
updatedAt: sql`now()`,
|
||||
}).where("id", "in", copiedIds).execute();
|
||||
}
|
||||
return {
|
||||
copied: copiedIds.length,
|
||||
skipped: selectedTargetIds.length - copiedIds.length,
|
||||
};
|
||||
}
|
||||
|
||||
const tableRows = await trx.selectFrom("catalogTables").select("id")
|
||||
.where("databaseId", "=", databaseId).execute();
|
||||
const rows = tableRows.length === 0 ? [] : await trx.selectFrom("catalogColumns")
|
||||
.select(["id", "generatedDescription"])
|
||||
.where("tableId", "in", tableRows.map((table) => table.id))
|
||||
.where("id", "in", selectedTargetIds)
|
||||
.orderBy("id")
|
||||
.forUpdate()
|
||||
.execute();
|
||||
if (rows.length !== selectedTargetIds.length) return undefined;
|
||||
const copiedIds = rows
|
||||
.filter((row) => Boolean(row.generatedDescription?.trim()))
|
||||
.map((row) => row.id);
|
||||
if (copiedIds.length > 0) {
|
||||
await trx.updateTable("catalogColumns").set({
|
||||
description: sql`generated_description`,
|
||||
version: sql`version + 1`,
|
||||
updatedAt: sql`now()`,
|
||||
}).where("id", "in", copiedIds).execute();
|
||||
}
|
||||
return {
|
||||
copied: copiedIds.length,
|
||||
skipped: selectedTargetIds.length - copiedIds.length,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async createDescriptionGenerationRun(
|
||||
databaseId: string,
|
||||
scope: DescriptionGenerationScope,
|
||||
modelId: string,
|
||||
language: DescriptionGenerationRun["language"],
|
||||
total: number,
|
||||
): Promise<DescriptionGenerationRun> {
|
||||
try {
|
||||
const row = await this.db.insertInto("descriptionGenerationRuns").values({
|
||||
id: randomUUID(),
|
||||
databaseId,
|
||||
scope,
|
||||
modelId,
|
||||
language,
|
||||
status: "queued",
|
||||
total,
|
||||
processed: 0,
|
||||
generated: 0,
|
||||
nonGeneratable: 0,
|
||||
failed: 0,
|
||||
startedAt: null,
|
||||
finishedAt: null,
|
||||
errorSummary: null,
|
||||
}).returningAll().executeTakeFirstOrThrow();
|
||||
return serializeDescriptionGenerationRun(row);
|
||||
} catch (error: any) {
|
||||
if (error?.code === "23505"
|
||||
&& error?.constraint === "description_generation_runs_one_active") {
|
||||
throw new DescriptionGenerationRunActiveError(
|
||||
"A description generation run is already active",
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async getDescriptionGenerationRun(
|
||||
runId: string,
|
||||
): Promise<DescriptionGenerationRun | undefined> {
|
||||
const row = await this.db.selectFrom("descriptionGenerationRuns")
|
||||
.selectAll()
|
||||
.where("id", "=", runId)
|
||||
.executeTakeFirst();
|
||||
return row ? serializeDescriptionGenerationRun(row) : undefined;
|
||||
}
|
||||
|
||||
async listDescriptionGenerationRuns(limit = 50): Promise<DescriptionGenerationRun[]> {
|
||||
const rows = await this.db.selectFrom("descriptionGenerationRuns")
|
||||
.selectAll()
|
||||
.orderBy("createdAt", "desc")
|
||||
.orderBy("id", "desc")
|
||||
.limit(limit)
|
||||
.execute();
|
||||
return rows.map(serializeDescriptionGenerationRun);
|
||||
}
|
||||
|
||||
async getActiveDescriptionGenerationRun(): Promise<DescriptionGenerationRun | undefined> {
|
||||
const row = await this.db.selectFrom("descriptionGenerationRuns")
|
||||
.selectAll()
|
||||
.where("status", "in", ["queued", "running"])
|
||||
.orderBy("createdAt", "desc")
|
||||
.executeTakeFirst();
|
||||
return row ? serializeDescriptionGenerationRun(row) : undefined;
|
||||
}
|
||||
|
||||
async interruptActiveDescriptionGenerationRuns(
|
||||
errorSummary: string,
|
||||
): Promise<DescriptionGenerationRun[]> {
|
||||
const rows = await this.db.updateTable("descriptionGenerationRuns")
|
||||
.set({
|
||||
status: "interrupted",
|
||||
finishedAt: sql`now()`,
|
||||
updatedAt: sql`now()`,
|
||||
errorSummary,
|
||||
})
|
||||
.where("status", "in", ["queued", "running"])
|
||||
.returningAll()
|
||||
.execute();
|
||||
return rows.map(serializeDescriptionGenerationRun);
|
||||
}
|
||||
|
||||
async updateDescriptionGenerationRun(
|
||||
runId: string,
|
||||
update: DescriptionGenerationRunUpdate,
|
||||
): Promise<DescriptionGenerationRun | undefined> {
|
||||
const values: any = { ...update, updatedAt: sql`now()` };
|
||||
const row = await this.db.updateTable("descriptionGenerationRuns")
|
||||
.set(values)
|
||||
.where("id", "=", runId)
|
||||
.returningAll()
|
||||
.executeTakeFirst();
|
||||
return row ? serializeDescriptionGenerationRun(row) : undefined;
|
||||
}
|
||||
|
||||
async appendDescriptionGenerationEvent(
|
||||
runId: string,
|
||||
level: DescriptionGenerationEvent["level"],
|
||||
message: string,
|
||||
): Promise<DescriptionGenerationEvent> {
|
||||
return await this.db.transaction().execute(async (trx) => {
|
||||
const run = await trx.selectFrom("descriptionGenerationRuns")
|
||||
.select("id")
|
||||
.where("id", "=", runId)
|
||||
.forUpdate()
|
||||
.executeTakeFirst();
|
||||
if (!run) throw new CatalogConflictError("Description Generation Run does not exist");
|
||||
const current = await trx.selectFrom("descriptionGenerationEvents")
|
||||
.select(sql<number>`coalesce(max(sequence), 0)::int`.as("sequence"))
|
||||
.where("runId", "=", runId)
|
||||
.executeTakeFirst();
|
||||
const row = await trx.insertInto("descriptionGenerationEvents").values({
|
||||
runId,
|
||||
sequence: Number(current?.sequence ?? 0) + 1,
|
||||
level,
|
||||
message,
|
||||
}).returningAll().executeTakeFirstOrThrow();
|
||||
return serializeDescriptionGenerationEvent(row);
|
||||
});
|
||||
}
|
||||
|
||||
async listDescriptionGenerationEvents(
|
||||
runId: string,
|
||||
afterSequence = 0,
|
||||
): Promise<DescriptionGenerationEvent[]> {
|
||||
const rows = await this.db.selectFrom("descriptionGenerationEvents")
|
||||
.selectAll()
|
||||
.where("runId", "=", runId)
|
||||
.where("sequence", ">", afterSequence)
|
||||
.orderBy("sequence")
|
||||
.execute();
|
||||
return rows.map(serializeDescriptionGenerationEvent);
|
||||
}
|
||||
|
||||
async listRelationships(databaseId: string): Promise<CatalogRelationship[]> {
|
||||
const rows = await this.db.selectFrom("catalogRelationships as relationship")
|
||||
.innerJoin("catalogTables as sourceTable", "sourceTable.id", "relationship.sourceTableId")
|
||||
@@ -1102,6 +1353,15 @@ export class UnavailableCatalogRepository implements CatalogRepository {
|
||||
async listColumns(): Promise<CatalogColumn[]> { return this.fail(); }
|
||||
async getColumn(): Promise<CatalogColumn | undefined> { return this.fail(); }
|
||||
async updateColumnMetadata(): Promise<CatalogColumn | undefined> { return this.fail(); }
|
||||
async consolidateGeneratedDescriptions(): Promise<CatalogDescriptionConsolidationCounts | undefined> { return this.fail(); }
|
||||
async createDescriptionGenerationRun(): Promise<DescriptionGenerationRun> { return this.fail(); }
|
||||
async getDescriptionGenerationRun(): Promise<DescriptionGenerationRun | undefined> { return this.fail(); }
|
||||
async listDescriptionGenerationRuns(): Promise<DescriptionGenerationRun[]> { return this.fail(); }
|
||||
async getActiveDescriptionGenerationRun(): Promise<DescriptionGenerationRun | undefined> { return this.fail(); }
|
||||
async interruptActiveDescriptionGenerationRuns(): Promise<DescriptionGenerationRun[]> { return this.fail(); }
|
||||
async updateDescriptionGenerationRun(): Promise<DescriptionGenerationRun | undefined> { return this.fail(); }
|
||||
async appendDescriptionGenerationEvent(): Promise<DescriptionGenerationEvent> { return this.fail(); }
|
||||
async listDescriptionGenerationEvents(): Promise<DescriptionGenerationEvent[]> { return this.fail(); }
|
||||
async listRelationships(): Promise<CatalogRelationship[]> { return this.fail(); }
|
||||
async deleteDatabaseMetadata(): Promise<CatalogMetadataDeleteCounts | undefined> { return this.fail(); }
|
||||
async deleteTableMetadata(): Promise<CatalogMetadataDeleteCounts | undefined> { return this.fail(); }
|
||||
|
||||
@@ -135,6 +135,7 @@ export interface CatalogRelationship {
|
||||
|
||||
export type CatalogDatabaseMetadataDeleteTarget = "tables" | "relationships";
|
||||
export type CatalogTableMetadataDeleteTarget = "columns" | "relationships";
|
||||
export type CatalogDescriptionTarget = "tables" | "columns";
|
||||
|
||||
export interface CatalogMetadataDeleteCounts {
|
||||
tables: number;
|
||||
@@ -142,6 +143,63 @@ export interface CatalogMetadataDeleteCounts {
|
||||
relationships: number;
|
||||
}
|
||||
|
||||
export interface CatalogDescriptionConsolidationCounts {
|
||||
copied: number;
|
||||
skipped: number;
|
||||
}
|
||||
|
||||
export type DescriptionGenerationScope =
|
||||
| "selected_columns"
|
||||
| "selected_tables"
|
||||
| "all"
|
||||
| "missing";
|
||||
export type DescriptionGenerationStatus =
|
||||
| "queued"
|
||||
| "running"
|
||||
| "completed"
|
||||
| "completed_with_errors"
|
||||
| "cancelled"
|
||||
| "failed"
|
||||
| "interrupted";
|
||||
|
||||
export interface DescriptionGenerationRun {
|
||||
id: string;
|
||||
databaseId: string;
|
||||
scope: DescriptionGenerationScope;
|
||||
modelId: string;
|
||||
language: "en" | "it";
|
||||
status: DescriptionGenerationStatus;
|
||||
total: number;
|
||||
processed: number;
|
||||
generated: number;
|
||||
nonGeneratable: number;
|
||||
failed: number;
|
||||
createdAt: string;
|
||||
startedAt: string | null;
|
||||
updatedAt: string;
|
||||
finishedAt: string | null;
|
||||
errorSummary: string | null;
|
||||
}
|
||||
|
||||
export interface DescriptionGenerationRunUpdate {
|
||||
status?: DescriptionGenerationStatus;
|
||||
processed?: number;
|
||||
generated?: number;
|
||||
nonGeneratable?: number;
|
||||
failed?: number;
|
||||
startedAt?: string | null;
|
||||
finishedAt?: string | null;
|
||||
errorSummary?: string | null;
|
||||
}
|
||||
|
||||
export interface DescriptionGenerationEvent {
|
||||
runId: string;
|
||||
sequence: number;
|
||||
level: "info" | "warning" | "error";
|
||||
message: string;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface ObservedRelationshipColumn {
|
||||
position: number;
|
||||
sourceColumnName: string;
|
||||
@@ -292,6 +350,37 @@ export interface CatalogRepository {
|
||||
description: string | null,
|
||||
generatedDescription: string | null,
|
||||
): Promise<CatalogColumn | undefined>;
|
||||
consolidateGeneratedDescriptions(
|
||||
databaseId: string,
|
||||
target: CatalogDescriptionTarget,
|
||||
targetIds: readonly string[],
|
||||
): Promise<CatalogDescriptionConsolidationCounts | undefined>;
|
||||
createDescriptionGenerationRun(
|
||||
databaseId: string,
|
||||
scope: DescriptionGenerationScope,
|
||||
modelId: string,
|
||||
language: DescriptionGenerationRun["language"],
|
||||
total: number,
|
||||
): Promise<DescriptionGenerationRun>;
|
||||
getDescriptionGenerationRun(runId: string): Promise<DescriptionGenerationRun | undefined>;
|
||||
listDescriptionGenerationRuns(limit?: number): Promise<DescriptionGenerationRun[]>;
|
||||
getActiveDescriptionGenerationRun(): Promise<DescriptionGenerationRun | undefined>;
|
||||
interruptActiveDescriptionGenerationRuns(
|
||||
errorSummary: string,
|
||||
): Promise<DescriptionGenerationRun[]>;
|
||||
updateDescriptionGenerationRun(
|
||||
runId: string,
|
||||
update: DescriptionGenerationRunUpdate,
|
||||
): Promise<DescriptionGenerationRun | undefined>;
|
||||
appendDescriptionGenerationEvent(
|
||||
runId: string,
|
||||
level: DescriptionGenerationEvent["level"],
|
||||
message: string,
|
||||
): Promise<DescriptionGenerationEvent>;
|
||||
listDescriptionGenerationEvents(
|
||||
runId: string,
|
||||
afterSequence?: number,
|
||||
): Promise<DescriptionGenerationEvent[]>;
|
||||
listRelationships(databaseId: string): Promise<CatalogRelationship[]>;
|
||||
deleteDatabaseMetadata(
|
||||
databaseIds: readonly string[],
|
||||
@@ -347,6 +436,7 @@ export interface CatalogRepository {
|
||||
}
|
||||
|
||||
export class CatalogConflictError extends Error {}
|
||||
export class DescriptionGenerationRunActiveError extends CatalogConflictError {}
|
||||
export class CatalogUnavailableError extends Error {}
|
||||
export class CatalogOperationInProgressError extends Error {}
|
||||
export class CatalogConnectorError extends Error {}
|
||||
|
||||
@@ -31,6 +31,7 @@ export interface AppConfig {
|
||||
ollamaEnsureTimeoutMs: number;
|
||||
piManagementTimeoutMs: number;
|
||||
secretsFile?: string;
|
||||
installationConfigFile?: string;
|
||||
piAuthFile?: string;
|
||||
secretFiles: Readonly<Record<string, string | undefined>>;
|
||||
modelApiKeyFile?: string;
|
||||
@@ -322,6 +323,13 @@ export function loadConfig(
|
||||
secretsFile.trim() !== secretsFile || secretsFile.length === 0 || secretsFile.includes("\0")
|
||||
|| !path.isAbsolute(secretsFile)
|
||||
)) throw new Error("secret bundle configuration is invalid");
|
||||
const installationConfigFile = env.THT_INSTALLATION_CONFIG_FILE;
|
||||
if (installationConfigFile !== undefined && (
|
||||
installationConfigFile.trim() !== installationConfigFile
|
||||
|| installationConfigFile.length === 0
|
||||
|| installationConfigFile.includes("\0")
|
||||
|| !path.isAbsolute(installationConfigFile)
|
||||
)) throw new Error("installation configuration is invalid");
|
||||
const piAuthFile = env.THT_PI_AUTH_FILE;
|
||||
if (piAuthFile !== undefined && (
|
||||
piAuthFile.trim() !== piAuthFile || piAuthFile.length === 0 || piAuthFile.includes("\0")
|
||||
@@ -404,6 +412,7 @@ export function loadConfig(
|
||||
ollamaEnsureTimeoutMs: Number(env.OLLAMA_ENSURE_TIMEOUT_MS ?? 60000),
|
||||
piManagementTimeoutMs: piManagementTimeout(env.PI_MANAGEMENT_TIMEOUT_MS),
|
||||
secretsFile,
|
||||
installationConfigFile,
|
||||
piAuthFile,
|
||||
secretFiles,
|
||||
modelApiKeyFile,
|
||||
|
||||
@@ -8,12 +8,19 @@ import {
|
||||
isUsableAuthenticationSecret,
|
||||
} from "../auth/secret-policy.js";
|
||||
|
||||
/** Credential names that metadata-generation model entries may reference. */
|
||||
export const METADATA_GENERATION_SECRET_KEYS = Object.freeze([
|
||||
"THT_METADATA_API_KEY", "ANTHROPIC_API_KEY", "AZURE_API_KEY", "GEMINI_API_KEY",
|
||||
"DEEPSEEK_API_KEY", "OPENAI_API_KEY", "OPENROUTER_API_KEY", "ZAI_API_KEY",
|
||||
] as const);
|
||||
|
||||
/** Keys accepted by the deployment bundle. Keep this list intentionally explicit. */
|
||||
export const SECRET_BUNDLE_KEYS = Object.freeze([
|
||||
"THT_MODEL_API_KEY", "THT_DWH_API_KEY", "THT_VEC_API_KEY", "THT_VEC_WRITE_API_KEY",
|
||||
"THT_CA", "THT_SSL_CA", "THT_VECTOR_BOOTSTRAP_PASSWORD", "THT_VECTOR_MIGRATOR_PASSWORD",
|
||||
"THT_VECTOR_READER_PASSWORD", "THT_VECTOR_WRITER_PASSWORD", "PI_PROVIDER_API_KEY",
|
||||
"THT_OIDC_CLIENT_SECRET", "THT_AUTHENTIK_API_TOKEN",
|
||||
...METADATA_GENERATION_SECRET_KEYS,
|
||||
] as const);
|
||||
|
||||
const ALLOWED = new Set<string>(SECRET_BUNDLE_KEYS);
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
||||
import { z } from "zod";
|
||||
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
|
||||
import type { CatalogOperationCoordinator } from "../catalog/operation-coordinator.js";
|
||||
import {
|
||||
CatalogOperationInProgressError,
|
||||
CatalogUnavailableError,
|
||||
type CatalogRepository,
|
||||
} from "../catalog/types.js";
|
||||
|
||||
const idSchema = z.uuid();
|
||||
const consolidationSchema = z.object({
|
||||
target: z.enum(["tables", "columns"]),
|
||||
targetIds: z.array(idSchema).min(1).max(10_000),
|
||||
}).strict();
|
||||
|
||||
function manage(request: FastifyRequest, reply: FastifyReply) {
|
||||
return isPrincipalContext(requirePermission(request, reply, "database.manage"));
|
||||
}
|
||||
|
||||
function safeError(reply: FastifyReply, error: unknown) {
|
||||
if (error instanceof CatalogUnavailableError) {
|
||||
return reply.code(503).send({ code: "catalog_unavailable", message: "Database catalog is unavailable." });
|
||||
}
|
||||
if (error instanceof CatalogOperationInProgressError) {
|
||||
return reply.code(409).send({
|
||||
code: "database_operation_in_progress",
|
||||
message: "A database operation is already in progress.",
|
||||
});
|
||||
}
|
||||
if (error instanceof z.ZodError) {
|
||||
return reply.code(400).send({
|
||||
code: "description_consolidation_invalid",
|
||||
message: "Description consolidation request is invalid.",
|
||||
});
|
||||
}
|
||||
return reply.code(500).send({
|
||||
code: "description_consolidation_failed",
|
||||
message: "Description consolidation failed.",
|
||||
});
|
||||
}
|
||||
|
||||
export function catalogDescriptionConsolidationRoutes(
|
||||
app: FastifyInstance,
|
||||
deps: { repository: CatalogRepository; operations: CatalogOperationCoordinator },
|
||||
): void {
|
||||
app.post("/catalog/databases/:databaseId/descriptions/consolidate", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
|
||||
const input = consolidationSchema.parse(request.body);
|
||||
const targetIds = [...new Set(input.targetIds)];
|
||||
const result = await deps.operations.run(
|
||||
databaseId,
|
||||
async () => await deps.repository.consolidateGeneratedDescriptions(
|
||||
databaseId,
|
||||
input.target,
|
||||
targetIds,
|
||||
),
|
||||
);
|
||||
if (!result) {
|
||||
return reply.code(404).send({
|
||||
code: "catalog_target_not_found",
|
||||
message: "The database or one or more selected catalog targets were not found.",
|
||||
});
|
||||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
return safeError(reply, error);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
||||
import { z } from "zod";
|
||||
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
|
||||
import {
|
||||
DescriptionGenerationDuplicateTargetIdsError,
|
||||
DescriptionGenerationNoEligibleTargetsError,
|
||||
DescriptionGenerationRunLiveError,
|
||||
DescriptionGenerationTargetIdsRequiredError,
|
||||
DescriptionGenerationTargetNotFoundError,
|
||||
DescriptionGenerationWorkspaceUnavailableError,
|
||||
type DescriptionGenerationWorker,
|
||||
} from "../catalog/description-generation-worker.js";
|
||||
import { MetadataGenerationModelUnavailableError } from "../catalog/metadata-generation-models.js";
|
||||
import {
|
||||
CatalogOperationInProgressError,
|
||||
CatalogUnavailableError,
|
||||
DescriptionGenerationRunActiveError,
|
||||
type CatalogRepository,
|
||||
type DescriptionGenerationEvent,
|
||||
type DescriptionGenerationRun,
|
||||
} from "../catalog/types.js";
|
||||
|
||||
const idSchema = z.uuid();
|
||||
const modelIdSchema = z.string().regex(/^[a-z][a-z0-9._-]{0,63}$/);
|
||||
const selectedTargetIdsSchema = z.array(idSchema).min(1);
|
||||
const startSchema = z.discriminatedUnion("scope", [
|
||||
z.object({
|
||||
modelId: modelIdSchema,
|
||||
scope: z.literal("selected_columns"),
|
||||
targetIds: selectedTargetIdsSchema,
|
||||
}).strict(),
|
||||
z.object({
|
||||
modelId: modelIdSchema,
|
||||
scope: z.literal("selected_tables"),
|
||||
targetIds: selectedTargetIdsSchema,
|
||||
}).strict(),
|
||||
z.object({ modelId: modelIdSchema, scope: z.literal("all") }).strict(),
|
||||
z.object({ modelId: modelIdSchema, scope: z.literal("missing") }).strict(),
|
||||
]);
|
||||
const eventQuerySchema = z.object({
|
||||
after: z.coerce.number().int().nonnegative().default(0),
|
||||
}).strict();
|
||||
const historyQuerySchema = z.object({
|
||||
limit: z.coerce.number().int().min(1).max(100).default(50),
|
||||
}).strict();
|
||||
const terminalStatuses = new Set<DescriptionGenerationRun["status"]>([
|
||||
"completed",
|
||||
"completed_with_errors",
|
||||
"cancelled",
|
||||
"failed",
|
||||
"interrupted",
|
||||
]);
|
||||
|
||||
function manage(request: FastifyRequest, reply: FastifyReply) {
|
||||
return isPrincipalContext(requirePermission(request, reply, "database.manage"));
|
||||
}
|
||||
|
||||
function publicEvent(event: DescriptionGenerationEvent) {
|
||||
return {
|
||||
sequence: event.sequence,
|
||||
level: event.level,
|
||||
message: event.message,
|
||||
createdAt: event.createdAt,
|
||||
};
|
||||
}
|
||||
|
||||
function publicRun(run: DescriptionGenerationRun) {
|
||||
return {
|
||||
id: run.id,
|
||||
databaseId: run.databaseId,
|
||||
scope: run.scope,
|
||||
modelId: run.modelId,
|
||||
language: run.language,
|
||||
status: run.status,
|
||||
total: run.total,
|
||||
processed: run.processed,
|
||||
generated: run.generated,
|
||||
nonGeneratable: run.nonGeneratable,
|
||||
failed: run.failed,
|
||||
createdAt: run.createdAt,
|
||||
startedAt: run.startedAt,
|
||||
updatedAt: run.updatedAt,
|
||||
finishedAt: run.finishedAt,
|
||||
errorSummary: run.errorSummary,
|
||||
};
|
||||
}
|
||||
|
||||
function safeError(reply: FastifyReply, error: unknown) {
|
||||
if (error instanceof CatalogUnavailableError) {
|
||||
return reply.code(503).send({
|
||||
code: "catalog_unavailable",
|
||||
message: "Database catalog is unavailable.",
|
||||
});
|
||||
}
|
||||
if (error instanceof DescriptionGenerationRunActiveError) {
|
||||
return reply.code(409).send({
|
||||
code: "description_generation_run_active",
|
||||
message: "A Description Generation Run is already active.",
|
||||
});
|
||||
}
|
||||
if (error instanceof DescriptionGenerationRunLiveError) {
|
||||
return reply.code(409).send({
|
||||
code: "description_generation_run_live",
|
||||
message: "A local Description Generation worker or helper is still running.",
|
||||
});
|
||||
}
|
||||
if (error instanceof CatalogOperationInProgressError) {
|
||||
return reply.code(409).send({
|
||||
code: "database_operation_in_progress",
|
||||
message: "A database operation is already in progress.",
|
||||
});
|
||||
}
|
||||
if (error instanceof MetadataGenerationModelUnavailableError) {
|
||||
return reply.code(409).send({
|
||||
code: "metadata_generation_model_unavailable",
|
||||
message: "The selected metadata-generation model is unavailable.",
|
||||
});
|
||||
}
|
||||
if (error instanceof DescriptionGenerationDuplicateTargetIdsError) {
|
||||
return reply.code(400).send({
|
||||
code: "description_generation_target_ids_duplicate",
|
||||
message: "Description generation target IDs must be unique.",
|
||||
});
|
||||
}
|
||||
if (error instanceof DescriptionGenerationTargetIdsRequiredError) {
|
||||
return reply.code(400).send({
|
||||
code: "description_generation_request_invalid",
|
||||
message: "At least one description generation target ID is required.",
|
||||
});
|
||||
}
|
||||
if (error instanceof DescriptionGenerationNoEligibleTargetsError) {
|
||||
return reply.code(409).send({
|
||||
code: "description_generation_no_eligible_targets",
|
||||
message: error.scope === "all"
|
||||
? "No Catalog Tables or Catalog Columns are available for description generation."
|
||||
: "No Catalog Tables or Catalog Columns have a missing Generated Description.",
|
||||
});
|
||||
}
|
||||
if (error instanceof DescriptionGenerationTargetNotFoundError) {
|
||||
const code = error.target === "database"
|
||||
? "database_not_found"
|
||||
: error.target === "table"
|
||||
? "catalog_table_not_found"
|
||||
: "catalog_column_not_found";
|
||||
const message = error.target === "database"
|
||||
? "Database configuration was not found."
|
||||
: error.target === "table"
|
||||
? "One or more selected Catalog Tables were not found."
|
||||
: "One or more selected Catalog Columns were not found.";
|
||||
return reply.code(404).send({
|
||||
code,
|
||||
message,
|
||||
});
|
||||
}
|
||||
if (error instanceof DescriptionGenerationWorkspaceUnavailableError) {
|
||||
return reply.code(409).send({
|
||||
code: "workspace_configuration_unavailable",
|
||||
message: "The database workspace configuration is unavailable.",
|
||||
});
|
||||
}
|
||||
if (error instanceof z.ZodError) {
|
||||
return reply.code(400).send({
|
||||
code: "description_generation_request_invalid",
|
||||
message: "Description generation request is invalid.",
|
||||
});
|
||||
}
|
||||
return reply.code(500).send({
|
||||
code: "description_generation_failed",
|
||||
message: "Description generation failed.",
|
||||
});
|
||||
}
|
||||
|
||||
export function catalogDescriptionGenerationRoutes(
|
||||
app: FastifyInstance,
|
||||
deps: { repository: CatalogRepository; worker: DescriptionGenerationWorker },
|
||||
): void {
|
||||
app.post("/catalog/databases/:databaseId/description-generation-runs", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
|
||||
const input = startSchema.parse(request.body);
|
||||
const targetIds = "targetIds" in input ? input.targetIds : [];
|
||||
const run = await deps.worker.start(databaseId, input.modelId, input.scope, targetIds);
|
||||
return reply.code(202).send(publicRun(run));
|
||||
} catch (error) {
|
||||
return safeError(reply, error);
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/catalog/description-generation-runs", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const { limit } = historyQuerySchema.parse(request.query);
|
||||
return (await deps.repository.listDescriptionGenerationRuns(limit)).map(publicRun);
|
||||
} catch (error) {
|
||||
return safeError(reply, error);
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/catalog/description-generation-runs/:runId", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
|
||||
const run = await deps.repository.getDescriptionGenerationRun(runId);
|
||||
return run ? publicRun(run) : reply.code(404).send({
|
||||
code: "description_generation_run_not_found",
|
||||
message: "Description Generation Run was not found.",
|
||||
});
|
||||
} catch (error) {
|
||||
return safeError(reply, error);
|
||||
}
|
||||
});
|
||||
|
||||
app.post("/catalog/description-generation-runs/:runId/cancel", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
|
||||
const run = await deps.worker.cancel(runId);
|
||||
return run ? publicRun(run) : reply.code(404).send({
|
||||
code: "description_generation_run_not_found",
|
||||
message: "Description Generation Run was not found.",
|
||||
});
|
||||
} catch (error) {
|
||||
return safeError(reply, error);
|
||||
}
|
||||
});
|
||||
|
||||
app.post("/catalog/description-generation-runs/unlock", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const run = await deps.worker.unlock();
|
||||
return run ? publicRun(run) : reply.code(404).send({
|
||||
code: "description_generation_run_not_found",
|
||||
message: "No stale active Description Generation Run was found.",
|
||||
});
|
||||
} catch (error) {
|
||||
return safeError(reply, error);
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/catalog/description-generation-runs/:runId/events", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
let unsubscribe: (() => void) | undefined;
|
||||
let hijacked = false;
|
||||
try {
|
||||
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
|
||||
let after = eventQuerySchema.parse(request.query).after;
|
||||
const headerCursor = Number(request.headers["last-event-id"]);
|
||||
if (Number.isInteger(headerCursor) && headerCursor >= 0) after = Math.max(after, headerCursor);
|
||||
if (!(await deps.repository.getDescriptionGenerationRun(runId))) {
|
||||
return reply.code(404).send({
|
||||
code: "description_generation_run_not_found",
|
||||
message: "Description Generation Run was not found.",
|
||||
});
|
||||
}
|
||||
|
||||
const buffered: DescriptionGenerationEvent[] = [];
|
||||
let ready = false;
|
||||
let closed = false;
|
||||
let lastRunSnapshot = "";
|
||||
let delivery = Promise.resolve();
|
||||
const close = () => {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
unsubscribe?.();
|
||||
if (!reply.raw.destroyed) reply.raw.end();
|
||||
};
|
||||
const writeRun = (run: DescriptionGenerationRun) => {
|
||||
if (closed) return;
|
||||
const snapshot = JSON.stringify(publicRun(run));
|
||||
if (snapshot === lastRunSnapshot) return;
|
||||
lastRunSnapshot = snapshot;
|
||||
reply.raw.write(`event: run\ndata: ${snapshot}\n\n`);
|
||||
if (terminalStatuses.has(run.status)) close();
|
||||
};
|
||||
const enqueue = (event: DescriptionGenerationEvent) => {
|
||||
delivery = delivery.then(async () => {
|
||||
if (closed || event.sequence <= after) return;
|
||||
after = event.sequence;
|
||||
reply.raw.write(
|
||||
`id: ${event.sequence}\nevent: log\ndata: ${JSON.stringify(publicEvent(event))}\n\n`,
|
||||
);
|
||||
const run = await deps.repository.getDescriptionGenerationRun(runId);
|
||||
if (run) writeRun(run);
|
||||
}).catch(close);
|
||||
};
|
||||
unsubscribe = deps.worker.subscribeEvents(runId, (event) => {
|
||||
if (ready) enqueue(event);
|
||||
else buffered.push(event);
|
||||
});
|
||||
const persisted = await deps.repository.listDescriptionGenerationEvents(runId, after);
|
||||
|
||||
reply.hijack();
|
||||
hijacked = true;
|
||||
reply.raw.writeHead(200, {
|
||||
"content-type": "text/event-stream; charset=utf-8",
|
||||
"cache-control": "no-cache, no-transform",
|
||||
connection: "keep-alive",
|
||||
"x-accel-buffering": "no",
|
||||
});
|
||||
reply.raw.once("close", close);
|
||||
request.raw.once("aborted", close);
|
||||
for (const event of persisted) {
|
||||
if (event.sequence <= after) continue;
|
||||
after = event.sequence;
|
||||
reply.raw.write(
|
||||
`id: ${event.sequence}\nevent: log\ndata: ${JSON.stringify(publicEvent(event))}\n\n`,
|
||||
);
|
||||
}
|
||||
ready = true;
|
||||
buffered.sort((a, b) => a.sequence - b.sequence).forEach(enqueue);
|
||||
let pending = delivery;
|
||||
await pending;
|
||||
while (pending !== delivery) {
|
||||
pending = delivery;
|
||||
await pending;
|
||||
}
|
||||
const current = await deps.repository.getDescriptionGenerationRun(runId);
|
||||
if (current) writeRun(current);
|
||||
return reply;
|
||||
} catch (error) {
|
||||
unsubscribe?.();
|
||||
if (hijacked) {
|
||||
if (!reply.raw.destroyed) reply.raw.end();
|
||||
return reply;
|
||||
}
|
||||
return safeError(reply, error);
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/catalog/description-generation-runs/:runId/events-list", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
|
||||
const { after } = eventQuerySchema.parse(request.query);
|
||||
if (!(await deps.repository.getDescriptionGenerationRun(runId))) {
|
||||
return reply.code(404).send({
|
||||
code: "description_generation_run_not_found",
|
||||
message: "Description Generation Run was not found.",
|
||||
});
|
||||
}
|
||||
return (await deps.repository.listDescriptionGenerationEvents(runId, after)).map(publicEvent);
|
||||
} catch (error) {
|
||||
return safeError(reply, error);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -43,7 +43,13 @@ function safeError(reply: FastifyReply, error: unknown) {
|
||||
if (error instanceof CatalogUnavailableError) {
|
||||
return reply.code(503).send({ code: "catalog_unavailable", message: "Database catalog is unavailable." });
|
||||
}
|
||||
if (error instanceof CatalogConflictError || error instanceof CatalogOperationInProgressError) {
|
||||
if (error instanceof CatalogOperationInProgressError) {
|
||||
return reply.code(409).send({
|
||||
code: "database_operation_in_progress",
|
||||
message: "A database operation is already in progress.",
|
||||
});
|
||||
}
|
||||
if (error instanceof CatalogConflictError) {
|
||||
return reply.code(409).send({ code: "schema_sync_conflict", message: error.message });
|
||||
}
|
||||
if (error instanceof CatalogConnectorError) {
|
||||
@@ -98,21 +104,24 @@ export function catalogSchemaRoutes(
|
||||
const tableId = idSchema.parse(params.tableId);
|
||||
const columnId = idSchema.parse(params.columnId);
|
||||
const input = metadataSchema.parse(request.body);
|
||||
const current = await deps.repository.getColumn(databaseId, tableId, columnId);
|
||||
if (!current) return reply.code(404).send({ code: "column_not_found", message: "Catalog column was not found." });
|
||||
if (current.version !== input.version) {
|
||||
return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
|
||||
}
|
||||
const updated = await deps.repository.updateColumnMetadata(
|
||||
databaseId,
|
||||
tableId,
|
||||
columnId,
|
||||
input.version,
|
||||
normalized(input.description),
|
||||
normalized(input.generatedDescription),
|
||||
);
|
||||
if (!updated) return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
|
||||
return updated;
|
||||
const operation = async () => {
|
||||
const current = await deps.repository.getColumn(databaseId, tableId, columnId);
|
||||
if (!current) return reply.code(404).send({ code: "column_not_found", message: "Catalog column was not found." });
|
||||
if (current.version !== input.version) {
|
||||
return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
|
||||
}
|
||||
const updated = await deps.repository.updateColumnMetadata(
|
||||
databaseId,
|
||||
tableId,
|
||||
columnId,
|
||||
input.version,
|
||||
normalized(input.description),
|
||||
normalized(input.generatedDescription),
|
||||
);
|
||||
if (!updated) return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
|
||||
return updated;
|
||||
};
|
||||
return deps.operations ? await deps.operations.run(databaseId, operation) : await operation();
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
||||
import { z } from "zod";
|
||||
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
|
||||
import type { CatalogTableService } from "../catalog/table-service.js";
|
||||
import type { CatalogOperationCoordinator } from "../catalog/operation-coordinator.js";
|
||||
import {
|
||||
CatalogConnectorError,
|
||||
CatalogOperationInProgressError,
|
||||
@@ -38,7 +39,11 @@ function safeError(reply: FastifyReply, error: unknown) {
|
||||
|
||||
export function catalogTableRoutes(
|
||||
app: FastifyInstance,
|
||||
deps: { repository: CatalogRepository; service: CatalogTableService },
|
||||
deps: {
|
||||
repository: CatalogRepository;
|
||||
service: CatalogTableService;
|
||||
operations: CatalogOperationCoordinator;
|
||||
},
|
||||
): void {
|
||||
app.get("/catalog/databases/:databaseId/tables", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
@@ -59,20 +64,22 @@ export function catalogTableRoutes(
|
||||
const tableId = idSchema.parse(params.tableId);
|
||||
const input = updateSchema.parse(request.body);
|
||||
const { version, description } = input;
|
||||
const current = await deps.repository.getTable(databaseId, tableId);
|
||||
if (!current) return reply.code(404).send({ code: "table_not_found", message: "Catalog table was not found." });
|
||||
if (current.version !== version) {
|
||||
return reply.code(409).send({ code: "table_stale", message: "Table description changed. Reload and try again." });
|
||||
}
|
||||
const updated = await deps.service.updateMetadata(
|
||||
databaseId,
|
||||
tableId,
|
||||
version,
|
||||
description,
|
||||
input.generatedDescription === undefined ? current.generatedDescription : input.generatedDescription,
|
||||
);
|
||||
if (!updated) return reply.code(409).send({ code: "table_stale", message: "Table description changed. Reload and try again." });
|
||||
return updated;
|
||||
return await deps.operations.run(databaseId, async () => {
|
||||
const current = await deps.repository.getTable(databaseId, tableId);
|
||||
if (!current) return reply.code(404).send({ code: "table_not_found", message: "Catalog table was not found." });
|
||||
if (current.version !== version) {
|
||||
return reply.code(409).send({ code: "table_stale", message: "Table description changed. Reload and try again." });
|
||||
}
|
||||
const updated = await deps.service.updateMetadata(
|
||||
databaseId,
|
||||
tableId,
|
||||
version,
|
||||
description,
|
||||
input.generatedDescription === undefined ? current.generatedDescription : input.generatedDescription,
|
||||
);
|
||||
if (!updated) return reply.code(409).send({ code: "table_stale", message: "Table description changed. Reload and try again." });
|
||||
return updated;
|
||||
});
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
|
||||
import type { MetadataGenerationModels } from "../catalog/metadata-generation-models.js";
|
||||
|
||||
export function metadataGenerationModelRoutes(
|
||||
app: FastifyInstance,
|
||||
models: MetadataGenerationModels,
|
||||
): void {
|
||||
app.get("/catalog/metadata-generation/models", async (request, reply) => {
|
||||
if (!isPrincipalContext(requirePermission(request, reply, "database.manage"))) return reply;
|
||||
return models.catalog();
|
||||
});
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,456 @@
|
||||
import { expect, test, vi } from "vitest";
|
||||
import { DescriptionGenerationWorker } from "../src/catalog/description-generation-worker.js";
|
||||
import type { DescriptionSourceSampler } from "../src/catalog/description-source-sampler.js";
|
||||
import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
|
||||
import type { MetadataGenerationModels } from "../src/catalog/metadata-generation-models.js";
|
||||
import { ModelCompletionCancelledError } from "../src/catalog/model-completer.js";
|
||||
import type { ModelCompleter, ModelCompletionRequest } from "../src/catalog/model-completer.js";
|
||||
import { CatalogOperationCoordinator } from "../src/catalog/operation-coordinator.js";
|
||||
import type { WorkspaceRegistry } from "../src/workspaces/registry.js";
|
||||
|
||||
test("serializes Unlock with Start so stale recovery cannot release a new reservation", async () => {
|
||||
let lookupStarted!: () => void;
|
||||
const started = new Promise<void>((resolve) => { lookupStarted = resolve; });
|
||||
let releaseLookup!: () => void;
|
||||
const gate = new Promise<void>((resolve) => { releaseLookup = resolve; });
|
||||
const repository = {
|
||||
getActiveDescriptionGenerationRun: vi.fn(async () => {
|
||||
lookupStarted();
|
||||
await gate;
|
||||
return undefined;
|
||||
}),
|
||||
} as unknown as MemoryCatalogRepository;
|
||||
const resolveModel = vi.fn();
|
||||
const worker = new DescriptionGenerationWorker(
|
||||
repository,
|
||||
{} as WorkspaceRegistry,
|
||||
{
|
||||
catalog: () => ({ models: [], default: "" }),
|
||||
resolve: resolveModel,
|
||||
} as MetadataGenerationModels,
|
||||
{} as ModelCompleter,
|
||||
new CatalogOperationCoordinator(),
|
||||
{ sample: vi.fn(async () => []) },
|
||||
);
|
||||
|
||||
const unlocking = worker.unlock();
|
||||
await started;
|
||||
await expect(worker.start(
|
||||
"11111111-1111-4111-8111-111111111111",
|
||||
"openai-mini",
|
||||
"missing",
|
||||
[],
|
||||
)).rejects.toThrow("already active");
|
||||
expect(resolveModel).not.toHaveBeenCalled();
|
||||
|
||||
releaseLookup();
|
||||
await expect(unlocking).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
test("exposes an awaitable background job and absorbs provider promise rejection", async () => {
|
||||
const repository = new MemoryCatalogRepository();
|
||||
const database = await repository.create({
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "datawarehouse",
|
||||
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
||||
});
|
||||
await repository.applySchemaSync(database.id, database.version, "all", [], {
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||
tables: [{ name: "patients", sourceComment: null }],
|
||||
columns: [{
|
||||
tableName: "patients",
|
||||
name: "birth_date",
|
||||
ordinalPosition: 1,
|
||||
dataType: "date",
|
||||
isNullable: true,
|
||||
defaultExpression: null,
|
||||
primaryKeyPosition: null,
|
||||
sourceComment: null,
|
||||
}],
|
||||
relationships: [],
|
||||
});
|
||||
const table = (await repository.listTables(database.id))[0]!;
|
||||
const column = (await repository.listColumns(database.id, table.id))[0]!;
|
||||
let rejectCompletion!: (error: Error) => void;
|
||||
const pendingCompletion = new Promise<string>((_resolve, reject) => { rejectCompletion = reject; });
|
||||
const completer: ModelCompleter = {
|
||||
complete: vi.fn(async () => await pendingCompletion),
|
||||
};
|
||||
const models: MetadataGenerationModels = {
|
||||
catalog: () => ({ models: [{ id: "openai-mini", label: "OpenAI Mini" }], default: "openai-mini" }),
|
||||
resolve: () => ({
|
||||
id: "openai-mini",
|
||||
provider: "openai",
|
||||
model: "gpt-4.1-mini",
|
||||
apiKeyEnv: "OPENAI_API_KEY",
|
||||
apiKey: "test-provider-secret",
|
||||
}),
|
||||
};
|
||||
const operations = new CatalogOperationCoordinator();
|
||||
const sourceSampler: DescriptionSourceSampler = {
|
||||
sample: vi.fn(async () => []),
|
||||
};
|
||||
const worker = new DescriptionGenerationWorker(
|
||||
repository,
|
||||
{
|
||||
read: vi.fn(async () => ({
|
||||
workspace: { workspace: { language: "it" } },
|
||||
revision: {},
|
||||
})),
|
||||
} as unknown as WorkspaceRegistry,
|
||||
models,
|
||||
completer,
|
||||
operations,
|
||||
sourceSampler,
|
||||
);
|
||||
|
||||
const run = await worker.start(database.id, "openai-mini", "selected_columns", [column.id]);
|
||||
let settled = false;
|
||||
const waiting = worker.waitForRun(run.id).then(() => { settled = true; });
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(settled).toBe(false);
|
||||
|
||||
rejectCompletion(new Error("test-provider-secret private prompt raw response"));
|
||||
await expect(waiting).resolves.toBeUndefined();
|
||||
expect(await repository.getDescriptionGenerationRun(run.id)).toMatchObject({
|
||||
status: "completed_with_errors",
|
||||
failed: 1,
|
||||
errorSummary: "Description generation completed with errors.",
|
||||
});
|
||||
const events = await repository.listDescriptionGenerationEvents(run.id);
|
||||
expect(JSON.stringify(events)).not.toMatch(/test-provider-secret|private prompt|raw response/);
|
||||
|
||||
const release = operations.reserve(database.id);
|
||||
release();
|
||||
|
||||
await expect(worker.start(
|
||||
database.id,
|
||||
"openai-mini",
|
||||
"selected_columns",
|
||||
[],
|
||||
)).rejects.toThrow("at least one target ID is required");
|
||||
});
|
||||
|
||||
test("marks an active run interrupted when the backend worker stops", async () => {
|
||||
const repository = new MemoryCatalogRepository();
|
||||
const database = await repository.create({
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "datawarehouse",
|
||||
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
||||
});
|
||||
await repository.applySchemaSync(database.id, database.version, "all", [], {
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||
tables: [{ name: "patients", sourceComment: null }],
|
||||
columns: [{
|
||||
tableName: "patients",
|
||||
name: "status",
|
||||
ordinalPosition: 1,
|
||||
dataType: "text",
|
||||
isNullable: true,
|
||||
defaultExpression: null,
|
||||
primaryKeyPosition: null,
|
||||
sourceComment: null,
|
||||
}],
|
||||
relationships: [],
|
||||
});
|
||||
const table = (await repository.listTables(database.id))[0]!;
|
||||
const column = (await repository.listColumns(database.id, table.id))[0]!;
|
||||
const completer: ModelCompleter = {
|
||||
complete: vi.fn(async (request) => await new Promise<string>((_resolve, reject) => {
|
||||
const cancel = () => reject(new ModelCompletionCancelledError());
|
||||
if (request.signal.aborted) cancel();
|
||||
else request.signal.addEventListener("abort", cancel, { once: true });
|
||||
})),
|
||||
};
|
||||
const worker = new DescriptionGenerationWorker(
|
||||
repository,
|
||||
{
|
||||
read: vi.fn(async () => ({
|
||||
workspace: { workspace: { language: "it" } },
|
||||
revision: {},
|
||||
})),
|
||||
} as unknown as WorkspaceRegistry,
|
||||
{
|
||||
catalog: () => ({ models: [{ id: "openai-mini", label: "OpenAI Mini" }], default: "openai-mini" }),
|
||||
resolve: () => ({
|
||||
id: "openai-mini",
|
||||
provider: "openai",
|
||||
model: "gpt-4.1-mini",
|
||||
apiKeyEnv: "OPENAI_API_KEY",
|
||||
apiKey: "test-provider-secret",
|
||||
}),
|
||||
},
|
||||
completer,
|
||||
new CatalogOperationCoordinator(),
|
||||
{ sample: vi.fn(async () => []) },
|
||||
);
|
||||
|
||||
const run = await worker.start(database.id, "openai-mini", "selected_columns", [column.id]);
|
||||
await vi.waitFor(() => expect(completer.complete).toHaveBeenCalledOnce());
|
||||
await worker.stop();
|
||||
|
||||
expect(await repository.getDescriptionGenerationRun(run.id)).toMatchObject({
|
||||
status: "interrupted",
|
||||
errorSummary: "Description generation was interrupted by backend shutdown.",
|
||||
});
|
||||
expect(await repository.listDescriptionGenerationEvents(run.id)).toContainEqual(
|
||||
expect.objectContaining({
|
||||
level: "warning",
|
||||
message: "Description generation was interrupted by backend shutdown.",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
test("adds only bounded transient source samples to the model request", async () => {
|
||||
const repository = new MemoryCatalogRepository();
|
||||
const database = await repository.create({
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "datawarehouse",
|
||||
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
||||
});
|
||||
await repository.applySchemaSync(database.id, database.version, "all", [], {
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||
tables: [{ name: "patients", sourceComment: null }],
|
||||
columns: [{
|
||||
tableName: "patients",
|
||||
name: "status",
|
||||
ordinalPosition: 1,
|
||||
dataType: "text",
|
||||
isNullable: true,
|
||||
defaultExpression: null,
|
||||
primaryKeyPosition: null,
|
||||
sourceComment: null,
|
||||
}, {
|
||||
tableName: "patients",
|
||||
name: "ward",
|
||||
ordinalPosition: 2,
|
||||
dataType: "text",
|
||||
isNullable: true,
|
||||
defaultExpression: null,
|
||||
primaryKeyPosition: null,
|
||||
sourceComment: null,
|
||||
}],
|
||||
relationships: [],
|
||||
});
|
||||
const table = (await repository.listTables(database.id))[0]!;
|
||||
const columns = await repository.listColumns(database.id, table.id);
|
||||
const column = columns.find((candidate) => candidate.name === "status")!;
|
||||
const ward = columns.find((candidate) => candidate.name === "ward")!;
|
||||
const sampleSecret = "ONLY_IN_TRANSIENT_SAMPLE_7f29c8";
|
||||
const sourceSampler: DescriptionSourceSampler = {
|
||||
sample: vi.fn(async () => [{
|
||||
targetId: column.id,
|
||||
tableName: table.name,
|
||||
rows: [
|
||||
{ fields: [{ name: column.name, value: sampleSecret }] },
|
||||
{ fields: [{ name: column.name, value: "row-2" }] },
|
||||
{ fields: [{ name: column.name, value: "row-3" }] },
|
||||
],
|
||||
representativeValues: [{
|
||||
column: column.name,
|
||||
values: [sampleSecret, sampleSecret, "two", "three"],
|
||||
}],
|
||||
}, {
|
||||
targetId: ward.id,
|
||||
tableName: table.name,
|
||||
rows: [
|
||||
{ fields: [{ name: ward.name, value: "row-4" }] },
|
||||
{ fields: [{ name: ward.name, value: "row-5" }] },
|
||||
{ fields: [{ name: ward.name, value: "row-6-must-be-omitted" }] },
|
||||
],
|
||||
representativeValues: [{
|
||||
column: ward.name,
|
||||
values: ["ward-1", "ward-2", "ward-3-must-be-omitted"],
|
||||
}],
|
||||
}]),
|
||||
};
|
||||
const completer: ModelCompleter = {
|
||||
complete: vi.fn(async () => JSON.stringify({
|
||||
results: [{
|
||||
targetId: column.id,
|
||||
outcome: "generated",
|
||||
description: "Stato amministrativo del paziente.",
|
||||
}, {
|
||||
targetId: ward.id,
|
||||
outcome: "generated",
|
||||
description: "Reparto associato al paziente.",
|
||||
}],
|
||||
})),
|
||||
};
|
||||
const models: MetadataGenerationModels = {
|
||||
catalog: () => ({ models: [{ id: "openai-mini", label: "OpenAI Mini" }], default: "openai-mini" }),
|
||||
resolve: () => ({
|
||||
id: "openai-mini",
|
||||
provider: "openai",
|
||||
model: "gpt-4.1-mini",
|
||||
apiKeyEnv: "OPENAI_API_KEY",
|
||||
apiKey: "test-provider-secret",
|
||||
}),
|
||||
};
|
||||
const worker = new DescriptionGenerationWorker(
|
||||
repository,
|
||||
{
|
||||
read: vi.fn(async () => ({
|
||||
workspace: { workspace: { language: "it" } },
|
||||
revision: {},
|
||||
})),
|
||||
} as unknown as WorkspaceRegistry,
|
||||
models,
|
||||
completer,
|
||||
new CatalogOperationCoordinator(),
|
||||
sourceSampler,
|
||||
);
|
||||
|
||||
const run = await worker.start(
|
||||
database.id,
|
||||
"openai-mini",
|
||||
"selected_columns",
|
||||
[column.id, ward.id],
|
||||
);
|
||||
await worker.waitForRun(run.id);
|
||||
|
||||
expect(sourceSampler.sample).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ id: database.id, binding: database.binding }),
|
||||
[
|
||||
{ targetId: column.id, tableName: table.name, columnNames: [column.name] },
|
||||
{ targetId: ward.id, tableName: table.name, columnNames: [ward.name] },
|
||||
],
|
||||
expect.any(AbortSignal),
|
||||
);
|
||||
const request = vi.mocked(completer.complete).mock.calls[0]![0] as ModelCompletionRequest;
|
||||
expect(request.messages[0]?.content).toContain("untrusted");
|
||||
const userMessage = request.messages[1]!.content;
|
||||
const context = JSON.parse(userMessage.slice(userMessage.indexOf("\n") + 1));
|
||||
const sampledRows = context.targets.flatMap(
|
||||
(targetContext: { sourceSample?: { rows: unknown[] } }) => targetContext.sourceSample?.rows ?? [],
|
||||
);
|
||||
const representativeValues = context.targets.flatMap(
|
||||
(targetContext: { sourceSample?: { representativeValues: Array<{ values: unknown[] }> } }) => (
|
||||
targetContext.sourceSample?.representativeValues.flatMap((entry) => entry.values) ?? []
|
||||
),
|
||||
);
|
||||
expect(sampledRows).toHaveLength(5);
|
||||
expect(representativeValues).toHaveLength(5);
|
||||
expect(context.targets[0].sourceSample.rows).toHaveLength(3);
|
||||
expect(context.targets[1].sourceSample.rows).toHaveLength(2);
|
||||
expect(context.targets[0].sourceSample.representativeValues).toEqual([{
|
||||
column: column.name,
|
||||
values: [sampleSecret, "two", "three"],
|
||||
}]);
|
||||
expect(context.targets[1].sourceSample.representativeValues).toEqual([{
|
||||
column: ward.name,
|
||||
values: ["ward-1", "ward-2"],
|
||||
}]);
|
||||
expect(userMessage).toContain(sampleSecret);
|
||||
expect(userMessage).not.toMatch(
|
||||
/row-6-must-be-omitted|ward-3-must-be-omitted/,
|
||||
);
|
||||
|
||||
const persisted = JSON.stringify({
|
||||
run: await repository.getDescriptionGenerationRun(run.id),
|
||||
events: await repository.listDescriptionGenerationEvents(run.id),
|
||||
database: await repository.get(database.id),
|
||||
table: await repository.getTable(database.id, table.id),
|
||||
column: await repository.getColumn(database.id, table.id, column.id),
|
||||
ward: await repository.getColumn(database.id, table.id, ward.id),
|
||||
});
|
||||
expect(persisted).not.toContain(sampleSecret);
|
||||
});
|
||||
|
||||
test("continues metadata-only with one safe warning when source sampling is unavailable", async () => {
|
||||
const repository = new MemoryCatalogRepository();
|
||||
const database = await repository.create({
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "datawarehouse",
|
||||
binding: {
|
||||
transport: "rest_api",
|
||||
baseUrl: "https://dwh.example.test",
|
||||
restPath: "/rpc/run_query",
|
||||
restAuth: "none",
|
||||
},
|
||||
});
|
||||
await repository.applySchemaSync(database.id, database.version, "all", [], {
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||
tables: [{ name: "patients", sourceComment: null }],
|
||||
columns: [{
|
||||
tableName: "patients",
|
||||
name: "status",
|
||||
ordinalPosition: 1,
|
||||
dataType: "text",
|
||||
isNullable: true,
|
||||
defaultExpression: null,
|
||||
primaryKeyPosition: null,
|
||||
sourceComment: null,
|
||||
}],
|
||||
relationships: [],
|
||||
});
|
||||
const table = (await repository.listTables(database.id))[0]!;
|
||||
const column = (await repository.listColumns(database.id, table.id))[0]!;
|
||||
const samplingFailureSecret = "UNAVAILABLE_SAMPLE_DETAIL_48b1f1";
|
||||
const sourceSampler: DescriptionSourceSampler = {
|
||||
sample: vi.fn(async () => { throw new Error(samplingFailureSecret); }),
|
||||
};
|
||||
const completer: ModelCompleter = {
|
||||
complete: vi.fn(async () => JSON.stringify({
|
||||
results: [{
|
||||
targetId: column.id,
|
||||
outcome: "generated",
|
||||
description: "Stato del paziente.",
|
||||
}],
|
||||
})),
|
||||
};
|
||||
const models: MetadataGenerationModels = {
|
||||
catalog: () => ({ models: [{ id: "openai-mini", label: "OpenAI Mini" }], default: "openai-mini" }),
|
||||
resolve: () => ({
|
||||
id: "openai-mini",
|
||||
provider: "openai",
|
||||
model: "gpt-4.1-mini",
|
||||
apiKeyEnv: "OPENAI_API_KEY",
|
||||
apiKey: "test-provider-secret",
|
||||
}),
|
||||
};
|
||||
const worker = new DescriptionGenerationWorker(
|
||||
repository,
|
||||
{
|
||||
read: vi.fn(async () => ({
|
||||
workspace: { workspace: { language: "it" } },
|
||||
revision: {},
|
||||
})),
|
||||
} as unknown as WorkspaceRegistry,
|
||||
models,
|
||||
completer,
|
||||
new CatalogOperationCoordinator(),
|
||||
sourceSampler,
|
||||
);
|
||||
|
||||
const run = await worker.start(database.id, "openai-mini", "selected_columns", [column.id]);
|
||||
await worker.waitForRun(run.id);
|
||||
|
||||
expect(await repository.getDescriptionGenerationRun(run.id)).toMatchObject({
|
||||
status: "completed",
|
||||
processed: 1,
|
||||
generated: 1,
|
||||
failed: 0,
|
||||
});
|
||||
const request = vi.mocked(completer.complete).mock.calls[0]![0] as ModelCompletionRequest;
|
||||
expect(request.messages[1]?.content).not.toMatch(/sourceSample|UNAVAILABLE_SAMPLE_DETAIL/);
|
||||
const events = await repository.listDescriptionGenerationEvents(run.id);
|
||||
expect(events.filter((event) => event.level === "warning")).toEqual([
|
||||
expect.objectContaining({
|
||||
message: "Source samples unavailable for this batch; generation continued with catalog metadata only.",
|
||||
}),
|
||||
]);
|
||||
expect(JSON.stringify(events)).not.toContain(samplingFailureSecret);
|
||||
});
|
||||
@@ -0,0 +1,384 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { PostgreSqlContainer } from "@testcontainers/postgresql";
|
||||
import { CamelCasePlugin, Kysely, PostgresDialect } from "kysely";
|
||||
import { Pool } from "pg";
|
||||
import { expect, test, vi } from "vitest";
|
||||
import { buildApp } from "../src/app.js";
|
||||
import type { DescriptionSourceSampler } from "../src/catalog/description-source-sampler.js";
|
||||
import type { MetadataGenerationModels } from "../src/catalog/metadata-generation-models.js";
|
||||
import { ModelCompletionProviderError, type ModelCompleter } from "../src/catalog/model-completer.js";
|
||||
import { up as upDatabases } from "../src/catalog/migrations/001_workspace_databases.js";
|
||||
import { up as upTables } from "../src/catalog/migrations/002_catalog_tables.js";
|
||||
import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema_sync.js";
|
||||
import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js";
|
||||
import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js";
|
||||
import { loadConfig } from "../src/config.js";
|
||||
import type { WorkspaceRegistry } from "../src/workspaces/registry.js";
|
||||
|
||||
const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
|
||||
|
||||
async function terminalRun(app: ReturnType<typeof buildApp>, runId: string) {
|
||||
for (let attempt = 0; attempt < 200; attempt += 1) {
|
||||
const response = await app.inject({
|
||||
method: "GET",
|
||||
url: `/catalog/description-generation-runs/${runId}`,
|
||||
});
|
||||
const run = response.json();
|
||||
if (["completed", "completed_with_errors", "cancelled", "failed", "interrupted"].includes(run.status)) {
|
||||
return run;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
throw new Error(`Description Generation Run ${runId} did not finish`);
|
||||
}
|
||||
|
||||
test.skipIf(!dockerAvailable)("Fastify persists Description Generation success and failure through PostgreSQL", async () => {
|
||||
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
|
||||
const db = new Kysely<CatalogDatabase>({
|
||||
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
|
||||
plugins: [new CamelCasePlugin()],
|
||||
});
|
||||
let app: ReturnType<typeof buildApp> | undefined;
|
||||
try {
|
||||
await upDatabases(db);
|
||||
await upTables(db);
|
||||
await upSchemaSync(db);
|
||||
await upDescriptionGeneration(db);
|
||||
const repository = new KyselyCatalogRepository(db);
|
||||
const database = await repository.create({
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "datawarehouse",
|
||||
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
||||
});
|
||||
await repository.applySchemaSync(database.id, database.version, "all", [], {
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||
tables: [{ name: "patients", sourceComment: "Clinical patients" }],
|
||||
columns: [
|
||||
{
|
||||
tableName: "patients",
|
||||
name: "birth_date",
|
||||
ordinalPosition: 1,
|
||||
dataType: "date",
|
||||
isNullable: true,
|
||||
defaultExpression: null,
|
||||
primaryKeyPosition: null,
|
||||
sourceComment: "Patient date of birth",
|
||||
},
|
||||
{
|
||||
tableName: "patients",
|
||||
name: "status",
|
||||
ordinalPosition: 2,
|
||||
dataType: "text",
|
||||
isNullable: true,
|
||||
defaultExpression: null,
|
||||
primaryKeyPosition: null,
|
||||
sourceComment: "Patient status",
|
||||
},
|
||||
],
|
||||
relationships: [],
|
||||
});
|
||||
const table = (await repository.listTables(database.id))[0]!;
|
||||
const columns = await repository.listColumns(database.id, table.id);
|
||||
const birthDate = columns.find((column) => column.name === "birth_date")!;
|
||||
const status = columns.find((column) => column.name === "status")!;
|
||||
const curatedTable = (await repository.updateTableMetadata(
|
||||
database.id,
|
||||
table.id,
|
||||
table.version,
|
||||
"Elenco curato dei pazienti.",
|
||||
null,
|
||||
))!;
|
||||
const curatedStatus = (await repository.updateColumnMetadata(
|
||||
database.id,
|
||||
table.id,
|
||||
status.id,
|
||||
status.version,
|
||||
"Stato curato del paziente.",
|
||||
null,
|
||||
))!;
|
||||
let call = 0;
|
||||
const modelCompleter: ModelCompleter = {
|
||||
complete: vi.fn(async () => {
|
||||
call += 1;
|
||||
if (call === 1) {
|
||||
return JSON.stringify({ results: [
|
||||
{
|
||||
targetId: birthDate.id,
|
||||
outcome: "generated",
|
||||
description: "Data di nascita del paziente.",
|
||||
},
|
||||
{ targetId: status.id, outcome: "non_generatable" },
|
||||
] });
|
||||
}
|
||||
if (call === 2) {
|
||||
return JSON.stringify({ results: [{
|
||||
targetId: table.id,
|
||||
outcome: "generated",
|
||||
description: "Elenco dei pazienti e dei loro dati clinici.",
|
||||
}] });
|
||||
}
|
||||
if (call === 4) {
|
||||
return JSON.stringify({ results: [
|
||||
{
|
||||
targetId: birthDate.id,
|
||||
outcome: "generated",
|
||||
description: "Descrizione rigenerata della data di nascita.",
|
||||
},
|
||||
{
|
||||
targetId: status.id,
|
||||
outcome: "generated",
|
||||
description: "Descrizione rigenerata dello stato.",
|
||||
},
|
||||
] });
|
||||
}
|
||||
if (call === 5) {
|
||||
return JSON.stringify({ results: [{
|
||||
targetId: table.id,
|
||||
outcome: "generated",
|
||||
description: "Descrizione rigenerata della tabella pazienti.",
|
||||
}] });
|
||||
}
|
||||
if (call === 6) {
|
||||
return JSON.stringify({ results: [{
|
||||
targetId: birthDate.id,
|
||||
outcome: "generated",
|
||||
description: "Descrizione recuperata della data di nascita.",
|
||||
}] });
|
||||
}
|
||||
throw new ModelCompletionProviderError();
|
||||
}),
|
||||
};
|
||||
const models: MetadataGenerationModels = {
|
||||
catalog: () => ({ models: [{ id: "openai-mini", label: "OpenAI Mini" }], default: "openai-mini" }),
|
||||
resolve: () => ({
|
||||
id: "openai-mini",
|
||||
provider: "openai",
|
||||
model: "gpt-4.1-mini",
|
||||
apiKeyEnv: "OPENAI_API_KEY",
|
||||
apiKey: "test-provider-secret",
|
||||
}),
|
||||
};
|
||||
const registry = {
|
||||
list: vi.fn(async () => []),
|
||||
read: vi.fn(async () => ({
|
||||
workspace: { workspace: { language: "it" } },
|
||||
revision: {},
|
||||
})),
|
||||
} as unknown as WorkspaceRegistry;
|
||||
const persistedSampleSecret = "POSTGRES_TRANSIENT_SAMPLE_6a0d7b";
|
||||
const descriptionSourceSampler: DescriptionSourceSampler = {
|
||||
sample: vi.fn(async (_database, targets) => targets.map((target) => ({
|
||||
targetId: target.targetId,
|
||||
tableName: target.tableName,
|
||||
rows: [{
|
||||
fields: target.columnNames.slice(0, 1).map((name) => ({
|
||||
name,
|
||||
value: persistedSampleSecret,
|
||||
})),
|
||||
}],
|
||||
representativeValues: target.columnNames.slice(0, 1).map((column) => ({
|
||||
column,
|
||||
values: [persistedSampleSecret],
|
||||
})),
|
||||
}))),
|
||||
};
|
||||
app = buildApp(loadConfig({ NODE_ENV: "test", THT_HARNESS_DIR: "/missing" }), {
|
||||
thtRunner: {} as never,
|
||||
workspaceRegistry: registry,
|
||||
workspaceDiagnoser: vi.fn(),
|
||||
catalogRepository: repository,
|
||||
metadataGenerationModels: models,
|
||||
modelCompleter,
|
||||
descriptionSourceSampler,
|
||||
});
|
||||
|
||||
const successfulStart = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/description-generation-runs`,
|
||||
payload: {
|
||||
modelId: "openai-mini",
|
||||
scope: "selected_columns",
|
||||
targetIds: [status.id, birthDate.id],
|
||||
},
|
||||
});
|
||||
expect(successfulStart.statusCode).toBe(202);
|
||||
expect(await terminalRun(app, successfulStart.json().id)).toMatchObject({
|
||||
status: "completed",
|
||||
total: 2,
|
||||
processed: 2,
|
||||
generated: 1,
|
||||
nonGeneratable: 1,
|
||||
failed: 0,
|
||||
});
|
||||
expect(await repository.getColumn(database.id, table.id, birthDate.id)).toMatchObject({
|
||||
generatedDescription: "Data di nascita del paziente.",
|
||||
version: birthDate.version + 1,
|
||||
});
|
||||
const generatedStatus = (await repository.getColumn(database.id, table.id, status.id))!;
|
||||
expect(generatedStatus).toMatchObject({
|
||||
description: "Stato curato del paziente.",
|
||||
generatedDescription: "Non generabile",
|
||||
version: curatedStatus.version + 1,
|
||||
});
|
||||
|
||||
const tableStart = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/description-generation-runs`,
|
||||
payload: { modelId: "openai-mini", scope: "selected_tables", targetIds: [table.id] },
|
||||
});
|
||||
expect(tableStart.statusCode).toBe(202);
|
||||
expect(await terminalRun(app, tableStart.json().id)).toMatchObject({
|
||||
scope: "selected_tables",
|
||||
status: "completed",
|
||||
processed: 1,
|
||||
generated: 1,
|
||||
nonGeneratable: 0,
|
||||
failed: 0,
|
||||
});
|
||||
expect(await repository.getTable(database.id, table.id)).toMatchObject({
|
||||
description: "Elenco curato dei pazienti.",
|
||||
generatedDescription: "Elenco dei pazienti e dei loro dati clinici.",
|
||||
version: curatedTable.version + 1,
|
||||
});
|
||||
|
||||
const failedStart = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/description-generation-runs`,
|
||||
payload: { modelId: "openai-mini", scope: "selected_columns", targetIds: [status.id] },
|
||||
});
|
||||
expect(failedStart.statusCode).toBe(202);
|
||||
const failedRun = await terminalRun(app, failedStart.json().id);
|
||||
expect(failedRun).toMatchObject({
|
||||
status: "completed_with_errors",
|
||||
processed: 1,
|
||||
generated: 0,
|
||||
failed: 1,
|
||||
errorSummary: "Description generation completed with errors.",
|
||||
});
|
||||
expect(await repository.getColumn(database.id, table.id, status.id)).toMatchObject({
|
||||
description: "Stato curato del paziente.",
|
||||
generatedDescription: "Non generabile",
|
||||
version: generatedStatus.version,
|
||||
});
|
||||
const events = await app.inject({
|
||||
method: "GET",
|
||||
url: `/catalog/description-generation-runs/${failedRun.id}/events-list`,
|
||||
});
|
||||
expect(events.statusCode).toBe(200);
|
||||
expect(events.json().find((event: { level: string }) => event.level === "error")).toEqual(expect.objectContaining({
|
||||
level: "error",
|
||||
message: `The model provider request failed. Affected Catalog Column target: ${status.id}.`,
|
||||
}));
|
||||
expect(events.body).not.toMatch(/test-provider-secret|gpt-4\.1-mini|raw provider/i);
|
||||
|
||||
const allStart = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/description-generation-runs`,
|
||||
payload: { modelId: "openai-mini", scope: "all" },
|
||||
});
|
||||
expect(allStart.statusCode).toBe(202);
|
||||
const allRun = await terminalRun(app, allStart.json().id);
|
||||
expect(allRun).toMatchObject({
|
||||
scope: "all",
|
||||
status: "completed",
|
||||
total: 3,
|
||||
processed: 3,
|
||||
generated: 3,
|
||||
nonGeneratable: 0,
|
||||
failed: 0,
|
||||
});
|
||||
expect(await repository.getColumn(database.id, table.id, birthDate.id)).toMatchObject({
|
||||
generatedDescription: "Descrizione rigenerata della data di nascita.",
|
||||
});
|
||||
expect(await repository.getColumn(database.id, table.id, status.id)).toMatchObject({
|
||||
generatedDescription: "Descrizione rigenerata dello stato.",
|
||||
});
|
||||
expect(await repository.getTable(database.id, table.id)).toMatchObject({
|
||||
generatedDescription: "Descrizione rigenerata della tabella pazienti.",
|
||||
});
|
||||
const allEvents = await app.inject({
|
||||
method: "GET",
|
||||
url: `/catalog/description-generation-runs/${allRun.id}/events-list`,
|
||||
});
|
||||
expect(allEvents.json()[0]).toEqual(expect.objectContaining({
|
||||
message: "Description generation queued (scope: all).",
|
||||
}));
|
||||
|
||||
const regeneratedBirthDate = (await repository.getColumn(
|
||||
database.id, table.id, birthDate.id,
|
||||
))!;
|
||||
await repository.updateColumnMetadata(
|
||||
database.id,
|
||||
table.id,
|
||||
birthDate.id,
|
||||
regeneratedBirthDate.version,
|
||||
regeneratedBirthDate.description,
|
||||
" ",
|
||||
);
|
||||
const missingStart = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/description-generation-runs`,
|
||||
payload: { modelId: "openai-mini", scope: "missing" },
|
||||
});
|
||||
expect(missingStart.statusCode).toBe(202);
|
||||
expect(await terminalRun(app, missingStart.json().id)).toMatchObject({
|
||||
scope: "missing",
|
||||
status: "completed",
|
||||
total: 1,
|
||||
processed: 1,
|
||||
generated: 1,
|
||||
nonGeneratable: 0,
|
||||
failed: 0,
|
||||
});
|
||||
expect(await repository.getColumn(database.id, table.id, birthDate.id)).toMatchObject({
|
||||
generatedDescription: "Descrizione recuperata della data di nascita.",
|
||||
});
|
||||
expect(modelCompleter.complete).toHaveBeenCalledTimes(6);
|
||||
expect(JSON.stringify(vi.mocked(modelCompleter.complete).mock.calls)).toContain(persistedSampleSecret);
|
||||
|
||||
const runIds = [
|
||||
successfulStart.json().id,
|
||||
tableStart.json().id,
|
||||
failedStart.json().id,
|
||||
allStart.json().id,
|
||||
missingStart.json().id,
|
||||
];
|
||||
const durableState = JSON.stringify({
|
||||
runs: await Promise.all(runIds.map(async (runId) => (
|
||||
await repository.getDescriptionGenerationRun(runId)
|
||||
))),
|
||||
events: await Promise.all(runIds.map(async (runId) => (
|
||||
await repository.listDescriptionGenerationEvents(runId)
|
||||
))),
|
||||
database: await repository.get(database.id),
|
||||
table: await repository.getTable(database.id, table.id),
|
||||
columns: await repository.listColumns(database.id, table.id),
|
||||
});
|
||||
expect(durableState).not.toContain(persistedSampleSecret);
|
||||
const apiResponses = await Promise.all([
|
||||
...runIds.flatMap((runId) => [
|
||||
app!.inject({ method: "GET", url: `/catalog/description-generation-runs/${runId}` }),
|
||||
app!.inject({
|
||||
method: "GET",
|
||||
url: `/catalog/description-generation-runs/${runId}/events-list`,
|
||||
}),
|
||||
]),
|
||||
app.inject({ method: "GET", url: `/catalog/databases/${database.id}` }),
|
||||
app.inject({ method: "GET", url: `/catalog/databases/${database.id}/tables` }),
|
||||
app.inject({
|
||||
method: "GET",
|
||||
url: `/catalog/databases/${database.id}/tables/${table.id}/columns`,
|
||||
}),
|
||||
]);
|
||||
expect(apiResponses.map((response) => response.body).join("\n")).not.toContain(
|
||||
persistedSampleSecret,
|
||||
);
|
||||
} finally {
|
||||
if (app) await app.close();
|
||||
await db.destroy();
|
||||
await container.stop();
|
||||
}
|
||||
}, 60_000);
|
||||
@@ -0,0 +1,111 @@
|
||||
import { expect, test, vi } from "vitest";
|
||||
import {
|
||||
PostgresDescriptionSourceSampler,
|
||||
type DescriptionSourceSamplingTarget,
|
||||
} from "../src/catalog/description-source-sampler.js";
|
||||
import type {
|
||||
CatalogDatabaseClient,
|
||||
CatalogPostgresAccess,
|
||||
} from "../src/catalog/postgres-access.js";
|
||||
import type { WorkspaceDatabase } from "../src/catalog/types.js";
|
||||
|
||||
const database: WorkspaceDatabase = {
|
||||
id: "11111111-1111-4111-8111-111111111111",
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: 'clinical"data',
|
||||
version: 1,
|
||||
createdAt: "2026-08-28T08:00:00Z",
|
||||
updatedAt: "2026-08-28T08:00:00Z",
|
||||
connectionStatus: "reachable",
|
||||
binding: {
|
||||
transport: "postgres_direct",
|
||||
host: "db.internal",
|
||||
port: 5432,
|
||||
username: "reader",
|
||||
},
|
||||
};
|
||||
|
||||
const target: DescriptionSourceSamplingTarget = {
|
||||
targetId: "22222222-2222-4222-8222-222222222222",
|
||||
tableName: 'patient"facts',
|
||||
columnNames: ['status"code', "ward"],
|
||||
};
|
||||
|
||||
test("samples at most five source rows and five distinct non-null examples in a read-only transaction", async () => {
|
||||
const query = vi.fn(async (sql: string) => {
|
||||
if (!sql.startsWith("SELECT")) return { rows: [] };
|
||||
return {
|
||||
rows: [
|
||||
{ 'status"code': "active", ward: null },
|
||||
{ 'status"code': "pending", ward: "A" },
|
||||
{ 'status"code': "closed", ward: "A" },
|
||||
{ 'status"code': "transferred", ward: "B" },
|
||||
{ 'status"code': "unknown", ward: "C" },
|
||||
{ 'status"code': "must-not-be-sampled", ward: "D" },
|
||||
],
|
||||
};
|
||||
});
|
||||
const end = vi.fn(async () => undefined);
|
||||
const access: CatalogPostgresAccess = {
|
||||
connect: vi.fn(async () => ({ query, end }) as CatalogDatabaseClient),
|
||||
};
|
||||
const sampler = new PostgresDescriptionSourceSampler(access);
|
||||
const controller = new AbortController();
|
||||
|
||||
const samples = await sampler.sample(database, [target], controller.signal);
|
||||
|
||||
expect(samples).toEqual([{
|
||||
targetId: target.targetId,
|
||||
tableName: target.tableName,
|
||||
rows: [
|
||||
{ fields: [{ name: 'status"code', value: "active" }, { name: "ward", value: null }] },
|
||||
{ fields: [{ name: 'status"code', value: "pending" }, { name: "ward", value: "A" }] },
|
||||
{ fields: [{ name: 'status"code', value: "closed" }, { name: "ward", value: "A" }] },
|
||||
{ fields: [{ name: 'status"code', value: "transferred" }, { name: "ward", value: "B" }] },
|
||||
{ fields: [{ name: 'status"code', value: "unknown" }, { name: "ward", value: "C" }] },
|
||||
],
|
||||
representativeValues: [
|
||||
{
|
||||
column: 'status"code',
|
||||
values: ["active", "pending", "closed", "transferred"],
|
||||
},
|
||||
{ column: "ward", values: ["A"] },
|
||||
],
|
||||
}]);
|
||||
expect(access.connect).toHaveBeenCalledWith(database, controller.signal);
|
||||
expect(samples[0]!.representativeValues.flatMap((entry) => entry.values)).toHaveLength(5);
|
||||
expect(query.mock.calls).toEqual([
|
||||
["BEGIN TRANSACTION READ ONLY", []],
|
||||
[
|
||||
'SELECT LEFT(("status""code")::text, $1) AS "status""code", LEFT(("ward")::text, $1) AS "ward" FROM "clinical""data"."patient""facts" LIMIT $2',
|
||||
[256, 5],
|
||||
],
|
||||
["ROLLBACK", []],
|
||||
]);
|
||||
expect(query.mock.calls.map(([sql]) => String(sql).split(" ")[0])).toEqual([
|
||||
"BEGIN",
|
||||
"SELECT",
|
||||
"ROLLBACK",
|
||||
]);
|
||||
expect(end).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
test("rolls back and closes the source connection when sampling fails", async () => {
|
||||
const query = vi.fn(async (sql: string) => {
|
||||
if (sql.startsWith("SELECT")) throw new Error("distinctive-source-secret");
|
||||
return { rows: [] };
|
||||
});
|
||||
const end = vi.fn(async () => undefined);
|
||||
const access: CatalogPostgresAccess = {
|
||||
connect: vi.fn(async () => ({ query, end }) as CatalogDatabaseClient),
|
||||
};
|
||||
const sampler = new PostgresDescriptionSourceSampler(access);
|
||||
const controller = new AbortController();
|
||||
|
||||
await expect(sampler.sample(database, [target], controller.signal)).rejects.toThrow();
|
||||
|
||||
expect(query).toHaveBeenCalledWith("ROLLBACK", []);
|
||||
expect(end).toHaveBeenCalledOnce();
|
||||
});
|
||||
@@ -18,3 +18,23 @@ test("reserves duplicate database ids only once for a batch operation", async ()
|
||||
expect(await coordinator.runMany(["database-a", "database-a"], async () => "completed"))
|
||||
.toBe("completed");
|
||||
});
|
||||
|
||||
test("stale generation recovery releases only its own reservation token", () => {
|
||||
const coordinator = new CatalogOperationCoordinator();
|
||||
const releaseOtherOperation = coordinator.reserve("database-a");
|
||||
|
||||
coordinator.releaseStale("database-a", "description_generation");
|
||||
expect(() => coordinator.reserve("database-a")).toThrow(
|
||||
"A database operation is already in progress",
|
||||
);
|
||||
releaseOtherOperation();
|
||||
|
||||
const releaseStaleGeneration = coordinator.reserve("database-a", "description_generation");
|
||||
coordinator.releaseStale("database-a", "description_generation");
|
||||
const releaseNewOperation = coordinator.reserve("database-a");
|
||||
releaseStaleGeneration();
|
||||
expect(() => coordinator.reserve("database-a")).toThrow(
|
||||
"A database operation is already in progress",
|
||||
);
|
||||
releaseNewOperation();
|
||||
});
|
||||
|
||||
@@ -169,3 +169,23 @@ test("connects pg through OpenSSH, supplies askpass, and releases all secret lea
|
||||
expect(child.kill).toHaveBeenCalledWith("SIGTERM");
|
||||
expect(leasedPaths.some(existsSync)).toBe(false);
|
||||
});
|
||||
|
||||
test("fails before creating a transport when the sampling signal is already aborted", async () => {
|
||||
const store = secretStore();
|
||||
store.putMany("psd-clinical", {
|
||||
[CATALOG_SECRET_IDS.password]: "db-password",
|
||||
[CATALOG_SECRET_IDS.sshPrivateKey]: "PRIVATE KEY\n",
|
||||
[CATALOG_SECRET_IDS.sshKnownHosts]: "bastion.internal ssh-ed25519 AAAATEST\n",
|
||||
});
|
||||
const spawnSsh = vi.fn(() => fakeChild());
|
||||
const createClient = vi.fn();
|
||||
const access = new ConcreteCatalogPostgresAccess(store, { spawnSsh, createClient });
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
await expect(access.connect(sshDatabase(), controller.signal)).rejects.toThrow(
|
||||
"PostgreSQL connector aborted",
|
||||
);
|
||||
expect(spawnSsh).not.toHaveBeenCalled();
|
||||
expect(createClient).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -9,6 +9,7 @@ import { up as upDatabases } from "../src/catalog/migrations/001_workspace_datab
|
||||
import { up as upTables } from "../src/catalog/migrations/002_catalog_tables.js";
|
||||
import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema_sync.js";
|
||||
import { up as upRuntimeSequencePrivileges } from "../src/catalog/migrations/004_catalog_runtime_sequence_privileges.js";
|
||||
import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js";
|
||||
|
||||
const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
|
||||
|
||||
@@ -200,3 +201,285 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository performs scoped metadata cl
|
||||
await container.stop();
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates selected table and column descriptions", async () => {
|
||||
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
|
||||
const db = new Kysely<CatalogDatabase>({
|
||||
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
|
||||
plugins: [new CamelCasePlugin()],
|
||||
});
|
||||
try {
|
||||
await upDatabases(db);
|
||||
await upTables(db);
|
||||
await upSchemaSync(db);
|
||||
const repository = new KyselyCatalogRepository(db);
|
||||
const database = await repository.create({
|
||||
workspaceId: "consolidation-test",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "public",
|
||||
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
||||
});
|
||||
const snapshot: ObservedSchemaSnapshot = {
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||
tables: [
|
||||
{ name: "patients", sourceComment: null },
|
||||
{ name: "visits", sourceComment: null },
|
||||
],
|
||||
columns: [
|
||||
{ tableName: "visits", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
||||
{ tableName: "visits", name: "patient_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
||||
],
|
||||
relationships: [],
|
||||
};
|
||||
await repository.applySchemaSync(database.id, database.version, "all", [], snapshot);
|
||||
const tables = await repository.listTables(database.id);
|
||||
const patients = tables.find((table) => table.name === "patients")!;
|
||||
const visits = tables.find((table) => table.name === "visits")!;
|
||||
await repository.updateTableMetadata(
|
||||
database.id, patients.id, patients.version, "Old patients", "Generated patients",
|
||||
);
|
||||
await repository.updateTableMetadata(
|
||||
database.id, visits.id, visits.version, "Keep visits", " ",
|
||||
);
|
||||
|
||||
expect(await repository.consolidateGeneratedDescriptions(
|
||||
database.id, "tables", [patients.id, visits.id],
|
||||
)).toEqual({ copied: 1, skipped: 1 });
|
||||
expect(await repository.getTable(database.id, patients.id)).toMatchObject({
|
||||
description: "Generated patients",
|
||||
generatedDescription: "Generated patients",
|
||||
version: patients.version + 2,
|
||||
});
|
||||
expect(await repository.getTable(database.id, visits.id)).toMatchObject({
|
||||
description: "Keep visits",
|
||||
generatedDescription: " ",
|
||||
version: visits.version + 1,
|
||||
});
|
||||
|
||||
const columns = await repository.listColumns(database.id, visits.id);
|
||||
const id = columns.find((column) => column.name === "id")!;
|
||||
const patientId = columns.find((column) => column.name === "patient_id")!;
|
||||
await repository.updateColumnMetadata(
|
||||
database.id, visits.id, id.id, id.version, "Old id", "Generated id",
|
||||
);
|
||||
await repository.updateColumnMetadata(
|
||||
database.id, visits.id, patientId.id, patientId.version, "Keep patient reference", null,
|
||||
);
|
||||
|
||||
expect(await repository.consolidateGeneratedDescriptions(
|
||||
database.id, "columns", [id.id, patientId.id],
|
||||
)).toEqual({ copied: 1, skipped: 1 });
|
||||
expect(await repository.getColumn(database.id, visits.id, id.id)).toMatchObject({
|
||||
description: "Generated id",
|
||||
generatedDescription: "Generated id",
|
||||
version: id.version + 2,
|
||||
});
|
||||
expect(await repository.getColumn(database.id, visits.id, patientId.id)).toMatchObject({
|
||||
description: "Keep patient reference",
|
||||
generatedDescription: null,
|
||||
version: patientId.version + 1,
|
||||
});
|
||||
|
||||
const currentVisits = (await repository.getTable(database.id, visits.id))!;
|
||||
await repository.updateTableMetadata(
|
||||
database.id, visits.id, currentVisits.version, "Still curated visits", "Generated visits",
|
||||
);
|
||||
expect(await repository.consolidateGeneratedDescriptions(database.id, "tables", [
|
||||
visits.id,
|
||||
"99999999-9999-4999-8999-999999999999",
|
||||
])).toBeUndefined();
|
||||
expect(await repository.getTable(database.id, visits.id)).toMatchObject({
|
||||
description: "Still curated visits",
|
||||
generatedDescription: "Generated visits",
|
||||
});
|
||||
} finally {
|
||||
await db.destroy();
|
||||
await container.stop();
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
test.skipIf(!dockerAvailable)("PostgreSQL repository persists globally exclusive Description Generation Runs and ordered events", async () => {
|
||||
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
|
||||
const db = new Kysely<CatalogDatabase>({
|
||||
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
|
||||
plugins: [new CamelCasePlugin()],
|
||||
});
|
||||
try {
|
||||
await upDatabases(db);
|
||||
await upTables(db);
|
||||
await upSchemaSync(db);
|
||||
await upDescriptionGeneration(db);
|
||||
const repository = new KyselyCatalogRepository(db);
|
||||
const firstDatabase = await repository.create({
|
||||
workspaceId: "generation-one",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse_one",
|
||||
schema: "public",
|
||||
binding: { transport: "postgres_direct", host: "one.internal", port: 5432, username: "reader" },
|
||||
});
|
||||
const secondDatabase = await repository.create({
|
||||
workspaceId: "generation-two",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse_two",
|
||||
schema: "public",
|
||||
binding: { transport: "postgres_direct", host: "two.internal", port: 5432, username: "reader" },
|
||||
});
|
||||
await repository.applySchemaSync(firstDatabase.id, firstDatabase.version, "all", [], {
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||
tables: [{ name: "patients", sourceComment: "Clinical patients" }],
|
||||
columns: [{
|
||||
tableName: "patients",
|
||||
name: "birth_date",
|
||||
ordinalPosition: 1,
|
||||
dataType: "date",
|
||||
isNullable: true,
|
||||
defaultExpression: null,
|
||||
primaryKeyPosition: null,
|
||||
sourceComment: "Patient date of birth",
|
||||
}],
|
||||
relationships: [],
|
||||
});
|
||||
const table = (await repository.listTables(firstDatabase.id))[0]!;
|
||||
const column = (await repository.listColumns(firstDatabase.id, table.id))[0]!;
|
||||
|
||||
const run = await repository.createDescriptionGenerationRun(
|
||||
firstDatabase.id,
|
||||
"selected_columns",
|
||||
"openai-mini",
|
||||
"it",
|
||||
1,
|
||||
);
|
||||
expect(run).toMatchObject({
|
||||
databaseId: firstDatabase.id,
|
||||
scope: "selected_columns",
|
||||
modelId: "openai-mini",
|
||||
language: "it",
|
||||
status: "queued",
|
||||
total: 1,
|
||||
processed: 0,
|
||||
generated: 0,
|
||||
nonGeneratable: 0,
|
||||
failed: 0,
|
||||
startedAt: null,
|
||||
finishedAt: null,
|
||||
errorSummary: null,
|
||||
});
|
||||
await expect(repository.createDescriptionGenerationRun(
|
||||
secondDatabase.id,
|
||||
"selected_columns",
|
||||
"openai-mini",
|
||||
"en",
|
||||
1,
|
||||
)).rejects.toThrow("A description generation run is already active");
|
||||
await repository.updateDescriptionGenerationRun(run.id, {
|
||||
status: "running",
|
||||
startedAt: new Date().toISOString(),
|
||||
});
|
||||
await expect(repository.createDescriptionGenerationRun(
|
||||
secondDatabase.id,
|
||||
"selected_columns",
|
||||
"openai-mini",
|
||||
"en",
|
||||
1,
|
||||
)).rejects.toThrow("A description generation run is already active");
|
||||
|
||||
await repository.appendDescriptionGenerationEvent(run.id, "info", "Description generation queued.");
|
||||
await repository.appendDescriptionGenerationEvent(run.id, "info", "Description generation started.");
|
||||
expect(await repository.listDescriptionGenerationEvents(run.id, 1)).toEqual([
|
||||
expect.objectContaining({
|
||||
runId: run.id,
|
||||
sequence: 2,
|
||||
level: "info",
|
||||
message: "Description generation started.",
|
||||
createdAt: expect.any(String),
|
||||
}),
|
||||
]);
|
||||
|
||||
const updatedColumn = await repository.updateColumnMetadata(
|
||||
firstDatabase.id,
|
||||
table.id,
|
||||
column.id,
|
||||
column.version,
|
||||
column.description,
|
||||
"Data di nascita del paziente.",
|
||||
);
|
||||
expect(updatedColumn).toMatchObject({
|
||||
generatedDescription: "Data di nascita del paziente.",
|
||||
version: column.version + 1,
|
||||
});
|
||||
expect(await repository.updateDescriptionGenerationRun(run.id, {
|
||||
status: "completed",
|
||||
processed: 1,
|
||||
generated: 1,
|
||||
startedAt: new Date().toISOString(),
|
||||
finishedAt: new Date().toISOString(),
|
||||
})).toMatchObject({
|
||||
status: "completed",
|
||||
processed: 1,
|
||||
generated: 1,
|
||||
});
|
||||
|
||||
const next = await repository.createDescriptionGenerationRun(
|
||||
secondDatabase.id,
|
||||
"missing",
|
||||
"openai-mini",
|
||||
"en",
|
||||
1,
|
||||
);
|
||||
expect(await repository.getDescriptionGenerationRun(next.id)).toMatchObject({
|
||||
scope: "missing",
|
||||
total: 1,
|
||||
});
|
||||
await repository.updateDescriptionGenerationRun(next.id, {
|
||||
status: "running",
|
||||
startedAt: new Date().toISOString(),
|
||||
});
|
||||
expect(await repository.updateDescriptionGenerationRun(next.id, {
|
||||
status: "failed",
|
||||
processed: 1,
|
||||
failed: 1,
|
||||
finishedAt: new Date().toISOString(),
|
||||
errorSummary: "The model provider request failed.",
|
||||
})).toMatchObject({
|
||||
status: "failed",
|
||||
processed: 1,
|
||||
failed: 1,
|
||||
errorSummary: "The model provider request failed.",
|
||||
});
|
||||
|
||||
const allRun = await repository.createDescriptionGenerationRun(
|
||||
firstDatabase.id,
|
||||
"all",
|
||||
"openai-mini",
|
||||
"it",
|
||||
2,
|
||||
);
|
||||
expect(await repository.getDescriptionGenerationRun(allRun.id)).toMatchObject({
|
||||
scope: "all",
|
||||
total: 2,
|
||||
});
|
||||
expect(await repository.getActiveDescriptionGenerationRun()).toMatchObject({ id: allRun.id });
|
||||
expect((await repository.listDescriptionGenerationRuns(2)).map((candidate) => candidate.id)).toEqual([
|
||||
allRun.id,
|
||||
next.id,
|
||||
]);
|
||||
|
||||
expect(await repository.interruptActiveDescriptionGenerationRuns(
|
||||
"Description generation was interrupted by backend restart.",
|
||||
)).toEqual([
|
||||
expect.objectContaining({
|
||||
id: allRun.id,
|
||||
status: "interrupted",
|
||||
finishedAt: expect.any(String),
|
||||
errorSummary: "Description generation was interrupted by backend restart.",
|
||||
}),
|
||||
]);
|
||||
expect(await repository.getActiveDescriptionGenerationRun()).toBeUndefined();
|
||||
} finally {
|
||||
await db.destroy();
|
||||
await container.stop();
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
@@ -5,6 +5,7 @@ import { afterEach, expect, test, vi } from "vitest";
|
||||
import { buildApp } from "../src/app.js";
|
||||
import { loadConfig } from "../src/config.js";
|
||||
import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
|
||||
import { CatalogOperationCoordinator } from "../src/catalog/operation-coordinator.js";
|
||||
import type { CatalogSchemaIntrospector } from "../src/catalog/schema-introspector.js";
|
||||
import type { CatalogSyncRun, ObservedSchemaSnapshot } from "../src/catalog/types.js";
|
||||
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||
@@ -95,7 +96,7 @@ async function waitFor(repository: MemoryCatalogRepository, runId: string, state
|
||||
throw new Error(`Run ${runId} did not reach ${state}`);
|
||||
}
|
||||
|
||||
async function setup() {
|
||||
async function setup(env: Record<string, string> = {}) {
|
||||
const secretRoot = mkdtempSync(join(tmpdir(), "catalog-schema-secret-"));
|
||||
const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-schema-runtime-"));
|
||||
roots.push(secretRoot, runtimeRoot);
|
||||
@@ -116,21 +117,23 @@ async function setup() {
|
||||
return structuredClone(observed);
|
||||
});
|
||||
const introspector: CatalogSchemaIntrospector = { scan };
|
||||
const operations = new CatalogOperationCoordinator();
|
||||
const registry = {
|
||||
list: vi.fn(async () => [revision]),
|
||||
listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]),
|
||||
read: vi.fn(async () => ({ workspace, revision })),
|
||||
} as unknown as WorkspaceRegistry;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test" }), {
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test", ...env }), {
|
||||
thtRunner: {} as never,
|
||||
workspaceRegistry: registry,
|
||||
workspaceSecretStore: new WorkspaceSecretStore({ root: secretRoot, runtimeRoot, installationId: "test" }),
|
||||
catalogRepository: repository,
|
||||
catalogSchemaIntrospector: introspector,
|
||||
catalogOperationCoordinator: operations,
|
||||
workspaceDiagnoser: vi.fn(),
|
||||
});
|
||||
return {
|
||||
app, repository, database: (await repository.get(created.id))!, scan,
|
||||
app, repository, database: (await repository.get(created.id))!, scan, operations,
|
||||
setObserved(next: ObservedSchemaSnapshot) { observed = next; },
|
||||
};
|
||||
}
|
||||
@@ -243,6 +246,207 @@ test("keeps generated descriptions editable and preserves them across synchroniz
|
||||
expect(await repository.getColumn(database.id, patients.id, idColumn.id)).toMatchObject({ description: "Reviewed key", generatedDescription: "Generated key draft" });
|
||||
});
|
||||
|
||||
test("consolidates non-empty generated table descriptions and reports skipped selections", async () => {
|
||||
const { app, repository, database, scan } = await setup();
|
||||
await seedCatalog(repository, database);
|
||||
const tables = await repository.listTables(database.id);
|
||||
const patients = tables.find((table) => table.name === "patients")!;
|
||||
const visits = tables.find((table) => table.name === "visits")!;
|
||||
await repository.updateTableMetadata(
|
||||
database.id,
|
||||
patients.id,
|
||||
patients.version,
|
||||
"Curated patients",
|
||||
"Generated patients",
|
||||
);
|
||||
await repository.updateTableMetadata(
|
||||
database.id,
|
||||
visits.id,
|
||||
visits.version,
|
||||
"Keep curated visits",
|
||||
null,
|
||||
);
|
||||
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
||||
payload: { target: "tables", targetIds: [patients.id, visits.id] },
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.json()).toEqual({ copied: 1, skipped: 1 });
|
||||
expect(await repository.getTable(database.id, patients.id)).toMatchObject({
|
||||
description: "Generated patients",
|
||||
generatedDescription: "Generated patients",
|
||||
version: patients.version + 2,
|
||||
});
|
||||
expect(await repository.getTable(database.id, visits.id)).toMatchObject({
|
||||
description: "Keep curated visits",
|
||||
generatedDescription: null,
|
||||
version: visits.version + 1,
|
||||
});
|
||||
expect(scan).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("consolidates non-empty generated column descriptions and preserves curated text for empty proposals", async () => {
|
||||
const { app, repository, database, scan } = await setup();
|
||||
await seedCatalog(repository, database);
|
||||
const visits = (await repository.listTables(database.id)).find((table) => table.name === "visits")!;
|
||||
const columns = await repository.listColumns(database.id, visits.id);
|
||||
const id = columns.find((column) => column.name === "id")!;
|
||||
const patientId = columns.find((column) => column.name === "patient_id")!;
|
||||
await repository.updateColumnMetadata(
|
||||
database.id,
|
||||
visits.id,
|
||||
id.id,
|
||||
id.version,
|
||||
"Curated visit identifier",
|
||||
"Generated visit identifier",
|
||||
);
|
||||
await repository.updateColumnMetadata(
|
||||
database.id,
|
||||
visits.id,
|
||||
patientId.id,
|
||||
patientId.version,
|
||||
"Keep curated patient reference",
|
||||
"",
|
||||
);
|
||||
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
||||
payload: { target: "columns", targetIds: [id.id, patientId.id] },
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.json()).toEqual({ copied: 1, skipped: 1 });
|
||||
expect(await repository.getColumn(database.id, visits.id, id.id)).toMatchObject({
|
||||
description: "Generated visit identifier",
|
||||
generatedDescription: "Generated visit identifier",
|
||||
version: id.version + 2,
|
||||
});
|
||||
expect(await repository.getColumn(database.id, visits.id, patientId.id)).toMatchObject({
|
||||
description: "Keep curated patient reference",
|
||||
generatedDescription: "",
|
||||
version: patientId.version + 1,
|
||||
});
|
||||
expect(scan).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("rejects description consolidation while the Workspace Database is reserved", async () => {
|
||||
const { app, repository, database, operations } = await setup();
|
||||
await seedCatalog(repository, database);
|
||||
const table = (await repository.listTables(database.id))[0]!;
|
||||
const edited = await repository.updateTableMetadata(
|
||||
database.id,
|
||||
table.id,
|
||||
table.version,
|
||||
"Existing curated text",
|
||||
"Generated text",
|
||||
);
|
||||
const release = operations.reserve(database.id);
|
||||
try {
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
||||
payload: { target: "tables", targetIds: [table.id] },
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(409);
|
||||
expect(response.json()).toEqual({
|
||||
code: "database_operation_in_progress",
|
||||
message: "A database operation is already in progress.",
|
||||
});
|
||||
expect(await repository.getTable(database.id, table.id)).toMatchObject({
|
||||
description: "Existing curated text",
|
||||
generatedDescription: "Generated text",
|
||||
version: edited!.version,
|
||||
});
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
});
|
||||
|
||||
test("requires database.manage for description consolidation", async () => {
|
||||
const { app, repository, database } = await setup({ AUTH_MODE: "upstream" });
|
||||
await seedCatalog(repository, database);
|
||||
const table = (await repository.listTables(database.id))[0]!;
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
||||
headers: {
|
||||
"x-thoth-principal-issuer": "portal",
|
||||
"x-thoth-principal-subject": "catalog-reader",
|
||||
"x-thoth-is-admin": "0",
|
||||
},
|
||||
payload: { target: "tables", targetIds: [table.id] },
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(403);
|
||||
expect(response.json()).toEqual({ code: "auth_forbidden", error: "This operation is not permitted" });
|
||||
});
|
||||
|
||||
test("strictly validates description consolidation database and target ids", async () => {
|
||||
const { app, repository, database } = await setup();
|
||||
await seedCatalog(repository, database);
|
||||
const table = (await repository.listTables(database.id))[0]!;
|
||||
|
||||
const responses = await Promise.all([
|
||||
app.inject({
|
||||
method: "POST",
|
||||
url: "/catalog/databases/not-a-uuid/descriptions/consolidate",
|
||||
payload: { target: "tables", targetIds: [table.id] },
|
||||
}),
|
||||
app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
||||
payload: { target: "tables", targetIds: ["not-a-uuid"] },
|
||||
}),
|
||||
app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
||||
payload: { target: "tables", targetIds: [table.id], unexpected: true },
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(responses.map((response) => response.statusCode)).toEqual([400, 400, 400]);
|
||||
for (const response of responses) {
|
||||
expect(response.json()).toEqual({
|
||||
code: "description_consolidation_invalid",
|
||||
message: "Description consolidation request is invalid.",
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test("rejects a missing consolidation target without copying valid selections", async () => {
|
||||
const { app, repository, database } = await setup();
|
||||
await seedCatalog(repository, database);
|
||||
const table = (await repository.listTables(database.id))[0]!;
|
||||
const edited = await repository.updateTableMetadata(
|
||||
database.id,
|
||||
table.id,
|
||||
table.version,
|
||||
"Existing curated text",
|
||||
"Generated text",
|
||||
);
|
||||
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/descriptions/consolidate`,
|
||||
payload: {
|
||||
target: "tables",
|
||||
targetIds: [table.id, "99999999-9999-4999-8999-999999999999"],
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.statusCode).toBe(404);
|
||||
expect(await repository.getTable(database.id, table.id)).toMatchObject({
|
||||
description: "Existing curated text",
|
||||
generatedDescription: "Generated text",
|
||||
version: edited!.version,
|
||||
});
|
||||
});
|
||||
|
||||
test("waits for confirmation and rescans before applying destructive changes", async () => {
|
||||
const { app, repository, database, scan, setObserved } = await setup();
|
||||
const first = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
||||
|
||||
@@ -290,6 +290,14 @@ test("loadConfig accepts only an absolute generic model key file", () => {
|
||||
.toThrow(/model credential configuration is invalid/);
|
||||
});
|
||||
|
||||
test("loadConfig accepts only an absolute runtime installation descriptor path", () => {
|
||||
expect(loadConfig({
|
||||
THT_INSTALLATION_CONFIG_FILE: "/run/thothii-installation/thothii-installation.yaml",
|
||||
}).installationConfigFile).toBe("/run/thothii-installation/thothii-installation.yaml");
|
||||
expect(() => loadConfig({ THT_INSTALLATION_CONFIG_FILE: "host/thothii-installation.yaml" }))
|
||||
.toThrow("installation configuration is invalid");
|
||||
});
|
||||
|
||||
test("loadConfig accepts a file-backed catalog role and rejects partial catalog configuration", () => {
|
||||
expect(loadConfig({
|
||||
THT_CATALOG_DB_HOST: "catalog-db",
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import { buildApp } from "../src/app.js";
|
||||
import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
|
||||
import {
|
||||
loadMetadataGenerationModels,
|
||||
MetadataGenerationModelUnavailableError,
|
||||
} from "../src/catalog/metadata-generation-models.js";
|
||||
import { loadConfig } from "../src/config.js";
|
||||
import type { WorkspaceRegistry } from "../src/workspaces/registry.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function metadataConfiguration(
|
||||
metadataGeneration: string,
|
||||
secrets = "OPENAI_API_KEY=raw-provider-secret\n",
|
||||
) {
|
||||
const root = mkdtempSync(join(tmpdir(), "thothii-metadata-models-"));
|
||||
roots.push(root);
|
||||
const installationFile = join(root, "thothii-installation.yaml");
|
||||
const secretsFile = join(root, "thothii.secrets");
|
||||
writeFileSync(installationFile, metadataGeneration, { mode: 0o600 });
|
||||
writeFileSync(secretsFile, secrets, { mode: 0o600 });
|
||||
chmodSync(installationFile, 0o600);
|
||||
chmodSync(secretsFile, 0o600);
|
||||
return { installationFile, secretsFile };
|
||||
}
|
||||
|
||||
function appFor(installationFile: string, secretsFile: string) {
|
||||
const config = loadConfig({
|
||||
NODE_ENV: "test",
|
||||
THT_HARNESS_DIR: "/missing",
|
||||
THT_INSTALLATION_CONFIG_FILE: installationFile,
|
||||
THT_SECRETS_FILE: secretsFile,
|
||||
PI_PROVIDER: "unrelated-pi-provider",
|
||||
PI_MODEL: "unrelated-pi-model",
|
||||
});
|
||||
return buildApp(config, {
|
||||
thtRunner: {} as never,
|
||||
workspaceRegistry: { list: vi.fn(async () => []) } as unknown as WorkspaceRegistry,
|
||||
workspaceDiagnoser: vi.fn(),
|
||||
catalogRepository: new MemoryCatalogRepository(),
|
||||
});
|
||||
}
|
||||
|
||||
test("exposes only safe metadata-generation choices and their configured default", async () => {
|
||||
const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration:
|
||||
default: openai-mini
|
||||
models:
|
||||
- id: openai-mini
|
||||
label: OpenAI Mini
|
||||
litellm:
|
||||
provider: openai
|
||||
model: gpt-4.1-mini
|
||||
endpoint:
|
||||
baseUrl: https://api.openai.example/v1
|
||||
apiVersion: "2026-08-01"
|
||||
apiKeyEnv: OPENAI_API_KEY
|
||||
`);
|
||||
const app = appFor(installationFile, secretsFile);
|
||||
|
||||
const response = await app.inject({ method: "GET", url: "/catalog/metadata-generation/models" });
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.json()).toEqual({
|
||||
models: [{ id: "openai-mini", label: "OpenAI Mini" }],
|
||||
default: "openai-mini",
|
||||
});
|
||||
expect(response.body).not.toMatch(/openai\/gpt|gpt-4\.1|api\.openai|OPENAI_API_KEY|raw-provider-secret/);
|
||||
await app.close();
|
||||
});
|
||||
|
||||
test("rejects an unprotected installation descriptor", () => {
|
||||
const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration:
|
||||
default: openai-mini
|
||||
models:
|
||||
- id: openai-mini
|
||||
label: OpenAI Mini
|
||||
litellm: {provider: openai, model: gpt-4.1-mini}
|
||||
apiKeyEnv: OPENAI_API_KEY
|
||||
`);
|
||||
chmodSync(installationFile, 0o644);
|
||||
|
||||
expect(() => loadMetadataGenerationModels({ installationFile, secretsFile }))
|
||||
.toThrow("metadata-generation installation is unavailable");
|
||||
});
|
||||
|
||||
test("returns an empty safe catalog when no metadata-generation model is configured", async () => {
|
||||
const { installationFile, secretsFile } = metadataConfiguration("profile: local\n");
|
||||
const app = appFor(installationFile, secretsFile);
|
||||
|
||||
const response = await app.inject({ method: "GET", url: "/catalog/metadata-generation/models" });
|
||||
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.json()).toEqual({ models: [], default: null });
|
||||
await app.close();
|
||||
});
|
||||
|
||||
test("resolves only a configured selection for the later generation boundary", () => {
|
||||
const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration:
|
||||
default: openai-mini
|
||||
models:
|
||||
- id: openai-mini
|
||||
label: OpenAI Mini
|
||||
litellm:
|
||||
provider: openai
|
||||
model: gpt-4.1-mini
|
||||
endpoint: {baseUrl: https://api.openai.example/v1, apiVersion: "2026-08-01"}
|
||||
apiKeyEnv: OPENAI_API_KEY
|
||||
`);
|
||||
const models = loadMetadataGenerationModels({ installationFile, secretsFile });
|
||||
|
||||
expect(models.resolve("openai-mini")).toEqual({
|
||||
id: "openai-mini",
|
||||
provider: "openai",
|
||||
model: "gpt-4.1-mini",
|
||||
endpoint: { baseUrl: "https://api.openai.example/v1", apiVersion: "2026-08-01" },
|
||||
apiKeyEnv: "OPENAI_API_KEY",
|
||||
apiKey: "raw-provider-secret",
|
||||
});
|
||||
expect(() => models.resolve("unknown-model")).toThrow(MetadataGenerationModelUnavailableError);
|
||||
});
|
||||
|
||||
test("loads DeepSeek, GLM, and an explicit keyless Qwen endpoint from installation setup", () => {
|
||||
const { installationFile, secretsFile } = metadataConfiguration(`metadataGeneration:
|
||||
default: glm-53
|
||||
models:
|
||||
- id: deepseek-v4-pro
|
||||
label: DeepSeek V4 Pro
|
||||
litellm: {provider: deepseek, model: deepseek-v4-pro}
|
||||
apiKeyEnv: DEEPSEEK_API_KEY
|
||||
- id: glm-53
|
||||
label: GLM 5.3
|
||||
litellm:
|
||||
provider: openai
|
||||
model: glm-5.3
|
||||
endpoint: {baseUrl: https://api.z.ai/api/coding/paas/v4}
|
||||
apiKeyEnv: ZAI_API_KEY
|
||||
- id: qwen-36
|
||||
label: Qwen 3.6
|
||||
litellm:
|
||||
provider: openai
|
||||
model: qwen3.6-35b-a3b
|
||||
disableThinking: true
|
||||
endpoint: {baseUrl: https://models.internal.example/v1}
|
||||
`, "DEEPSEEK_API_KEY=deepseek-secret\nZAI_API_KEY=zai-secret\n");
|
||||
|
||||
const models = loadMetadataGenerationModels({ installationFile, secretsFile });
|
||||
|
||||
expect(models.catalog()).toEqual({
|
||||
models: [
|
||||
{ id: "deepseek-v4-pro", label: "DeepSeek V4 Pro" },
|
||||
{ id: "glm-53", label: "GLM 5.3" },
|
||||
{ id: "qwen-36", label: "Qwen 3.6" },
|
||||
],
|
||||
default: "glm-53",
|
||||
});
|
||||
expect(models.resolve("deepseek-v4-pro")).toMatchObject({
|
||||
apiKeyEnv: "DEEPSEEK_API_KEY",
|
||||
apiKey: "deepseek-secret",
|
||||
});
|
||||
expect(models.resolve("qwen-36")).toEqual({
|
||||
id: "qwen-36",
|
||||
provider: "openai",
|
||||
model: "qwen3.6-35b-a3b",
|
||||
disableThinking: true,
|
||||
endpoint: { baseUrl: "https://models.internal.example/v1" },
|
||||
});
|
||||
});
|
||||
|
||||
test("loads an explicit keyless endpoint without a secret bundle", () => {
|
||||
const { installationFile } = metadataConfiguration(`metadataGeneration:
|
||||
default: qwen-36
|
||||
models:
|
||||
- id: qwen-36
|
||||
label: Qwen 3.6
|
||||
litellm:
|
||||
provider: openai
|
||||
model: qwen3.6-35b-a3b
|
||||
disableThinking: true
|
||||
endpoint: {baseUrl: https://models.internal.example/v1}
|
||||
`);
|
||||
|
||||
expect(loadMetadataGenerationModels({ installationFile }).resolve("qwen-36")).toEqual({
|
||||
id: "qwen-36",
|
||||
provider: "openai",
|
||||
model: "qwen3.6-35b-a3b",
|
||||
disableThinking: true,
|
||||
endpoint: { baseUrl: "https://models.internal.example/v1" },
|
||||
});
|
||||
});
|
||||
|
||||
test.each([
|
||||
["invalid YAML", "metadataGeneration: [\n", "OPENAI_API_KEY=secret\n", /invalid YAML/],
|
||||
["duplicate ids", `metadataGeneration:
|
||||
default: openai-mini
|
||||
models:
|
||||
- {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY}
|
||||
- {id: openai-mini, label: Two, litellm: {provider: openai, model: gpt-4.1}, apiKeyEnv: OPENAI_API_KEY}
|
||||
`, "OPENAI_API_KEY=secret\n", /model id "openai-mini" is duplicated/],
|
||||
["missing default", `metadataGeneration:
|
||||
models:
|
||||
- {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY}
|
||||
`, "OPENAI_API_KEY=secret\n", /default is required/],
|
||||
["unknown default", `metadataGeneration:
|
||||
default: absent
|
||||
models:
|
||||
- {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY}
|
||||
`, "OPENAI_API_KEY=secret\n", /default "absent" is not configured/],
|
||||
["malformed settings", `metadataGeneration:
|
||||
default: openai-mini
|
||||
models:
|
||||
- {id: openai-mini, label: One, litellm: {provider: "open ai", model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY}
|
||||
`, "OPENAI_API_KEY=secret\n", /configuration is invalid/],
|
||||
["malformed endpoint", `metadataGeneration:
|
||||
default: openai-mini
|
||||
models:
|
||||
- id: openai-mini
|
||||
label: One
|
||||
litellm: {provider: openai, model: gpt-4.1-mini, endpoint: {baseUrl: not-a-url}}
|
||||
apiKeyEnv: OPENAI_API_KEY
|
||||
`, "OPENAI_API_KEY=secret\n", /configuration is invalid/],
|
||||
["keyless hosted model without endpoint", `metadataGeneration:
|
||||
default: openai-mini
|
||||
models:
|
||||
- {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}}
|
||||
`, "", /configuration is invalid/],
|
||||
["disable thinking without endpoint", `metadataGeneration:
|
||||
default: openai-mini
|
||||
models:
|
||||
- id: openai-mini
|
||||
label: One
|
||||
litellm: {provider: openai, model: gpt-4.1-mini, disableThinking: true}
|
||||
apiKeyEnv: OPENAI_API_KEY
|
||||
`, "OPENAI_API_KEY=secret\n", /configuration is invalid/],
|
||||
["unallowed secret reference", `metadataGeneration:
|
||||
default: openai-mini
|
||||
models:
|
||||
- {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: THT_DWH_API_KEY}
|
||||
`, "THT_DWH_API_KEY=secret\n", /configuration is invalid/],
|
||||
["missing referenced secret", `metadataGeneration:
|
||||
default: openai-mini
|
||||
models:
|
||||
- {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY}
|
||||
`, "THT_DWH_API_KEY=secret\n", /secret "OPENAI_API_KEY" is missing/],
|
||||
["unusable referenced secret", `metadataGeneration:
|
||||
default: openai-mini
|
||||
models:
|
||||
- {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY}
|
||||
`, "OPENAI_API_KEY=secret with whitespace\n", /secret "OPENAI_API_KEY" is unusable/],
|
||||
] as const)("rejects %s metadata-generation configuration", (_name, yaml, secrets, expected) => {
|
||||
const { installationFile, secretsFile } = metadataConfiguration(yaml, secrets);
|
||||
expect(() => loadMetadataGenerationModels({ installationFile, secretsFile })).toThrow(expected);
|
||||
});
|
||||
|
||||
test("rejects a missing secret-bundle declaration for configured models", () => {
|
||||
const { installationFile } = metadataConfiguration(`metadataGeneration:
|
||||
default: openai-mini
|
||||
models:
|
||||
- {id: openai-mini, label: One, litellm: {provider: openai, model: gpt-4.1-mini}, apiKeyEnv: OPENAI_API_KEY}
|
||||
`);
|
||||
|
||||
expect(() => loadMetadataGenerationModels({ installationFile }))
|
||||
.toThrow("metadata-generation keyed models require THT_SECRETS_FILE");
|
||||
});
|
||||
@@ -0,0 +1,256 @@
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, test } from "vitest";
|
||||
import {
|
||||
ModelCompletionProviderError,
|
||||
PythonModelCompleter,
|
||||
} from "../src/catalog/model-completer.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function helper(source: string, options: { terminationGraceMs?: number } = {}) {
|
||||
const root = mkdtempSync(join(tmpdir(), "thothii-model-completer-"));
|
||||
roots.push(root);
|
||||
writeFileSync(join(root, "fake_completion_helper.py"), source, "utf8");
|
||||
return new PythonModelCompleter({
|
||||
pythonExecutable: "python3",
|
||||
cwd: root,
|
||||
helperModule: "fake_completion_helper",
|
||||
timeoutMs: 5_000,
|
||||
...options,
|
||||
});
|
||||
}
|
||||
|
||||
async function waitUntil(predicate: () => boolean, timeoutMs = 2_000): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (!predicate()) {
|
||||
if (Date.now() >= deadline) throw new Error("condition was not met before timeout");
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
}
|
||||
|
||||
test("uses the short-lived Python helper stdin/stdout protocol without process arguments", async () => {
|
||||
const completer = helper(`
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
request = json.loads(sys.stdin.read())
|
||||
pathlib.Path("request.json").write_text(
|
||||
json.dumps({"request": request, "argv": sys.argv}, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
sys.stdout.write(json.dumps({"ok": True, "content": "Descrizione italiana"}))
|
||||
`);
|
||||
|
||||
const content = await completer.complete({
|
||||
model: {
|
||||
id: "openai-mini",
|
||||
provider: "openai",
|
||||
model: "gpt-4.1-mini",
|
||||
endpoint: { baseUrl: "https://models.example.test/v1", apiVersion: "2026-08-01" },
|
||||
apiKeyEnv: "OPENAI_API_KEY",
|
||||
apiKey: "test-provider-secret",
|
||||
},
|
||||
messages: [
|
||||
{ role: "system", content: "Return one description." },
|
||||
{ role: "user", content: "Private metadata prompt." },
|
||||
],
|
||||
signal: new AbortController().signal,
|
||||
});
|
||||
|
||||
expect(content).toBe("Descrizione italiana");
|
||||
const captured = JSON.parse(readFileSync(join(roots[0]!, "request.json"), "utf8"));
|
||||
expect(captured.request).toEqual({
|
||||
model: "openai/gpt-4.1-mini",
|
||||
api_key: "test-provider-secret",
|
||||
messages: [
|
||||
{ role: "system", content: "Return one description." },
|
||||
{ role: "user", content: "Private metadata prompt." },
|
||||
],
|
||||
api_base: "https://models.example.test/v1",
|
||||
api_version: "2026-08-01",
|
||||
});
|
||||
expect(JSON.stringify(captured.argv)).not.toMatch(/test-provider-secret|Private metadata prompt/);
|
||||
});
|
||||
|
||||
test("omits api_key for an explicitly configured keyless endpoint", async () => {
|
||||
const completer = helper(`
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
request = json.loads(sys.stdin.read())
|
||||
pathlib.Path("request.json").write_text(json.dumps(request, sort_keys=True), encoding="utf-8")
|
||||
sys.stdout.write(json.dumps({"ok": True, "content": "Descrizione Qwen"}))
|
||||
`);
|
||||
|
||||
await expect(completer.complete({
|
||||
model: {
|
||||
id: "qwen-36",
|
||||
provider: "openai",
|
||||
model: "qwen3.6-35b-a3b",
|
||||
disableThinking: true,
|
||||
endpoint: { baseUrl: "https://models.internal.example/v1" },
|
||||
},
|
||||
messages: [{ role: "user", content: "Describe invented metadata." }],
|
||||
signal: new AbortController().signal,
|
||||
})).resolves.toBe("Descrizione Qwen");
|
||||
|
||||
expect(JSON.parse(readFileSync(join(roots[0]!, "request.json"), "utf8"))).toEqual({
|
||||
model: "openai/qwen3.6-35b-a3b",
|
||||
messages: [{ role: "user", content: "Describe invented metadata." }],
|
||||
api_base: "https://models.internal.example/v1",
|
||||
disable_thinking: true,
|
||||
});
|
||||
});
|
||||
|
||||
test("normalizes helper failures and rejects non-pristine stdout without leaking diagnostics", async () => {
|
||||
const secret = "test-provider-secret";
|
||||
const prompt = "private metadata prompt";
|
||||
const completers = [
|
||||
helper(`
|
||||
import json
|
||||
import sys
|
||||
request = json.loads(sys.stdin.read())
|
||||
print(request["api_key"] + " " + request["messages"][0]["content"], file=sys.stderr)
|
||||
sys.stdout.write(json.dumps({"ok": False, "error": "provider_failure"}))
|
||||
`),
|
||||
helper(`
|
||||
import json
|
||||
import sys
|
||||
sys.stdin.read()
|
||||
sys.stdout.write(json.dumps({"ok": True, "content": "first"}) + "\\n" + json.dumps({"ok": True, "content": "second"}))
|
||||
`),
|
||||
];
|
||||
|
||||
for (const completer of completers) {
|
||||
let failure: unknown;
|
||||
try {
|
||||
await completer.complete({
|
||||
model: {
|
||||
id: "openai-mini",
|
||||
provider: "openai",
|
||||
model: "gpt-4.1-mini",
|
||||
apiKeyEnv: "OPENAI_API_KEY",
|
||||
apiKey: secret,
|
||||
},
|
||||
messages: [{ role: "user", content: prompt }],
|
||||
signal: new AbortController().signal,
|
||||
});
|
||||
} catch (error) {
|
||||
failure = error;
|
||||
}
|
||||
expect(failure).toBeInstanceOf(ModelCompletionProviderError);
|
||||
expect(String(failure)).not.toMatch(new RegExp(`${secret}|${prompt}`));
|
||||
}
|
||||
});
|
||||
|
||||
test("aborting a completion terminates its Python helper and returns a cancellation error", async () => {
|
||||
const completer = helper(`
|
||||
import os
|
||||
import pathlib
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.stdin.read()
|
||||
|
||||
def terminate(_signum, _frame):
|
||||
pathlib.Path("terminated.txt").write_text("SIGTERM", encoding="utf-8")
|
||||
raise SystemExit(0)
|
||||
|
||||
signal.signal(signal.SIGTERM, terminate)
|
||||
pathlib.Path("pid.txt").write_text(str(os.getpid()), encoding="utf-8")
|
||||
while True:
|
||||
time.sleep(0.05)
|
||||
`);
|
||||
const controller = new AbortController();
|
||||
const completion = completer.complete({
|
||||
model: {
|
||||
id: "openai-mini",
|
||||
provider: "openai",
|
||||
model: "gpt-4.1-mini",
|
||||
apiKeyEnv: "OPENAI_API_KEY",
|
||||
apiKey: "test-provider-secret",
|
||||
},
|
||||
messages: [{ role: "user", content: "Private metadata prompt." }],
|
||||
signal: controller.signal,
|
||||
});
|
||||
const observed = completion.then(
|
||||
() => undefined,
|
||||
(error: unknown) => error,
|
||||
);
|
||||
const root = roots[0]!;
|
||||
await waitUntil(() => existsSync(join(root, "pid.txt")));
|
||||
const pid = Number(readFileSync(join(root, "pid.txt"), "utf8"));
|
||||
|
||||
controller.abort();
|
||||
|
||||
await expect(observed).resolves.toMatchObject({ name: "ModelCompletionCancelledError" });
|
||||
await waitUntil(() => {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return false;
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
});
|
||||
expect(readFileSync(join(root, "terminated.txt"), "utf8")).toBe("SIGTERM");
|
||||
});
|
||||
|
||||
test("aborting escalates to SIGKILL when the Python helper does not exit after SIGTERM", async () => {
|
||||
const completer = helper(`
|
||||
import os
|
||||
import pathlib
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.stdin.read()
|
||||
|
||||
def ignore_term(_signum, _frame):
|
||||
pathlib.Path("sigterm.txt").write_text("received", encoding="utf-8")
|
||||
|
||||
signal.signal(signal.SIGTERM, ignore_term)
|
||||
pathlib.Path("pid.txt").write_text(str(os.getpid()), encoding="utf-8")
|
||||
while True:
|
||||
time.sleep(0.05)
|
||||
`, { terminationGraceMs: 25 });
|
||||
const controller = new AbortController();
|
||||
const observed = completer.complete({
|
||||
model: {
|
||||
id: "openai-mini",
|
||||
provider: "openai",
|
||||
model: "gpt-4.1-mini",
|
||||
apiKeyEnv: "OPENAI_API_KEY",
|
||||
apiKey: "test-provider-secret",
|
||||
},
|
||||
messages: [{ role: "user", content: "Private metadata prompt." }],
|
||||
signal: controller.signal,
|
||||
}).then(
|
||||
() => undefined,
|
||||
(error: unknown) => error,
|
||||
);
|
||||
const root = roots[0]!;
|
||||
await waitUntil(() => existsSync(join(root, "pid.txt")));
|
||||
const pid = Number(readFileSync(join(root, "pid.txt"), "utf8"));
|
||||
|
||||
controller.abort();
|
||||
|
||||
await expect(observed).resolves.toMatchObject({ name: "ModelCompletionCancelledError" });
|
||||
expect(readFileSync(join(root, "sigterm.txt"), "utf8")).toBe("received");
|
||||
await waitUntil(() => {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return false;
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -2,7 +2,12 @@ import { afterEach, expect, test } from "vitest";
|
||||
import { chmodSync, mkdtempSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { loadSecretBundle, loadSecretBundleWithFs, secretValue } from "../src/config/secret-bundle.js";
|
||||
import {
|
||||
loadSecretBundle,
|
||||
loadSecretBundleWithFs,
|
||||
METADATA_GENERATION_SECRET_KEYS,
|
||||
secretValue,
|
||||
} from "../src/config/secret-bundle.js";
|
||||
|
||||
const dirs: string[] = [];
|
||||
afterEach(() => { for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); });
|
||||
@@ -22,6 +27,15 @@ test("parses comments, blank lines and values containing equals", () => {
|
||||
]));
|
||||
});
|
||||
|
||||
test("existing secret consumers accept a bundle containing an allowed metadata-model key", () => {
|
||||
const file = bundle("THT_DWH_API_KEY=dwh-secret\nOPENAI_API_KEY=metadata-secret\n");
|
||||
expect(secretValue({ secretsFile: file }, "THT_DWH_API_KEY")).toBe("dwh-secret");
|
||||
});
|
||||
|
||||
test.each(METADATA_GENERATION_SECRET_KEYS)("accepts audited metadata-model key %s", (name) => {
|
||||
expect(loadSecretBundle(bundle(`${name}=metadata-secret\n`)).get(name)).toBe("metadata-secret");
|
||||
});
|
||||
|
||||
test("accepts the fixed OIDC and Authentik secret references", () => {
|
||||
const file = bundle("THT_OIDC_CLIENT_SECRET=oidc-secret\nTHT_AUTHENTIK_API_TOKEN=authentik-token\n");
|
||||
expect(loadSecretBundle(file)).toEqual(new Map([
|
||||
|
||||
Reference in New Issue
Block a user